From 4dface66707688ea440a6a7569795a85631d1ff0 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Sun, 26 Jun 2016 10:27:31 +0200 Subject: [PATCH 01/17] first draft (from v1) --- .../CoreBundle/Helper/ContentProxy.php | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 8019df42a..ddeffa77d 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -151,4 +151,172 @@ class ContentProxy { return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']); } + + /** + * Changing pictures URL in article content. + */ + public static function filterPicture($content, $url, $id) + { + $matches = array(); + $processing_pictures = array(); // list of processing image to avoid processing the same pictures twice + preg_match_all('#<\s*(img)[^>]+src="([^"]*)"[^>]*>#Si', $content, $matches, PREG_SET_ORDER); + foreach ($matches as $i => $link) { + $link[1] = trim($link[1]); + if (!preg_match('#^(([a-z]+://)|(\#))#', $link[1])) { + $absolute_path = self::_getAbsoluteLink($link[2], $url); + $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); + $directory = self::_createAssetsDirectory($id); + $fullpath = $directory.'/'.$filename; + + if (in_array($absolute_path, $processing_pictures) === true) { + // replace picture's URL only if processing is OK : already processing -> go to next picture + continue; + } + + if (self::_downloadPictures($absolute_path, $fullpath) === true) { + $content = str_replace($matches[$i][2], Tools::getPocheUrl().$fullpath, $content); + } + + $processing_pictures[] = $absolute_path; + } + } + + return $content; + } + + /** + * Get absolute URL. + */ + private static function _getAbsoluteLink($relativeLink, $url) + { + /* return if already absolute URL */ + if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { + return $relativeLink; + } + + /* queries and anchors */ + if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { + return $url.$relativeLink; + } + + /* parse base URL and convert to local variables: + $scheme, $host, $path */ + extract(parse_url($url)); + + /* remove non-directory element from path */ + $path = preg_replace('#/[^/]*$#', '', $path); + + /* destroy path if relative url points to root */ + if ($relativeLink[0] == '/') { + $path = ''; + } + + /* dirty absolute URL */ + $abs = $host.$path.'/'.$relativeLink; + + /* replace '//' or '/./' or '/foo/../' with '/' */ + $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); + for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { + } + + /* absolute URL is ready! */ + return $scheme.'://'.$abs; + } + + /** + * Downloading pictures. + * + * @return bool true if the download and processing is OK, false else + */ + private static function _downloadPictures($absolute_path, $fullpath) + { + $rawdata = Tools::getFile($absolute_path); + $fullpath = urldecode($fullpath); + + if (file_exists($fullpath)) { + unlink($fullpath); + } + + // check extension + $file_ext = strrchr($fullpath, '.'); + $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); + if (!(in_array($file_ext, $whitelist))) { + Tools::logm('processed image with not allowed extension. Skipping '.$fullpath); + + return false; + } + + // check headers + $imageinfo = getimagesize($absolute_path); + if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { + Tools::logm('processed image with bad header. Skipping '.$fullpath); + + return false; + } + + // regenerate image + $im = imagecreatefromstring($rawdata); + if ($im === false) { + Tools::logm('error while regenerating image '.$fullpath); + + return false; + } + + switch ($imageinfo['mime']) { + case 'image/gif': + $result = imagegif($im, $fullpath); + break; + case 'image/jpeg': + case 'image/jpg': + $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); + break; + case 'image/png': + $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); + break; + } + imagedestroy($im); + + return $result; + } + + /** + * Create a directory for an article. + * + * @param $id ID of the article + * + * @return string + */ + private static function _createAssetsDirectory($id) + { + $assets_path = ABS_PATH; + if (!is_dir($assets_path)) { + mkdir($assets_path, 0715); + } + + $article_directory = $assets_path.$id; + if (!is_dir($article_directory)) { + mkdir($article_directory, 0715); + } + + return $article_directory; + } + + /** + * Remove the directory. + * + * @param $directory + * + * @return bool + */ + public static function removeDirectory($directory) + { + if (is_dir($directory)) { + $files = array_diff(scandir($directory), array('.', '..')); + foreach ($files as $file) { + (is_dir("$directory/$file")) ? self::removeDirectory("$directory/$file") : unlink("$directory/$file"); + } + + return rmdir($directory); + } + } } From 419214d7221e0821ef2b73eb2b3db816ed0cf173 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 28 Jun 2016 19:07:55 +0200 Subject: [PATCH 02/17] Download pictures successfully Needs to rewrite them properly (get base url) --- composer.json | 3 +- .../CoreBundle/Helper/ContentProxy.php | 178 +----------------- .../CoreBundle/Helper/DownloadImages.php | 147 +++++++++++++++ 3 files changed, 158 insertions(+), 170 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Helper/DownloadImages.php diff --git a/composer.json b/composer.json index ebc0a7dc5..e7b30fa1e 100644 --- a/composer.json +++ b/composer.json @@ -82,7 +82,8 @@ "white-october/pagerfanta-bundle": "^1.0", "php-amqplib/rabbitmq-bundle": "^1.8", "predis/predis": "^1.0", - "javibravo/simpleue": "^1.0" + "javibravo/simpleue": "^1.0", + "symfony/dom-crawler": "^3.1" }, "require-dev": { "doctrine/doctrine-fixtures-bundle": "~2.2", diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index ddeffa77d..bbad705fa 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -65,7 +65,7 @@ class ContentProxy $entry->setUrl($content['url'] ?: $url); $entry->setTitle($title); - $entry->setContent($html); + $entry->setLanguage($content['language']); $entry->setMimetype($content['content_type']); $entry->setReadingTime(Utils::getReadingTime($html)); @@ -75,6 +75,14 @@ class ContentProxy $entry->setDomainName($domainName); } + if (true) { + $this->logger->log('debug','Starting to download images'); + $downloadImages = new DownloadImages($html, $url, $this->logger); + $html = $downloadImages->process(); + } + + $entry->setContent($html); + if (isset($content['open_graph']['og_image'])) { $entry->setPreviewPicture($content['open_graph']['og_image']); } @@ -151,172 +159,4 @@ class ContentProxy { return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']); } - - /** - * Changing pictures URL in article content. - */ - public static function filterPicture($content, $url, $id) - { - $matches = array(); - $processing_pictures = array(); // list of processing image to avoid processing the same pictures twice - preg_match_all('#<\s*(img)[^>]+src="([^"]*)"[^>]*>#Si', $content, $matches, PREG_SET_ORDER); - foreach ($matches as $i => $link) { - $link[1] = trim($link[1]); - if (!preg_match('#^(([a-z]+://)|(\#))#', $link[1])) { - $absolute_path = self::_getAbsoluteLink($link[2], $url); - $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); - $directory = self::_createAssetsDirectory($id); - $fullpath = $directory.'/'.$filename; - - if (in_array($absolute_path, $processing_pictures) === true) { - // replace picture's URL only if processing is OK : already processing -> go to next picture - continue; - } - - if (self::_downloadPictures($absolute_path, $fullpath) === true) { - $content = str_replace($matches[$i][2], Tools::getPocheUrl().$fullpath, $content); - } - - $processing_pictures[] = $absolute_path; - } - } - - return $content; - } - - /** - * Get absolute URL. - */ - private static function _getAbsoluteLink($relativeLink, $url) - { - /* return if already absolute URL */ - if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { - return $relativeLink; - } - - /* queries and anchors */ - if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { - return $url.$relativeLink; - } - - /* parse base URL and convert to local variables: - $scheme, $host, $path */ - extract(parse_url($url)); - - /* remove non-directory element from path */ - $path = preg_replace('#/[^/]*$#', '', $path); - - /* destroy path if relative url points to root */ - if ($relativeLink[0] == '/') { - $path = ''; - } - - /* dirty absolute URL */ - $abs = $host.$path.'/'.$relativeLink; - - /* replace '//' or '/./' or '/foo/../' with '/' */ - $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); - for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { - } - - /* absolute URL is ready! */ - return $scheme.'://'.$abs; - } - - /** - * Downloading pictures. - * - * @return bool true if the download and processing is OK, false else - */ - private static function _downloadPictures($absolute_path, $fullpath) - { - $rawdata = Tools::getFile($absolute_path); - $fullpath = urldecode($fullpath); - - if (file_exists($fullpath)) { - unlink($fullpath); - } - - // check extension - $file_ext = strrchr($fullpath, '.'); - $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); - if (!(in_array($file_ext, $whitelist))) { - Tools::logm('processed image with not allowed extension. Skipping '.$fullpath); - - return false; - } - - // check headers - $imageinfo = getimagesize($absolute_path); - if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { - Tools::logm('processed image with bad header. Skipping '.$fullpath); - - return false; - } - - // regenerate image - $im = imagecreatefromstring($rawdata); - if ($im === false) { - Tools::logm('error while regenerating image '.$fullpath); - - return false; - } - - switch ($imageinfo['mime']) { - case 'image/gif': - $result = imagegif($im, $fullpath); - break; - case 'image/jpeg': - case 'image/jpg': - $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); - break; - case 'image/png': - $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); - break; - } - imagedestroy($im); - - return $result; - } - - /** - * Create a directory for an article. - * - * @param $id ID of the article - * - * @return string - */ - private static function _createAssetsDirectory($id) - { - $assets_path = ABS_PATH; - if (!is_dir($assets_path)) { - mkdir($assets_path, 0715); - } - - $article_directory = $assets_path.$id; - if (!is_dir($article_directory)) { - mkdir($article_directory, 0715); - } - - return $article_directory; - } - - /** - * Remove the directory. - * - * @param $directory - * - * @return bool - */ - public static function removeDirectory($directory) - { - if (is_dir($directory)) { - $files = array_diff(scandir($directory), array('.', '..')); - foreach ($files as $file) { - (is_dir("$directory/$file")) ? self::removeDirectory("$directory/$file") : unlink("$directory/$file"); - } - - return rmdir($directory); - } - } } diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php new file mode 100644 index 000000000..32a9dbb2f --- /dev/null +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -0,0 +1,147 @@ +html = $html; + $this->url = $url; + $this->setFolder(); + $this->logger = $logger; + } + + public function setFolder($folder = "assets/images") { + // if folder doesn't exist, attempt to create one and store the folder name in property $folder + if(!file_exists($folder)) { + mkdir($folder); + } + $this->folder = $folder; + } + + public function process() { + //instantiate the symfony DomCrawler Component + $crawler = new Crawler($this->html); + // create an array of all scrapped image links + $this->logger->log('debug', 'Finding images inside document'); + $result = $crawler + ->filterXpath('//img') + ->extract(array('src')); + + // download and save the image to the folder + foreach ($result as $image) { + $file = file_get_contents($image); + + // Checks + $absolute_path = self::getAbsoluteLink($image, $this->url); + $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); + $fullpath = $this->folder."/".$filename; + self::checks($file, $fullpath, $absolute_path); + $this->html = str_replace($image, $fullpath, $this->html); + } + + return $this->html; + } + + private function checks($rawdata, $fullpath, $absolute_path) { + $fullpath = urldecode($fullpath); + + if (file_exists($fullpath)) { + unlink($fullpath); + } + + // check extension + $this->logger->log('debug','Checking extension'); + + $file_ext = strrchr($fullpath, '.'); + $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); + if (!(in_array($file_ext, $whitelist))) { + $this->logger->log('debug','processed image with not allowed extension. Skipping '.$fullpath); + + return false; + } + + // check headers + $this->logger->log('debug','Checking headers'); + $imageinfo = getimagesize($absolute_path); + if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { + $this->logger->log('debug','processed image with bad header. Skipping '.$fullpath); + + return false; + } + + // regenerate image + $this->logger->log('debug','regenerating image'); + $im = imagecreatefromstring($rawdata); + if ($im === false) { + $this->logger->log('error','error while regenerating image '.$fullpath); + + return false; + } + + switch ($imageinfo['mime']) { + case 'image/gif': + $result = imagegif($im, $fullpath); + $this->logger->log('debug','Re-creating gif'); + break; + case 'image/jpeg': + case 'image/jpg': + $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); + $this->logger->log('debug','Re-creating jpg'); + break; + case 'image/png': + $this->logger->log('debug','Re-creating png'); + $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); + break; + } + imagedestroy($im); + + return $result; + } + + private static function getAbsoluteLink($relativeLink, $url) + { + /* return if already absolute URL */ + if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { + return $relativeLink; + } + + /* queries and anchors */ + if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { + return $url.$relativeLink; + } + + /* parse base URL and convert to local variables: + $scheme, $host, $path */ + extract(parse_url($url)); + + /* remove non-directory element from path */ + $path = preg_replace('#/[^/]*$#', '', $path); + + /* destroy path if relative url points to root */ + if ($relativeLink[0] == '/') { + $path = ''; + } + + /* dirty absolute URL */ + $abs = $host.$path.'/'.$relativeLink; + + /* replace '//' or '/./' or '/foo/../' with '/' */ + $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); + for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { + } + + /* absolute URL is ready! */ + return $scheme.'://'.$abs; + } +} From 94654765cca6771c2f54eeaa056b7e65f3353105 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 28 Jun 2016 22:06:00 +0200 Subject: [PATCH 03/17] Working --- .../CoreBundle/Helper/DownloadImages.php | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 32a9dbb2f..14f0aa1bf 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -6,6 +6,9 @@ use Psr\Log\LoggerInterface as Logger; use Symfony\Component\DomCrawler\Crawler; define('REGENERATE_PICTURES_QUALITY', 75); +define('HTTP_PORT', 80); +define('SSL_PORT', 443); +define('BASE_URL',''); class DownloadImages { private $folder; @@ -47,7 +50,7 @@ class DownloadImages { $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); $fullpath = $this->folder."/".$filename; self::checks($file, $fullpath, $absolute_path); - $this->html = str_replace($image, $fullpath, $this->html); + $this->html = str_replace($image, self::getPocheUrl() . '/' . $fullpath, $this->html); } return $this->html; @@ -144,4 +147,43 @@ class DownloadImages { /* absolute URL is ready! */ return $scheme.'://'.$abs; } + + public static function getPocheUrl() + { + $baseUrl = ""; + $https = (!empty($_SERVER['HTTPS']) + && (strtolower($_SERVER['HTTPS']) == 'on')) + || (isset($_SERVER["SERVER_PORT"]) + && $_SERVER["SERVER_PORT"] == '443') // HTTPS detection. + || (isset($_SERVER["SERVER_PORT"]) //Custom HTTPS port detection + && $_SERVER["SERVER_PORT"] == SSL_PORT) + || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) + && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https'); + $serverport = (!isset($_SERVER["SERVER_PORT"]) + || $_SERVER["SERVER_PORT"] == '80' + || $_SERVER["SERVER_PORT"] == HTTP_PORT + || ($https && $_SERVER["SERVER_PORT"] == '443') + || ($https && $_SERVER["SERVER_PORT"]==SSL_PORT) //Custom HTTPS port detection + ? '' : ':' . $_SERVER["SERVER_PORT"]); + + if (isset($_SERVER["HTTP_X_FORWARDED_PORT"])) { + $serverport = ':' . $_SERVER["HTTP_X_FORWARDED_PORT"]; + } + // $scriptname = str_replace('/index.php', '/', $_SERVER["SCRIPT_NAME"]); + // if (!isset($_SERVER["HTTP_HOST"])) { + // return $scriptname; + // } + $host = (isset($_SERVER['HTTP_X_FORWARDED_HOST']) ? $_SERVER['HTTP_X_FORWARDED_HOST'] : (isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : $_SERVER['SERVER_NAME'])); + if (strpos($host, ':') !== false) { + $serverport = ''; + } + // check if BASE_URL is configured + if(BASE_URL) { + $baseUrl = BASE_URL; + } else { + $baseUrl = 'http' . ($https ? 's' : '') . '://' . $host . $serverport; + } + return $baseUrl; + + } } From 156bf62758080153668a65db611c4241d0fc8a00 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 22 Oct 2016 09:22:30 +0200 Subject: [PATCH 04/17] CS --- .../CoreBundle/Helper/ContentProxy.php | 2 +- .../CoreBundle/Helper/DownloadImages.php | 77 ++++++++++--------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index bbad705fa..8ed11205f 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -76,7 +76,7 @@ class ContentProxy } if (true) { - $this->logger->log('debug','Starting to download images'); + $this->logger->log('debug', 'Starting to download images'); $downloadImages = new DownloadImages($html, $url, $this->logger); $html = $downloadImages->process(); } diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 14f0aa1bf..e23e0c556 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -8,31 +8,35 @@ use Symfony\Component\DomCrawler\Crawler; define('REGENERATE_PICTURES_QUALITY', 75); define('HTTP_PORT', 80); define('SSL_PORT', 443); -define('BASE_URL',''); +define('BASE_URL', ''); -class DownloadImages { +class DownloadImages +{ private $folder; private $url; private $html; private $fileName; private $logger; - public function __construct($html, $url, Logger $logger) { + public function __construct($html, $url, Logger $logger) + { $this->html = $html; $this->url = $url; $this->setFolder(); $this->logger = $logger; } - public function setFolder($folder = "assets/images") { + public function setFolder($folder = 'assets/images') + { // if folder doesn't exist, attempt to create one and store the folder name in property $folder - if(!file_exists($folder)) { + if (!file_exists($folder)) { mkdir($folder); } $this->folder = $folder; } - public function process() { + public function process() + { //instantiate the symfony DomCrawler Component $crawler = new Crawler($this->html); // create an array of all scrapped image links @@ -48,15 +52,16 @@ class DownloadImages { // Checks $absolute_path = self::getAbsoluteLink($image, $this->url); $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); - $fullpath = $this->folder."/".$filename; + $fullpath = $this->folder.'/'.$filename; self::checks($file, $fullpath, $absolute_path); - $this->html = str_replace($image, self::getPocheUrl() . '/' . $fullpath, $this->html); + $this->html = str_replace($image, self::getPocheUrl().'/'.$fullpath, $this->html); } return $this->html; } - private function checks($rawdata, $fullpath, $absolute_path) { + private function checks($rawdata, $fullpath, $absolute_path) + { $fullpath = urldecode($fullpath); if (file_exists($fullpath)) { @@ -64,30 +69,30 @@ class DownloadImages { } // check extension - $this->logger->log('debug','Checking extension'); + $this->logger->log('debug', 'Checking extension'); $file_ext = strrchr($fullpath, '.'); $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); if (!(in_array($file_ext, $whitelist))) { - $this->logger->log('debug','processed image with not allowed extension. Skipping '.$fullpath); + $this->logger->log('debug', 'processed image with not allowed extension. Skipping '.$fullpath); return false; } // check headers - $this->logger->log('debug','Checking headers'); + $this->logger->log('debug', 'Checking headers'); $imageinfo = getimagesize($absolute_path); if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { - $this->logger->log('debug','processed image with bad header. Skipping '.$fullpath); + $this->logger->log('debug', 'processed image with bad header. Skipping '.$fullpath); return false; } // regenerate image - $this->logger->log('debug','regenerating image'); + $this->logger->log('debug', 'regenerating image'); $im = imagecreatefromstring($rawdata); if ($im === false) { - $this->logger->log('error','error while regenerating image '.$fullpath); + $this->logger->log('error', 'error while regenerating image '.$fullpath); return false; } @@ -95,15 +100,15 @@ class DownloadImages { switch ($imageinfo['mime']) { case 'image/gif': $result = imagegif($im, $fullpath); - $this->logger->log('debug','Re-creating gif'); + $this->logger->log('debug', 'Re-creating gif'); break; case 'image/jpeg': case 'image/jpg': $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); - $this->logger->log('debug','Re-creating jpg'); + $this->logger->log('debug', 'Re-creating jpg'); break; case 'image/png': - $this->logger->log('debug','Re-creating png'); + $this->logger->log('debug', 'Re-creating png'); $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); break; } @@ -150,24 +155,24 @@ class DownloadImages { public static function getPocheUrl() { - $baseUrl = ""; + $baseUrl = ''; $https = (!empty($_SERVER['HTTPS']) && (strtolower($_SERVER['HTTPS']) == 'on')) - || (isset($_SERVER["SERVER_PORT"]) - && $_SERVER["SERVER_PORT"] == '443') // HTTPS detection. - || (isset($_SERVER["SERVER_PORT"]) //Custom HTTPS port detection - && $_SERVER["SERVER_PORT"] == SSL_PORT) + || (isset($_SERVER['SERVER_PORT']) + && $_SERVER['SERVER_PORT'] == '443') // HTTPS detection. + || (isset($_SERVER['SERVER_PORT']) //Custom HTTPS port detection + && $_SERVER['SERVER_PORT'] == SSL_PORT) || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https'); - $serverport = (!isset($_SERVER["SERVER_PORT"]) - || $_SERVER["SERVER_PORT"] == '80' - || $_SERVER["SERVER_PORT"] == HTTP_PORT - || ($https && $_SERVER["SERVER_PORT"] == '443') - || ($https && $_SERVER["SERVER_PORT"]==SSL_PORT) //Custom HTTPS port detection - ? '' : ':' . $_SERVER["SERVER_PORT"]); - - if (isset($_SERVER["HTTP_X_FORWARDED_PORT"])) { - $serverport = ':' . $_SERVER["HTTP_X_FORWARDED_PORT"]; + $serverport = (!isset($_SERVER['SERVER_PORT']) + || $_SERVER['SERVER_PORT'] == '80' + || $_SERVER['SERVER_PORT'] == HTTP_PORT + || ($https && $_SERVER['SERVER_PORT'] == '443') + || ($https && $_SERVER['SERVER_PORT'] == SSL_PORT) //Custom HTTPS port detection + ? '' : ':'.$_SERVER['SERVER_PORT']); + + if (isset($_SERVER['HTTP_X_FORWARDED_PORT'])) { + $serverport = ':'.$_SERVER['HTTP_X_FORWARDED_PORT']; } // $scriptname = str_replace('/index.php', '/', $_SERVER["SCRIPT_NAME"]); // if (!isset($_SERVER["HTTP_HOST"])) { @@ -178,12 +183,12 @@ class DownloadImages { $serverport = ''; } // check if BASE_URL is configured - if(BASE_URL) { + if (BASE_URL) { $baseUrl = BASE_URL; } else { - $baseUrl = 'http' . ($https ? 's' : '') . '://' . $host . $serverport; + $baseUrl = 'http'.($https ? 's' : '').'://'.$host.$serverport; } - return $baseUrl; - + + return $baseUrl; } } From 535bfcbe80de5d697b768c3a657214fdeff0eac3 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 09:58:39 +0100 Subject: [PATCH 05/17] Move related event things in Event folder --- .gitignore | 3 ++- app/config/services.yml | 4 ++-- .../{EventListener => Event/Listener}/LocaleListener.php | 2 +- .../{EventListener => Event/Listener}/UserLocaleListener.php | 2 +- .../{ => Event}/Subscriber/SQLiteCascadeDeleteSubscriber.php | 2 +- .../{ => Event}/Subscriber/TablePrefixSubscriber.php | 2 +- src/Wallabag/CoreBundle/Resources/config/services.yml | 4 ++-- data/assets/.gitignore => web/assets/images/.gitkeep | 0 8 files changed, 10 insertions(+), 9 deletions(-) rename src/Wallabag/CoreBundle/{EventListener => Event/Listener}/LocaleListener.php (96%) rename src/Wallabag/CoreBundle/{EventListener => Event/Listener}/UserLocaleListener.php (95%) rename src/Wallabag/CoreBundle/{ => Event}/Subscriber/SQLiteCascadeDeleteSubscriber.php (97%) rename src/Wallabag/CoreBundle/{ => Event}/Subscriber/TablePrefixSubscriber.php (97%) rename data/assets/.gitignore => web/assets/images/.gitkeep (100%) diff --git a/.gitignore b/.gitignore index 32b0fbbb5..84fb95d7e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ web/uploads/ !web/bundles web/bundles/* !web/bundles/wallabagcore +/web/assets/images/* +!web/assets/images/.gitkeep # Build /app/build @@ -34,7 +36,6 @@ web/bundles/* /composer.phar # Data for wallabag -data/assets/* data/db/wallabag*.sqlite # Docker container logs and data diff --git a/app/config/services.yml b/app/config/services.yml index a57ef0f3d..9a1ce80b4 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -32,13 +32,13 @@ services: - { name: twig.extension } wallabag.locale_listener: - class: Wallabag\CoreBundle\EventListener\LocaleListener + class: Wallabag\CoreBundle\Event\Listener\LocaleListener arguments: ["%kernel.default_locale%"] tags: - { name: kernel.event_subscriber } wallabag.user_locale_listener: - class: Wallabag\CoreBundle\EventListener\UserLocaleListener + class: Wallabag\CoreBundle\Event\Listener\UserLocaleListener arguments: ["@session"] tags: - { name: kernel.event_listener, event: security.interactive_login, method: onInteractiveLogin } diff --git a/src/Wallabag/CoreBundle/EventListener/LocaleListener.php b/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php similarity index 96% rename from src/Wallabag/CoreBundle/EventListener/LocaleListener.php rename to src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php index a1c7e5ab6..b435d99ec 100644 --- a/src/Wallabag/CoreBundle/EventListener/LocaleListener.php +++ b/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php @@ -1,6 +1,6 @@ Date: Sun, 30 Oct 2016 09:58:53 +0100 Subject: [PATCH 06/17] Cleanup --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 4 ++-- src/Wallabag/ImportBundle/Resources/config/services.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 8ed11205f..219b90d30 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -3,7 +3,7 @@ namespace Wallabag\CoreBundle\Helper; use Graby\Graby; -use Psr\Log\LoggerInterface as Logger; +use Psr\Log\LoggerInterface; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\CoreBundle\Tools\Utils; @@ -20,7 +20,7 @@ class ContentProxy protected $logger; protected $tagRepository; - public function __construct(Graby $graby, RuleBasedTagger $tagger, TagRepository $tagRepository, Logger $logger) + public function __construct(Graby $graby, RuleBasedTagger $tagger, TagRepository $tagRepository, LoggerInterface $logger) { $this->graby = $graby; $this->tagger = $tagger; diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 89adc71b3..d600be0f8 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -20,7 +20,6 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" - - "@craue_config" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] From 7f55941856549a3f5f45c42fdc171d66ff7ee297 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 10:48:29 +0100 Subject: [PATCH 07/17] Use doctrine event to download images --- .../Subscriber/DownloadImagesSubscriber.php | 129 +++++++++ .../CoreBundle/Helper/ContentProxy.php | 6 - .../CoreBundle/Helper/DownloadImages.php | 256 +++++++++--------- .../CoreBundle/Resources/config/services.yml | 19 ++ .../CoreBundle/Helper/DownloadImagesTest.php | 123 +++++++++ .../Wallabag/CoreBundle/fixtures/unnamed.png | Bin 0 -> 3688 bytes 6 files changed, 401 insertions(+), 132 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php create mode 100644 tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php create mode 100644 tests/Wallabag/CoreBundle/fixtures/unnamed.png diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php new file mode 100644 index 000000000..654edf310 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -0,0 +1,129 @@ +downloadImages = $downloadImages; + $this->configClass = $configClass; + $this->logger = $logger; + } + + public function getSubscribedEvents() + { + return array( + 'prePersist', + 'preUpdate', + ); + } + + /** + * In case of an entry has been updated. + * We won't update the content field if it wasn't updated. + * + * @param LifecycleEventArgs $args + */ + public function preUpdate(LifecycleEventArgs $args) + { + $entity = $args->getEntity(); + + if (!$entity instanceof Entry) { + return; + } + + $em = $args->getEntityManager(); + + // field content has been updated + if ($args->hasChangedField('content')) { + $html = $this->downloadImages($em, $entity); + + if (null !== $html) { + $args->setNewValue('content', $html); + } + } + + // field preview picture has been updated + if ($args->hasChangedField('previewPicture')) { + $previewPicture = $this->downloadPreviewImage($em, $entity); + + if (null !== $previewPicture) { + $entity->setPreviewPicture($previewPicture); + } + } + } + + /** + * When a new entry is saved. + * + * @param LifecycleEventArgs $args + */ + public function prePersist(LifecycleEventArgs $args) + { + $entity = $args->getEntity(); + + if (!$entity instanceof Entry) { + return; + } + + $config = new $this->configClass(); + $config->setEntityManager($args->getEntityManager()); + + // update all images inside the html + $html = $this->downloadImages($config, $entity); + if (null !== $html) { + $entity->setContent($html); + } + + // update preview picture + $previewPicture = $this->downloadPreviewImage($config, $entity); + if (null !== $previewPicture) { + $entity->setPreviewPicture($previewPicture); + } + } + + public function downloadImages(Config $config, Entry $entry) + { + // if ($config->get('download_images_with_rabbitmq')) { + + // } else if ($config->get('download_images_with_redis')) { + + // } + + return $this->downloadImages->processHtml( + $entry->getContent(), + $entry->getUrl() + ); + } + + public function downloadPreviewImage(Config $config, Entry $entry) + { + // if ($config->get('download_images_with_rabbitmq')) { + + // } else if ($config->get('download_images_with_redis')) { + + // } + + return $this->downloadImages->processSingleImage( + $entry->getPreviewPicture(), + $entry->getUrl() + ); + } +} diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 219b90d30..d90d3dc8c 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -75,12 +75,6 @@ class ContentProxy $entry->setDomainName($domainName); } - if (true) { - $this->logger->log('debug', 'Starting to download images'); - $downloadImages = new DownloadImages($html, $url, $this->logger); - $html = $downloadImages->process(); - } - $entry->setContent($html); if (isset($content['open_graph']['og_image'])) { diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index e23e0c556..426cbe481 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -2,193 +2,197 @@ namespace Wallabag\CoreBundle\Helper; -use Psr\Log\LoggerInterface as Logger; +use Psr\Log\LoggerInterface; use Symfony\Component\DomCrawler\Crawler; - -define('REGENERATE_PICTURES_QUALITY', 75); -define('HTTP_PORT', 80); -define('SSL_PORT', 443); -define('BASE_URL', ''); +use GuzzleHttp\Client; +use Symfony\Component\HttpFoundation\File\MimeType\MimeTypeExtensionGuesser; class DownloadImages { - private $folder; - private $url; - private $html; - private $fileName; - private $logger; + const REGENERATE_PICTURES_QUALITY = 80; - public function __construct($html, $url, Logger $logger) + private $client; + private $baseFolder; + private $logger; + private $mimeGuesser; + + public function __construct(Client $client, $baseFolder, LoggerInterface $logger) { - $this->html = $html; - $this->url = $url; - $this->setFolder(); + $this->client = $client; + $this->baseFolder = $baseFolder; $this->logger = $logger; + $this->mimeGuesser = new MimeTypeExtensionGuesser(); + + $this->setFolder(); } - public function setFolder($folder = 'assets/images') + /** + * Setup base folder where all images are going to be saved. + */ + private function setFolder() { // if folder doesn't exist, attempt to create one and store the folder name in property $folder - if (!file_exists($folder)) { - mkdir($folder); + if (!file_exists($this->baseFolder)) { + mkdir($this->baseFolder, 0777, true); } - $this->folder = $folder; } - public function process() + /** + * Process the html and extract image from it, save them to local and return the updated html. + * + * @param string $html + * @param string $url Used as a base path for relative image and folder + * + * @return string + */ + public function processHtml($html, $url) { - //instantiate the symfony DomCrawler Component - $crawler = new Crawler($this->html); - // create an array of all scrapped image links - $this->logger->log('debug', 'Finding images inside document'); + $crawler = new Crawler($html); $result = $crawler ->filterXpath('//img') ->extract(array('src')); + $relativePath = $this->getRelativePath($url); + // download and save the image to the folder foreach ($result as $image) { - $file = file_get_contents($image); + $imagePath = $this->processSingleImage($image, $url, $relativePath); - // Checks - $absolute_path = self::getAbsoluteLink($image, $this->url); - $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); - $fullpath = $this->folder.'/'.$filename; - self::checks($file, $fullpath, $absolute_path); - $this->html = str_replace($image, self::getPocheUrl().'/'.$fullpath, $this->html); + if (false === $imagePath) { + continue; + } + + $html = str_replace($image, $imagePath, $html); } - return $this->html; + return $html; } - private function checks($rawdata, $fullpath, $absolute_path) + /** + * Process a single image: + * - retrieve it + * - re-saved it (for security reason) + * - return the new local path. + * + * @param string $imagePath Path to the image to retrieve + * @param string $url Url from where the image were found + * @param string $relativePath Relative local path to saved the image + * + * @return string Relative url to access the image from the web + */ + public function processSingleImage($imagePath, $url, $relativePath = null) { - $fullpath = urldecode($fullpath); - - if (file_exists($fullpath)) { - unlink($fullpath); + if (null == $relativePath) { + $relativePath = $this->getRelativePath($url); } - // check extension - $this->logger->log('debug', 'Checking extension'); + $folderPath = $this->baseFolder.'/'.$relativePath; - $file_ext = strrchr($fullpath, '.'); - $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); - if (!(in_array($file_ext, $whitelist))) { - $this->logger->log('debug', 'processed image with not allowed extension. Skipping '.$fullpath); + // build image path + $absolutePath = $this->getAbsoluteLink($url, $imagePath); + if (false === $absolutePath) { + $this->logger->log('debug', 'Can not determine the absolute path for that image, skipping.'); return false; } - // check headers - $this->logger->log('debug', 'Checking headers'); - $imageinfo = getimagesize($absolute_path); - if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { - $this->logger->log('debug', 'processed image with bad header. Skipping '.$fullpath); + $res = $this->client->get( + $absolutePath, + ['exceptions' => false] + ); + + $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); + $this->logger->log('debug', 'Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); + if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'])) { + $this->logger->log('debug', 'Processed image with not allowed extension. Skipping '.$imagePath); return false; } + $hashImage = hash('crc32', $absolutePath); + $localPath = $folderPath.'/'.$hashImage.'.'.$ext; + + try { + $im = imagecreatefromstring($res->getBody()); + } catch (\Exception $e) { + $im = false; + } - // regenerate image - $this->logger->log('debug', 'regenerating image'); - $im = imagecreatefromstring($rawdata); if ($im === false) { - $this->logger->log('error', 'error while regenerating image '.$fullpath); + $this->logger->log('error', 'Error while regenerating image', ['path' => $localPath]); return false; } - switch ($imageinfo['mime']) { - case 'image/gif': - $result = imagegif($im, $fullpath); + switch ($ext) { + case 'gif': + $result = imagegif($im, $localPath); $this->logger->log('debug', 'Re-creating gif'); break; - case 'image/jpeg': - case 'image/jpg': - $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); + case 'jpeg': + case 'jpg': + $result = imagejpeg($im, $localPath, self::REGENERATE_PICTURES_QUALITY); $this->logger->log('debug', 'Re-creating jpg'); break; - case 'image/png': + case 'png': + $result = imagepng($im, $localPath, ceil(self::REGENERATE_PICTURES_QUALITY / 100 * 9)); $this->logger->log('debug', 'Re-creating png'); - $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); - break; } + imagedestroy($im); - return $result; + return '/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; } - private static function getAbsoluteLink($relativeLink, $url) + /** + * Generate the folder where we are going to save images based on the entry url. + * + * @param string $url + * + * @return string + */ + private function getRelativePath($url) { - /* return if already absolute URL */ - if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { - return $relativeLink; + $hashUrl = hash('crc32', $url); + $relativePath = $hashUrl[0].'/'.$hashUrl[1].'/'.$hashUrl; + $folderPath = $this->baseFolder.'/'.$relativePath; + + if (!file_exists($folderPath)) { + mkdir($folderPath, 0777, true); } - /* queries and anchors */ - if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { - return $url.$relativeLink; - } + $this->logger->log('debug', 'Folder used for that url', ['folder' => $folderPath, 'url' => $url]); - /* parse base URL and convert to local variables: - $scheme, $host, $path */ - extract(parse_url($url)); - - /* remove non-directory element from path */ - $path = preg_replace('#/[^/]*$#', '', $path); - - /* destroy path if relative url points to root */ - if ($relativeLink[0] == '/') { - $path = ''; - } - - /* dirty absolute URL */ - $abs = $host.$path.'/'.$relativeLink; - - /* replace '//' or '/./' or '/foo/../' with '/' */ - $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); - for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { - } - - /* absolute URL is ready! */ - return $scheme.'://'.$abs; + return $relativePath; } - public static function getPocheUrl() + /** + * Make an $url absolute based on the $base. + * + * @see Graby->makeAbsoluteStr + * + * @param string $base Base url + * @param string $url Url to make it absolute + * + * @return false|string + */ + private function getAbsoluteLink($base, $url) { - $baseUrl = ''; - $https = (!empty($_SERVER['HTTPS']) - && (strtolower($_SERVER['HTTPS']) == 'on')) - || (isset($_SERVER['SERVER_PORT']) - && $_SERVER['SERVER_PORT'] == '443') // HTTPS detection. - || (isset($_SERVER['SERVER_PORT']) //Custom HTTPS port detection - && $_SERVER['SERVER_PORT'] == SSL_PORT) - || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) - && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https'); - $serverport = (!isset($_SERVER['SERVER_PORT']) - || $_SERVER['SERVER_PORT'] == '80' - || $_SERVER['SERVER_PORT'] == HTTP_PORT - || ($https && $_SERVER['SERVER_PORT'] == '443') - || ($https && $_SERVER['SERVER_PORT'] == SSL_PORT) //Custom HTTPS port detection - ? '' : ':'.$_SERVER['SERVER_PORT']); - - if (isset($_SERVER['HTTP_X_FORWARDED_PORT'])) { - $serverport = ':'.$_SERVER['HTTP_X_FORWARDED_PORT']; - } - // $scriptname = str_replace('/index.php', '/', $_SERVER["SCRIPT_NAME"]); - // if (!isset($_SERVER["HTTP_HOST"])) { - // return $scriptname; - // } - $host = (isset($_SERVER['HTTP_X_FORWARDED_HOST']) ? $_SERVER['HTTP_X_FORWARDED_HOST'] : (isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : $_SERVER['SERVER_NAME'])); - if (strpos($host, ':') !== false) { - $serverport = ''; - } - // check if BASE_URL is configured - if (BASE_URL) { - $baseUrl = BASE_URL; - } else { - $baseUrl = 'http'.($https ? 's' : '').'://'.$host.$serverport; + if (preg_match('!^https?://!i', $url)) { + // already absolute + return $url; } - return $baseUrl; + $base = new \SimplePie_IRI($base); + + // remove '//' in URL path (causes URLs not to resolve properly) + if (isset($base->ipath)) { + $base->ipath = preg_replace('!//+!', '/', $base->ipath); + } + + if ($absolute = \SimplePie_IRI::absolutize($base, $url)) { + return $absolute->get_uri(); + } + + return false; } } diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index 4b7751fe9..1fb81a461 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -136,3 +136,22 @@ services: - "@doctrine" tags: - { name: doctrine.event_subscriber } + + wallabag_core.subscriber.download_images: + class: Wallabag\CoreBundle\Event\Subscriber\DownloadImagesSubscriber + arguments: + - "@wallabag_core.entry.download_images" + - "%craue_config.config.class%" + - "@logger" + tags: + - { name: doctrine.event_subscriber } + + wallabag_core.entry.download_images: + class: Wallabag\CoreBundle\Helper\DownloadImages + arguments: + - "@wallabag_core.entry.download_images.client" + - "%kernel.root_dir%/../web/assets/images" + - "@logger" + + wallabag_core.entry.download_images.client: + class: GuzzleHttp\Client diff --git a/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php b/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php new file mode 100644 index 000000000..0273693ec --- /dev/null +++ b/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php @@ -0,0 +1,123 @@ + 'image/png'], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $res = $download->processHtml('
', 'http://imgur.com/gallery/WxtWY'); + + $this->assertContains('/assets/images/4/2/4258f71e/c638b4c2.png', $res); + } + + public function testProcessHtmlWithBadImage() + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => 'application/json'], Stream::factory('')), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $res = $download->processHtml('
', 'http://imgur.com/gallery/WxtWY'); + + $this->assertContains('http://i.imgur.com/T9qgcHc.jpg', $res, 'Image were not replace because of content-type'); + } + + public function singleImage() + { + return [ + ['image/pjpeg', 'jpeg'], + ['image/jpeg', 'jpeg'], + ['image/png', 'png'], + ['image/gif', 'gif'], + ]; + } + + /** + * @dataProvider singleImage + */ + public function testProcessSingleImage($header, $extension) + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => $header], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $res = $download->processSingleImage('T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + + $this->assertContains('/assets/images/4/2/4258f71e/ebe60399.'.$extension, $res); + } + + public function testProcessSingleImageWithBadImage() + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => 'image/png'], Stream::factory('')), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $res = $download->processSingleImage('http://i.imgur.com/T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + + $this->assertFalse($res, 'Image can not be loaded, so it will not be replaced'); + } + + public function testProcessSingleImageFailAbsolute() + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => 'image/png'], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $res = $download->processSingleImage('/i.imgur.com/T9qgcHc.jpg', 'imgur.com/gallery/WxtWY'); + + $this->assertFalse($res, 'Absolute image can not be determined, so it will not be replaced'); + } +} diff --git a/tests/Wallabag/CoreBundle/fixtures/unnamed.png b/tests/Wallabag/CoreBundle/fixtures/unnamed.png new file mode 100644 index 0000000000000000000000000000000000000000..e6dd9caadb29929e09d80dcc30e593a9b6020e54 GIT binary patch literal 3688 zcmV-u4wvzXP)K!~nK5rQotCocwP|LL#lEZI==N9E75sehvaa z5`t+m{3bX*LnJ{wEjf&lv#l2ZRL*UW2Co6e+Q7KHOQ3t}%7NrbHcHWy7+vhU`kERp zgv)qtt0^I^zzHP;&{TQj*8Lj~G$8CnyYnPyNzE8gKJ4NiS%={)q{!5D`P(0UzN0CD zuosNCf+1Zok6zcma4=~BfB+JiYkL*F^4r;F7fy*gZ1yd$uKnxbRB0HL000-^!hQoQrM0)S_0U)>$u(VtSp)1Y~!1%f6($n#!Z{Z&^zbM#oE6U3kGy>#sN$CM3R z^Tv;7^(|E!;%G_VF|4Zg%?0}o)n767nPkR2w}c0teDR^hD;~UR+N&?!G}5afY2|Pi zXdXF|CG%MGkTU<1k6z>R*o^a58H5laxYuKD+_v+F3l@(UaK*;e_xa2G|GW9KX?N9D z$>FW<-cnX+h^Um;<@=QW50U- zgC#HQgg;yv@^r}&+mQfVmhKkZS|6WscWqp6%5X;7P*VcqzD^|<{0U_jQ z^dLkFvVH+lIA{V0fC*OQJCU-`Y~~hn$^??O-4d&M=|eZ>5CdRD0Z}r7#1q*_*Z|P2 z$QU%y|L2#Enu@Q*R8JKl2t?L_G5CRduWb$T915tM^X9@q=Ws(n0Hy{*J2}$!I%_-i zXqV*}C1kDSNe%iv*Sr@7lOWLI;MBIs%dmZZqA*-!3!_uHmS4slV)3-0D-79OW% zimV%{k1x1k=!juGfsTz=_h_3o?4Et=tFJ%x{gI=)KmOb*aGp(MKmPd0=n>t1_|u2q zd}o^y9P4-tP}-3M0l%^R>*(CMt5&~yQ&UUB3vav)TAiys3I}a7q^w1AM=n}4{@fEM zU0Uf;=&k=zJ9x7H_uJ|ay!5c9-uvL%3x-OszqdsTjLSxXpcK5L-Ufg`tRlSQaP;Qe z-aNr}9&At$D2A?5Aqpv&2m#}zN1I30RIYhtURM_Ay6*S|%Rk=!traeXX+T#|-x7{B zHHCxld-INQw%$bmlt7jON21(GCW?x4*|*B86=KtjBb}W(zTf`iv1gw6W2e)F5Hs(* zZ}ZMSTE1dv8dMoAsdB%5!BxbMb0Y8zsy0pm)T<94W7*R@q|y!Fnyzmni(BH)2Uk+!?p?yv8dF#WoL z7d^jq>57jUB??1@%RAm|i5Mqh)rP$4(d#OE_}IeLt3E!2kWX|GG$#OKAmsXM2NOcZ zPVD=_@)`5*TwTAxhU$W!5wip!ORzHNHbod3BnlzmxTO*Alqr2FrME5^K)?Rdb8~uC1sP)%eY4meMOx)uY`5|)DTO*{C|UOKwFCM@as&0z z7{I^_4|(il;w6MYZb*BZf$s1b=Stb9yy61x$)=hMVT&UmDOe`F`sorF7pB(l!lKpz zRW(4QoEsFDT*xV9WX2@`>A0}$f&>CsId>qWgG5dM&ZE)vDMM|O&cF7cl%7{zd8&l7 z6w{qDekZm6Ao`~SeO^5xfSiL-3UU^s?96mp+xL9KCQh-FoXmV{(*YF*`cx@pMSeJ0 z-LWlM!fDmI5RfFO@|n+C4tM0ZNd_v=1*rBE-=0+w{qvFP?t=ZLegMhfH|eu~Uz#22G_U{o2ca z$)lvyXBe%>yR_3HcV3KZnhd~o-GP(meEC&!?XtUukL{DSPru#rnCjmBjSKI4=d0tf zSa~SFivr+WGZg^)4qEA1$spyd@*Ui7<%RLPrtz16tUk9CyW%SUB=O4Xx-tBMLCt~TcAUKDi z{;fH^^07@<-q`!%i8K9?nvY(*zxRMr0FjV>ri9tEYq%s9|7?A?h{raMKmEJa2#$xU z5ENRPHYIB!&aUSDfBxl?nz7ZLG9Ek>yK`afJJA$O9RzS(U&5*kHyesrv;3a!)sapm zy!+0c6~9{liSkvm6M|X_U8(YDg*c)BLf&rmKJopT||N0DyoAOKz**83!RCq@DJh`<|k&O@bsL=tGMn zF~&3%gmhdd3P6#d%BQ_8Bu*0o+eJEyq=lpf#vw}xc+7|2MA2e0_S=<;6S$>oaN+i#jKy)p>YIf{SBvDp0&uK)EfCf&SfMVZud(zxDB zfAi_64aF@Q*}m`F8CL%iLmcB`@#DWge5B#9_2p~p>cg(dOl`^ClP>I|uimt6&Nag~ zezt4Rj(8$w`+UA+4DW3{0owy3|G}W^=0QNP&*d*~-nH{+;xBs-y?&s%Dd~bM0OA?@ zi^GkGTmFm;iIz+#<7h7HTM^D?GG>wY z@}Xf6%hl|sYPWs8o38j>T@;mxlHgl^IucKr<)y)^hgA(JqOhHug>(tD0KhpIpLbFD zqUj??_4W-NSh@VVVUxdC4w9ZTq3;8CO_()t$dXwThSh|jxC^J(bPJhUiT|=O)e#eL zt&vsTpZK>c?^!yvXJz@1XHQ5~Fv{FymOXOo^glgzRZ=}(+?3q1sqR?AkwZJaw>m*bbD)vaBaZ6h33r6%( z0;XjbO&vGHQEU_$@MztO22~H}?v{A{#)Sw+UatLgKt=y;yT5wui7f-F24C1c+^o^p zKiJh6RX1$izhMt+h!ihdzkg)quz8EGFuZP4G~2?l?)@WIkGW*tqJR1Mlk2zFX$bfL zNL|c`Wa`R%qi-`{<(1Wb|yAghwdwH|~BfJu~lb;Wi;i9@MwJ4})sS%RT@(iVWU zsZjx@E0SL~EGJtUbW~a5g6bi{h3dFg$m=SS#6|9}6aNo!J|kNjX+@3z0000 Date: Sun, 30 Oct 2016 11:27:09 +0100 Subject: [PATCH 08/17] Fixing tests --- .../Subscriber/DownloadImagesSubscriber.php | 31 ++++++++++++++----- .../CoreBundle/Helper/ContentProxy.php | 3 +- .../CoreBundle/Helper/DownloadImages.php | 21 ++++++++----- .../Listener}/LocaleListenerTest.php | 4 +-- .../Listener}/UserLocaleListenerTest.php | 4 +-- .../Subscriber/TablePrefixSubscriberTest.php | 4 +-- .../CoreBundle/Helper/DownloadImagesTest.php | 19 ++++++++++++ 7 files changed, 63 insertions(+), 23 deletions(-) rename tests/Wallabag/CoreBundle/{EventListener => Event/Listener}/LocaleListenerTest.php (96%) rename tests/Wallabag/CoreBundle/{EventListener => Event/Listener}/UserLocaleListenerTest.php (93%) rename tests/Wallabag/CoreBundle/{ => Event}/Subscriber/TablePrefixSubscriberTest.php (97%) diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 654edf310..09f8e9119 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -49,22 +49,23 @@ class DownloadImagesSubscriber implements EventSubscriber return; } - $em = $args->getEntityManager(); + $config = new $this->configClass(); + $config->setEntityManager($args->getEntityManager()); // field content has been updated if ($args->hasChangedField('content')) { - $html = $this->downloadImages($em, $entity); + $html = $this->downloadImages($config, $entity); - if (null !== $html) { + if (false !== $html) { $args->setNewValue('content', $html); } } // field preview picture has been updated if ($args->hasChangedField('previewPicture')) { - $previewPicture = $this->downloadPreviewImage($em, $entity); + $previewPicture = $this->downloadPreviewImage($config, $entity); - if (null !== $previewPicture) { + if (false !== $previewPicture) { $entity->setPreviewPicture($previewPicture); } } @@ -88,17 +89,25 @@ class DownloadImagesSubscriber implements EventSubscriber // update all images inside the html $html = $this->downloadImages($config, $entity); - if (null !== $html) { + if (false !== $html) { $entity->setContent($html); } // update preview picture $previewPicture = $this->downloadPreviewImage($config, $entity); - if (null !== $previewPicture) { + if (false !== $previewPicture) { $entity->setPreviewPicture($previewPicture); } } + /** + * Download all images from the html. + * + * @param Config $config + * @param Entry $entry + * + * @return string|false False in case of async + */ public function downloadImages(Config $config, Entry $entry) { // if ($config->get('download_images_with_rabbitmq')) { @@ -113,6 +122,14 @@ class DownloadImagesSubscriber implements EventSubscriber ); } + /** + * Download the preview picture. + * + * @param Config $config + * @param Entry $entry + * + * @return string|false False in case of async + */ public function downloadPreviewImage(Config $config, Entry $entry) { // if ($config->get('download_images_with_rabbitmq')) { diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index d90d3dc8c..1986ab334 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -65,6 +65,7 @@ class ContentProxy $entry->setUrl($content['url'] ?: $url); $entry->setTitle($title); + $entry->setContent($html); $entry->setLanguage($content['language']); $entry->setMimetype($content['content_type']); @@ -75,8 +76,6 @@ class ContentProxy $entry->setDomainName($domainName); } - $entry->setContent($html); - if (isset($content['open_graph']['og_image'])) { $entry->setPreviewPicture($content['open_graph']['og_image']); } diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 426cbe481..004bb2775 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -91,20 +91,23 @@ class DownloadImages // build image path $absolutePath = $this->getAbsoluteLink($url, $imagePath); if (false === $absolutePath) { - $this->logger->log('debug', 'Can not determine the absolute path for that image, skipping.'); + $this->logger->log('error', 'Can not determine the absolute path for that image, skipping.'); return false; } - $res = $this->client->get( - $absolutePath, - ['exceptions' => false] - ); + try { + $res = $this->client->get($absolutePath); + } catch (\Exception $e) { + $this->logger->log('error', 'Can not retrieve image, skipping.', ['exception' => $e]); + + return false; + } $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); $this->logger->log('debug', 'Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); - if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'])) { - $this->logger->log('debug', 'Processed image with not allowed extension. Skipping '.$imagePath); + if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { + $this->logger->log('error', 'Processed image with not allowed extension. Skipping '.$imagePath); return false; } @@ -117,7 +120,7 @@ class DownloadImages $im = false; } - if ($im === false) { + if (false === $im) { $this->logger->log('error', 'Error while regenerating image', ['path' => $localPath]); return false; @@ -193,6 +196,8 @@ class DownloadImages return $absolute->get_uri(); } + $this->logger->log('error', 'Can not make an absolute link', ['base' => $base, 'url' => $url]); + return false; } } diff --git a/tests/Wallabag/CoreBundle/EventListener/LocaleListenerTest.php b/tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php similarity index 96% rename from tests/Wallabag/CoreBundle/EventListener/LocaleListenerTest.php rename to tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php index 078bb69ae..84a54d3ab 100644 --- a/tests/Wallabag/CoreBundle/EventListener/LocaleListenerTest.php +++ b/tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php @@ -1,6 +1,6 @@ assertContains('/assets/images/4/2/4258f71e/ebe60399.'.$extension, $res); } + public function testProcessSingleImageWithBadUrl() + { + $client = new Client(); + + $mock = new Mock([ + new Response(404, []), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $res = $download->processSingleImage('T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + + $this->assertFalse($res, 'Image can not be found, so it will not be replaced'); + } + public function testProcessSingleImageWithBadImage() { $client = new Client(); From 41ada277f066ea57947bce05bcda63962b7fea55 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 19:50:00 +0100 Subject: [PATCH 09/17] Add instance url to the downloaded images --- .../Event/Subscriber/DownloadImagesSubscriber.php | 4 ++++ src/Wallabag/CoreBundle/Helper/DownloadImages.php | 14 +++++++++++++- .../CoreBundle/Helper/DownloadImagesTest.php | 4 +++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 09f8e9119..0792653e0 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -110,6 +110,8 @@ class DownloadImagesSubscriber implements EventSubscriber */ public function downloadImages(Config $config, Entry $entry) { + $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); + // if ($config->get('download_images_with_rabbitmq')) { // } else if ($config->get('download_images_with_redis')) { @@ -132,6 +134,8 @@ class DownloadImagesSubscriber implements EventSubscriber */ public function downloadPreviewImage(Config $config, Entry $entry) { + $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); + // if ($config->get('download_images_with_rabbitmq')) { // } else if ($config->get('download_images_with_redis')) { diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 004bb2775..e7982c560 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -15,6 +15,7 @@ class DownloadImages private $baseFolder; private $logger; private $mimeGuesser; + private $wallabagUrl; public function __construct(Client $client, $baseFolder, LoggerInterface $logger) { @@ -26,6 +27,17 @@ class DownloadImages $this->setFolder(); } + /** + * Since we can't inject CraueConfig service because it'll generate a circular reference when injected in the subscriber + * we use a different way to inject the current wallabag url. + * + * @param string $url Usually from `$config->get('wallabag_url')` + */ + public function setWallabagUrl($url) + { + $this->wallabagUrl = rtrim($url, '/'); + } + /** * Setup base folder where all images are going to be saved. */ @@ -143,7 +155,7 @@ class DownloadImages imagedestroy($im); - return '/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; + return $this->wallabagUrl.'/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; } /** diff --git a/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php b/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php index e000d6817..33d2e389c 100644 --- a/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php +++ b/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php @@ -27,9 +27,11 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logger = new Logger('test', array($logHandler)); $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); + $download->setWallabagUrl('http://wallabag.io/'); + $res = $download->processHtml('
', 'http://imgur.com/gallery/WxtWY'); - $this->assertContains('/assets/images/4/2/4258f71e/c638b4c2.png', $res); + $this->assertContains('http://wallabag.io/assets/images/4/2/4258f71e/c638b4c2.png', $res); } public function testProcessHtmlWithBadImage() From 309e13c11b54277626f18616c41f68ae9656a403 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 20:12:34 +0100 Subject: [PATCH 10/17] Move settings before Entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because we need wallabag_url to be defined when we’ll insert entries --- src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index a5e1be65f..12f66c19b 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -158,6 +158,6 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface */ public function getOrder() { - return 50; + return 29; } } From d1495dd0a456f0e35a09fb68679ee51f8d17bfe4 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 21:30:45 +0100 Subject: [PATCH 11/17] Ability to enable/disable downloading images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will speed up the test suite because it won’t download everything when we add new entry… Add a custom test with downloading image enabled --- .../CoreBundle/Command/InstallCommand.php | 15 +++++++ .../DataFixtures/ORM/LoadSettingData.php | 15 +++++++ .../Subscriber/DownloadImagesSubscriber.php | 8 ++++ .../Controller/EntryControllerTest.php | 40 +++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 277f85242..aedccfe4c 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -398,6 +398,21 @@ class InstallCommand extends ContainerAwareCommand 'value' => 'wallabag', 'section' => 'misc', ], + [ + 'name' => 'download_images_enabled', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_rabbitmq', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_redis', + 'value' => '0', + 'section' => 'image', + ], ]; foreach ($settings as $setting) { diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index 12f66c19b..70a7a4ac0 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -140,6 +140,21 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface 'value' => 'wallabag', 'section' => 'misc', ], + [ + 'name' => 'download_images_enabled', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_rabbitmq', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_redis', + 'value' => '0', + 'section' => 'image', + ], ]; foreach ($settings as $setting) { diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 0792653e0..3f2d460c1 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -52,6 +52,10 @@ class DownloadImagesSubscriber implements EventSubscriber $config = new $this->configClass(); $config->setEntityManager($args->getEntityManager()); + if (!$config->get('download_images_enabled')) { + return; + } + // field content has been updated if ($args->hasChangedField('content')) { $html = $this->downloadImages($config, $entity); @@ -87,6 +91,10 @@ class DownloadImagesSubscriber implements EventSubscriber $config = new $this->configClass(); $config->setEntityManager($args->getEntityManager()); + if (!$config->get('download_images_enabled')) { + return; + } + // update all images inside the html $html = $this->downloadImages($config, $entity); if (false !== $html) { diff --git a/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php b/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php index 051136503..514e9d89f 100644 --- a/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php +++ b/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php @@ -836,4 +836,44 @@ class EntryControllerTest extends WallabagCoreTestCase $client->request('GET', '/share/'.$content->getUuid()); $this->assertEquals(404, $client->getResponse()->getStatusCode()); } + + public function testNewEntryWithDownloadImagesEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $url = 'http://www.20minutes.fr/montpellier/1952003-20161030-video-car-tombe-panne-rugbymen-perpignan-improvisent-melee-route'; + $client->getContainer()->get('craue_config')->set('download_images_enabled', 1); + + $crawler = $client->request('GET', '/new'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + + $form = $crawler->filter('form[name=entry]')->form(); + + $data = [ + 'entry[url]' => $url, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $em = $client->getContainer() + ->get('doctrine.orm.entity_manager'); + + $entry = $em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->getLoggedInUserId()); + + $this->assertInstanceOf('Wallabag\CoreBundle\Entity\Entry', $entry); + $this->assertEquals($url, $entry->getUrl()); + $this->assertContains('Perpignan', $entry->getTitle()); + $this->assertContains('assets/images/8/e/8ec9229a/d9bc0fcd.jpeg', $entry->getContent()); + + $em->remove($entry); + $em->flush(); + + $client->getContainer()->get('craue_config')->set('download_images_enabled', 0); + } } From aedd6ca0fd212abd07ec59c5fd58ea2ca99198c5 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 31 Oct 2016 13:29:33 +0100 Subject: [PATCH 12/17] Add translations & migration --- .../Version20161031132655.php | 44 +++++++++++++++++++ .../translations/CraueConfigBundle.da.yml | 1 + .../translations/CraueConfigBundle.de.yml | 1 + .../translations/CraueConfigBundle.en.yml | 1 + .../translations/CraueConfigBundle.es.yml | 1 + .../translations/CraueConfigBundle.fa.yml | 1 + .../translations/CraueConfigBundle.fr.yml | 1 + .../translations/CraueConfigBundle.it.yml | 1 + .../translations/CraueConfigBundle.oc.yml | 1 + .../translations/CraueConfigBundle.pl.yml | 1 + .../translations/CraueConfigBundle.ro.yml | 1 + .../translations/CraueConfigBundle.tr.yml | 1 + .../CoreBundle/Command/InstallCommand.php | 14 +----- .../DataFixtures/ORM/LoadSettingData.php | 12 +---- .../Subscriber/DownloadImagesSubscriber.php | 16 ++----- 15 files changed, 62 insertions(+), 35 deletions(-) create mode 100644 app/DoctrineMigrations/Version20161031132655.php diff --git a/app/DoctrineMigrations/Version20161031132655.php b/app/DoctrineMigrations/Version20161031132655.php new file mode 100644 index 000000000..c73644288 --- /dev/null +++ b/app/DoctrineMigrations/Version20161031132655.php @@ -0,0 +1,44 @@ +container = $container; + } + + private function getTable($tableName) + { + return $this->container->getParameter('database_table_prefix') . $tableName; + } + + /** + * @param Schema $schema + */ + public function up(Schema $schema) + { + $this->addSql("INSERT INTO \"".$this->getTable('craue_config_setting')."\" (name, value, section) VALUES ('download_images_enabled', 0, 'misc')"); + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.'); + + $this->addSql("DELETE FROM \"".$this->getTable('craue_config_setting')."\" WHERE name = 'download_images_enabled';"); + } +} diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml index 3e11d675c..7c3237833 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml @@ -29,3 +29,4 @@ piwik_enabled: Aktiver Piwik demo_mode_enabled: "Aktiver demo-indstilling? (anvendes kun til wallabags offentlige demo)" demo_mode_username: "Demobruger" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml index c74b5c1ff..438eb74a2 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml @@ -29,3 +29,4 @@ piwik_enabled: Piwik aktivieren demo_mode_enabled: "Test-Modus aktivieren? (nur für die öffentliche wallabag-Demo genutzt)" demo_mode_username: "Test-Benutzer" share_public: Erlaube eine öffentliche URL für Einträge +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml index 77c09db43..c2f2b3fbb 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml @@ -29,3 +29,4 @@ piwik_enabled: Enable Piwik demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" demo_mode_username: "Demo user" share_public: Allow public url for entries +download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml index baa838493..76feea50d 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml @@ -29,3 +29,4 @@ piwik_enabled: Activar Piwik demo_mode_enabled: "Activar modo demo (sólo usado para la demo de wallabag)" demo_mode_username: "Nombre de usuario demo" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml index b394977e2..30df00863 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml @@ -29,3 +29,4 @@ modify_settings: "اعمال" # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" # demo_mode_username: "Demo user" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml index 31a808804..a60341b34 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml @@ -29,3 +29,4 @@ piwik_enabled: Activer Piwik demo_mode_enabled: "Activer le mode démo ? (utiliser uniquement pour la démo publique de wallabag)" demo_mode_username: "Utilisateur de la démo" share_public: Autoriser une URL publique pour les articles +download_images_enabled: Télécharger les images en local diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml index ba0385561..3ad5f7d09 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml @@ -29,3 +29,4 @@ piwik_enabled: Abilita Piwik demo_mode_enabled: "Abilita modalità demo ? (usato solo per la demo pubblica di wallabag)" demo_mode_username: "Utente Demo" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml index 55249e33b..fd83b4372 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml @@ -29,3 +29,4 @@ piwik_enabled: Activar Piwik demo_mode_enabled: "Activar lo mode demostracion ? (utilizar solament per la demostracion publica de wallabag)" demo_mode_username: "Utilizaire de la demostracion" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml index 42cc5b525..3a63eebb0 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml @@ -29,3 +29,4 @@ piwik_enabled: Włacz Piwik demo_mode_enabled: "Włacz tryb demo? (używany wyłącznie dla publicznej demonstracji Wallabag)" demo_mode_username: "Użytkownik Demonstracyjny" share_public: Zezwalaj na publiczny adres url dla wpisow +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml index 8e72b9555..4fb42e984 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml @@ -29,3 +29,4 @@ modify_settings: "aplică" # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" # demo_mode_username: "Demo user" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml index 55f708433..ebfadf29b 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml @@ -29,3 +29,4 @@ # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" # demo_mode_username: "Demo user" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index aedccfe4c..9fe903577 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -370,7 +370,7 @@ class InstallCommand extends ContainerAwareCommand ], [ 'name' => 'wallabag_url', - 'value' => 'http://v2.wallabag.org', + 'value' => '', 'section' => 'misc', ], [ @@ -401,17 +401,7 @@ class InstallCommand extends ContainerAwareCommand [ 'name' => 'download_images_enabled', 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_rabbitmq', - 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_redis', - 'value' => '0', - 'section' => 'image', + 'section' => 'misc', ], ]; diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index 70a7a4ac0..d0085660f 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -143,17 +143,7 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface [ 'name' => 'download_images_enabled', 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_rabbitmq', - 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_redis', - 'value' => '0', - 'section' => 'image', + 'section' => 'misc', ], ]; diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 3f2d460c1..6fddcea9e 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -111,6 +111,8 @@ class DownloadImagesSubscriber implements EventSubscriber /** * Download all images from the html. * + * @todo If we want to add async download, it should be done in that method + * * @param Config $config * @param Entry $entry * @@ -120,12 +122,6 @@ class DownloadImagesSubscriber implements EventSubscriber { $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - // if ($config->get('download_images_with_rabbitmq')) { - - // } else if ($config->get('download_images_with_redis')) { - - // } - return $this->downloadImages->processHtml( $entry->getContent(), $entry->getUrl() @@ -135,6 +131,8 @@ class DownloadImagesSubscriber implements EventSubscriber /** * Download the preview picture. * + * @todo If we want to add async download, it should be done in that method + * * @param Config $config * @param Entry $entry * @@ -144,12 +142,6 @@ class DownloadImagesSubscriber implements EventSubscriber { $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - // if ($config->get('download_images_with_rabbitmq')) { - - // } else if ($config->get('download_images_with_redis')) { - - // } - return $this->downloadImages->processSingleImage( $entry->getPreviewPicture(), $entry->getUrl() From e0597476d1d5f6a4a7d6ea9b76966465f3d22fb8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 1 Nov 2016 14:49:02 +0100 Subject: [PATCH 13/17] Use custom event instead of Doctrine ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This give us ability to use Entry ID to determine where to store images and it’s then more easy to remove them when we remove the entry. --- .../CoreBundle/Controller/EntryController.php | 14 ++ .../CoreBundle/Event/EntryDeletedEvent.php | 26 ++++ .../CoreBundle/Event/EntrySavedEvent.php | 26 ++++ .../Subscriber/DownloadImagesSubscriber.php | 135 +++++++----------- .../CoreBundle/Helper/DownloadImages.php | 84 ++++++----- .../CoreBundle/Resources/config/services.yml | 6 +- .../Controller/EntryControllerTest.php | 26 +++- .../CoreBundle/Helper/DownloadImagesTest.php | 29 ++-- 8 files changed, 211 insertions(+), 135 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php create mode 100644 src/Wallabag/CoreBundle/Event/EntrySavedEvent.php diff --git a/src/Wallabag/CoreBundle/Controller/EntryController.php b/src/Wallabag/CoreBundle/Controller/EntryController.php index 97bb3d12f..3f4eb17d1 100644 --- a/src/Wallabag/CoreBundle/Controller/EntryController.php +++ b/src/Wallabag/CoreBundle/Controller/EntryController.php @@ -13,6 +13,8 @@ use Wallabag\CoreBundle\Form\Type\EntryFilterType; use Wallabag\CoreBundle\Form\Type\EditEntryType; use Wallabag\CoreBundle\Form\Type\NewEntryType; use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; class EntryController extends Controller { @@ -81,6 +83,9 @@ class EntryController extends Controller $em->persist($entry); $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + return $this->redirect($this->generateUrl('homepage')); } @@ -107,6 +112,9 @@ class EntryController extends Controller $em = $this->getDoctrine()->getManager(); $em->persist($entry); $em->flush(); + + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); } return $this->redirect($this->generateUrl('homepage')); @@ -343,6 +351,9 @@ class EntryController extends Controller $em->persist($entry); $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()])); } @@ -431,6 +442,9 @@ class EntryController extends Controller UrlGeneratorInterface::ABSOLUTE_PATH ); + // entry deleted, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry)); + $em = $this->getDoctrine()->getManager(); $em->remove($entry); $em->flush(); diff --git a/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php b/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php new file mode 100644 index 000000000..e9061d044 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php @@ -0,0 +1,26 @@ +entry = $entry; + } + + public function getEntry() + { + return $this->entry; + } +} diff --git a/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php b/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php new file mode 100644 index 000000000..5fdb52212 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php @@ -0,0 +1,26 @@ +entry = $entry; + } + + public function getEntry() + { + return $this->entry; + } +} diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 6fddcea9e..4ebe837b6 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -2,110 +2,85 @@ namespace Wallabag\CoreBundle\Event\Subscriber; -use Doctrine\Common\EventSubscriber; -use Doctrine\ORM\Event\LifecycleEventArgs; +use Symfony\Component\EventDispatcher\EventSubscriberInterface; use Psr\Log\LoggerInterface; use Wallabag\CoreBundle\Helper\DownloadImages; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; use Doctrine\ORM\EntityManager; -use Craue\ConfigBundle\Util\Config; -class DownloadImagesSubscriber implements EventSubscriber +class DownloadImagesSubscriber implements EventSubscriberInterface { - private $configClass; + private $em; private $downloadImages; + private $enabled; private $logger; - /** - * We inject the class instead of the service otherwise it generates a circular reference with the EntityManager. - * So we build the service ourself when we got the EntityManager (in downloadImages). - */ - public function __construct(DownloadImages $downloadImages, $configClass, LoggerInterface $logger) + public function __construct(EntityManager $em, DownloadImages $downloadImages, $enabled, LoggerInterface $logger) { + $this->em = $em; $this->downloadImages = $downloadImages; - $this->configClass = $configClass; + $this->enabled = $enabled; $this->logger = $logger; } - public function getSubscribedEvents() + public static function getSubscribedEvents() { - return array( - 'prePersist', - 'preUpdate', - ); + return [ + EntrySavedEvent::NAME => 'onEntrySaved', + EntryDeletedEvent::NAME => 'onEntryDeleted', + ]; } /** - * In case of an entry has been updated. - * We won't update the content field if it wasn't updated. + * Download images and updated the data into the entry. * - * @param LifecycleEventArgs $args + * @param EntrySavedEvent $event */ - public function preUpdate(LifecycleEventArgs $args) + public function onEntrySaved(EntrySavedEvent $event) { - $entity = $args->getEntity(); + if (!$this->enabled) { + $this->logger->debug('DownloadImagesSubscriber: disabled.'); - if (!$entity instanceof Entry) { return; } - $config = new $this->configClass(); - $config->setEntityManager($args->getEntityManager()); + $entry = $event->getEntry(); - if (!$config->get('download_images_enabled')) { - return; - } - - // field content has been updated - if ($args->hasChangedField('content')) { - $html = $this->downloadImages($config, $entity); - - if (false !== $html) { - $args->setNewValue('content', $html); - } - } - - // field preview picture has been updated - if ($args->hasChangedField('previewPicture')) { - $previewPicture = $this->downloadPreviewImage($config, $entity); - - if (false !== $previewPicture) { - $entity->setPreviewPicture($previewPicture); - } - } - } - - /** - * When a new entry is saved. - * - * @param LifecycleEventArgs $args - */ - public function prePersist(LifecycleEventArgs $args) - { - $entity = $args->getEntity(); - - if (!$entity instanceof Entry) { - return; - } - - $config = new $this->configClass(); - $config->setEntityManager($args->getEntityManager()); - - if (!$config->get('download_images_enabled')) { - return; - } - - // update all images inside the html - $html = $this->downloadImages($config, $entity); + $html = $this->downloadImages($entry); if (false !== $html) { - $entity->setContent($html); + $this->logger->debug('DownloadImagesSubscriber: updated html.'); + + $entry->setContent($html); } // update preview picture - $previewPicture = $this->downloadPreviewImage($config, $entity); + $previewPicture = $this->downloadPreviewImage($entry); if (false !== $previewPicture) { - $entity->setPreviewPicture($previewPicture); + $this->logger->debug('DownloadImagesSubscriber: update preview picture.'); + + $entry->setPreviewPicture($previewPicture); } + + $this->em->persist($entry); + $this->em->flush(); + } + + /** + * Remove images related to the entry. + * + * @param EntryDeletedEvent $event + */ + public function onEntryDeleted(EntryDeletedEvent $event) + { + if (!$this->enabled) { + $this->logger->debug('DownloadImagesSubscriber: disabled.'); + + return; + } + + $this->downloadImages->removeImages($event->getEntry()->getId()); } /** @@ -113,16 +88,14 @@ class DownloadImagesSubscriber implements EventSubscriber * * @todo If we want to add async download, it should be done in that method * - * @param Config $config - * @param Entry $entry + * @param Entry $entry * * @return string|false False in case of async */ - public function downloadImages(Config $config, Entry $entry) + private function downloadImages(Entry $entry) { - $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - return $this->downloadImages->processHtml( + $entry->getId(), $entry->getContent(), $entry->getUrl() ); @@ -133,16 +106,14 @@ class DownloadImagesSubscriber implements EventSubscriber * * @todo If we want to add async download, it should be done in that method * - * @param Config $config - * @param Entry $entry + * @param Entry $entry * * @return string|false False in case of async */ - public function downloadPreviewImage(Config $config, Entry $entry) + private function downloadPreviewImage(Entry $entry) { - $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - return $this->downloadImages->processSingleImage( + $entry->getId(), $entry->getPreviewPicture(), $entry->getUrl() ); diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index e7982c560..c5298236a 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -6,6 +6,7 @@ use Psr\Log\LoggerInterface; use Symfony\Component\DomCrawler\Crawler; use GuzzleHttp\Client; use Symfony\Component\HttpFoundation\File\MimeType\MimeTypeExtensionGuesser; +use Symfony\Component\Finder\Finder; class DownloadImages { @@ -17,27 +18,17 @@ class DownloadImages private $mimeGuesser; private $wallabagUrl; - public function __construct(Client $client, $baseFolder, LoggerInterface $logger) + public function __construct(Client $client, $baseFolder, $wallabagUrl, LoggerInterface $logger) { $this->client = $client; $this->baseFolder = $baseFolder; + $this->wallabagUrl = rtrim($wallabagUrl, '/'); $this->logger = $logger; $this->mimeGuesser = new MimeTypeExtensionGuesser(); $this->setFolder(); } - /** - * Since we can't inject CraueConfig service because it'll generate a circular reference when injected in the subscriber - * we use a different way to inject the current wallabag url. - * - * @param string $url Usually from `$config->get('wallabag_url')` - */ - public function setWallabagUrl($url) - { - $this->wallabagUrl = rtrim($url, '/'); - } - /** * Setup base folder where all images are going to be saved. */ @@ -52,23 +43,24 @@ class DownloadImages /** * Process the html and extract image from it, save them to local and return the updated html. * + * @param int $entryId ID of the entry * @param string $html - * @param string $url Used as a base path for relative image and folder + * @param string $url Used as a base path for relative image and folder * * @return string */ - public function processHtml($html, $url) + public function processHtml($entryId, $html, $url) { $crawler = new Crawler($html); $result = $crawler ->filterXpath('//img') ->extract(array('src')); - $relativePath = $this->getRelativePath($url); + $relativePath = $this->getRelativePath($entryId); // download and save the image to the folder foreach ($result as $image) { - $imagePath = $this->processSingleImage($image, $url, $relativePath); + $imagePath = $this->processSingleImage($entryId, $image, $url, $relativePath); if (false === $imagePath) { continue; @@ -86,24 +78,27 @@ class DownloadImages * - re-saved it (for security reason) * - return the new local path. * + * @param int $entryId ID of the entry * @param string $imagePath Path to the image to retrieve * @param string $url Url from where the image were found * @param string $relativePath Relative local path to saved the image * * @return string Relative url to access the image from the web */ - public function processSingleImage($imagePath, $url, $relativePath = null) + public function processSingleImage($entryId, $imagePath, $url, $relativePath = null) { - if (null == $relativePath) { - $relativePath = $this->getRelativePath($url); + if (null === $relativePath) { + $relativePath = $this->getRelativePath($entryId); } + $this->logger->debug('DownloadImages: working on image: '.$imagePath); + $folderPath = $this->baseFolder.'/'.$relativePath; // build image path $absolutePath = $this->getAbsoluteLink($url, $imagePath); if (false === $absolutePath) { - $this->logger->log('error', 'Can not determine the absolute path for that image, skipping.'); + $this->logger->error('DownloadImages: Can not determine the absolute path for that image, skipping.'); return false; } @@ -111,15 +106,15 @@ class DownloadImages try { $res = $this->client->get($absolutePath); } catch (\Exception $e) { - $this->logger->log('error', 'Can not retrieve image, skipping.', ['exception' => $e]); + $this->logger->error('DownloadImages: Can not retrieve image, skipping.', ['exception' => $e]); return false; } $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); - $this->logger->log('debug', 'Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); + $this->logger->debug('DownloadImages: Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { - $this->logger->log('error', 'Processed image with not allowed extension. Skipping '.$imagePath); + $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping '.$imagePath); return false; } @@ -133,7 +128,7 @@ class DownloadImages } if (false === $im) { - $this->logger->log('error', 'Error while regenerating image', ['path' => $localPath]); + $this->logger->error('DownloadImages: Error while regenerating image', ['path' => $localPath]); return false; } @@ -141,16 +136,16 @@ class DownloadImages switch ($ext) { case 'gif': $result = imagegif($im, $localPath); - $this->logger->log('debug', 'Re-creating gif'); + $this->logger->debug('DownloadImages: Re-creating gif'); break; case 'jpeg': case 'jpg': $result = imagejpeg($im, $localPath, self::REGENERATE_PICTURES_QUALITY); - $this->logger->log('debug', 'Re-creating jpg'); + $this->logger->debug('DownloadImages: Re-creating jpg'); break; case 'png': $result = imagepng($im, $localPath, ceil(self::REGENERATE_PICTURES_QUALITY / 100 * 9)); - $this->logger->log('debug', 'Re-creating png'); + $this->logger->debug('DownloadImages: Re-creating png'); } imagedestroy($im); @@ -158,24 +153,47 @@ class DownloadImages return $this->wallabagUrl.'/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; } + /** + * Remove all images for the given entry id. + * + * @param int $entryId ID of the entry + */ + public function removeImages($entryId) + { + $relativePath = $this->getRelativePath($entryId); + $folderPath = $this->baseFolder.'/'.$relativePath; + + $finder = new Finder(); + $finder + ->files() + ->ignoreDotFiles(true) + ->in($folderPath); + + foreach ($finder as $file) { + @unlink($file->getRealPath()); + } + + @rmdir($folderPath); + } + /** * Generate the folder where we are going to save images based on the entry url. * - * @param string $url + * @param int $entryId ID of the entry * * @return string */ - private function getRelativePath($url) + private function getRelativePath($entryId) { - $hashUrl = hash('crc32', $url); - $relativePath = $hashUrl[0].'/'.$hashUrl[1].'/'.$hashUrl; + $hashId = hash('crc32', $entryId); + $relativePath = $hashId[0].'/'.$hashId[1].'/'.$hashId; $folderPath = $this->baseFolder.'/'.$relativePath; if (!file_exists($folderPath)) { mkdir($folderPath, 0777, true); } - $this->logger->log('debug', 'Folder used for that url', ['folder' => $folderPath, 'url' => $url]); + $this->logger->debug('DownloadImages: Folder used for that Entry id', ['folder' => $folderPath, 'entryId' => $entryId]); return $relativePath; } @@ -208,7 +226,7 @@ class DownloadImages return $absolute->get_uri(); } - $this->logger->log('error', 'Can not make an absolute link', ['base' => $base, 'url' => $url]); + $this->logger->error('DownloadImages: Can not make an absolute link', ['base' => $base, 'url' => $url]); return false; } diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index 1fb81a461..56d776adb 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -140,17 +140,19 @@ services: wallabag_core.subscriber.download_images: class: Wallabag\CoreBundle\Event\Subscriber\DownloadImagesSubscriber arguments: + - "@doctrine.orm.default_entity_manager" - "@wallabag_core.entry.download_images" - - "%craue_config.config.class%" + - '@=service(''craue_config'').get(''download_images_enabled'')' - "@logger" tags: - - { name: doctrine.event_subscriber } + - { name: kernel.event_subscriber } wallabag_core.entry.download_images: class: Wallabag\CoreBundle\Helper\DownloadImages arguments: - "@wallabag_core.entry.download_images.client" - "%kernel.root_dir%/../web/assets/images" + - '@=service(''craue_config'').get(''wallabag_url'')' - "@logger" wallabag_core.entry.download_images.client: diff --git a/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php b/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php index 514e9d89f..4ab06dbff 100644 --- a/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php +++ b/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php @@ -869,10 +869,30 @@ class EntryControllerTest extends WallabagCoreTestCase $this->assertInstanceOf('Wallabag\CoreBundle\Entity\Entry', $entry); $this->assertEquals($url, $entry->getUrl()); $this->assertContains('Perpignan', $entry->getTitle()); - $this->assertContains('assets/images/8/e/8ec9229a/d9bc0fcd.jpeg', $entry->getContent()); + $this->assertContains('/d9bc0fcd.jpeg', $entry->getContent()); - $em->remove($entry); - $em->flush(); + $client->getContainer()->get('craue_config')->set('download_images_enabled', 0); + } + + /** + * @depends testNewEntryWithDownloadImagesEnabled + */ + public function testRemoveEntryWithDownloadImagesEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $url = 'http://www.20minutes.fr/montpellier/1952003-20161030-video-car-tombe-panne-rugbymen-perpignan-improvisent-melee-route'; + $client->getContainer()->get('craue_config')->set('download_images_enabled', 1); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->getLoggedInUserId()); + + $client->request('GET', '/delete/'.$content->getId()); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); $client->getContainer()->get('craue_config')->set('download_images_enabled', 0); } diff --git a/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php b/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php index 33d2e389c..920c21d99 100644 --- a/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php +++ b/tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php @@ -26,12 +26,11 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logHandler = new TestHandler(); $logger = new Logger('test', array($logHandler)); - $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); - $download->setWallabagUrl('http://wallabag.io/'); + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); - $res = $download->processHtml('
', 'http://imgur.com/gallery/WxtWY'); + $res = $download->processHtml(123, '
', 'http://imgur.com/gallery/WxtWY'); - $this->assertContains('http://wallabag.io/assets/images/4/2/4258f71e/c638b4c2.png', $res); + $this->assertContains('http://wallabag.io/assets/images/9/b/9b0ead26/c638b4c2.png', $res); } public function testProcessHtmlWithBadImage() @@ -47,8 +46,8 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logHandler = new TestHandler(); $logger = new Logger('test', array($logHandler)); - $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); - $res = $download->processHtml('
', 'http://imgur.com/gallery/WxtWY'); + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processHtml(123, '
', 'http://imgur.com/gallery/WxtWY'); $this->assertContains('http://i.imgur.com/T9qgcHc.jpg', $res, 'Image were not replace because of content-type'); } @@ -79,10 +78,10 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logHandler = new TestHandler(); $logger = new Logger('test', array($logHandler)); - $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); - $res = $download->processSingleImage('T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, 'T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); - $this->assertContains('/assets/images/4/2/4258f71e/ebe60399.'.$extension, $res); + $this->assertContains('/assets/images/9/b/9b0ead26/ebe60399.'.$extension, $res); } public function testProcessSingleImageWithBadUrl() @@ -98,8 +97,8 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logHandler = new TestHandler(); $logger = new Logger('test', array($logHandler)); - $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); - $res = $download->processSingleImage('T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, 'T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); $this->assertFalse($res, 'Image can not be found, so it will not be replaced'); } @@ -117,8 +116,8 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logHandler = new TestHandler(); $logger = new Logger('test', array($logHandler)); - $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); - $res = $download->processSingleImage('http://i.imgur.com/T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, 'http://i.imgur.com/T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); $this->assertFalse($res, 'Image can not be loaded, so it will not be replaced'); } @@ -136,8 +135,8 @@ class DownloadImagesTest extends \PHPUnit_Framework_TestCase $logHandler = new TestHandler(); $logger = new Logger('test', array($logHandler)); - $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', $logger); - $res = $download->processSingleImage('/i.imgur.com/T9qgcHc.jpg', 'imgur.com/gallery/WxtWY'); + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, '/i.imgur.com/T9qgcHc.jpg', 'imgur.com/gallery/WxtWY'); $this->assertFalse($res, 'Absolute image can not be determined, so it will not be replaced'); } From 7816eb622df2353cea0ede0a3674d5eb3a01a1a9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:10:23 +0100 Subject: [PATCH 14/17] Add entry.saved event to import & rest --- .../Controller/WallabagRestController.php | 9 ++++- .../Consumer/AbstractConsumer.php | 8 ++++- .../ImportBundle/Import/AbstractImport.php | 24 +++++++++++++- .../ImportBundle/Import/BrowserImport.php | 17 ++++++++++ .../ImportBundle/Import/PocketImport.php | 7 ---- .../ImportBundle/Resources/config/rabbit.yml | 7 ++++ .../ImportBundle/Resources/config/redis.yml | 7 ++++ .../Resources/config/services.yml | 7 ++++ .../Consumer/AMQPEntryConsumerTest.php | 33 +++++++++++++++++-- .../Consumer/RedisEntryConsumerTest.php | 33 +++++++++++++++++-- .../ImportBundle/Import/ChromeImportTest.php | 16 ++++++--- .../ImportBundle/Import/FirefoxImportTest.php | 16 ++++++--- .../Import/InstapaperImportTest.php | 16 ++++++--- .../ImportBundle/Import/PocketImportTest.php | 21 +++++++----- .../Import/ReadabilityImportTest.php | 16 ++++++--- .../Import/WallabagV1ImportTest.php | 16 ++++++--- .../Import/WallabagV2ImportTest.php | 18 +++++++--- 17 files changed, 222 insertions(+), 49 deletions(-) diff --git a/src/Wallabag/ApiBundle/Controller/WallabagRestController.php b/src/Wallabag/ApiBundle/Controller/WallabagRestController.php index a73d44ca2..50652b777 100644 --- a/src/Wallabag/ApiBundle/Controller/WallabagRestController.php +++ b/src/Wallabag/ApiBundle/Controller/WallabagRestController.php @@ -14,6 +14,8 @@ use Symfony\Component\Security\Core\Exception\AccessDeniedException; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\AnnotationBundle\Entity\Annotation; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; class WallabagRestController extends FOSRestController { @@ -233,9 +235,11 @@ class WallabagRestController extends FOSRestController $em = $this->getDoctrine()->getManager(); $em->persist($entry); - $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + $json = $this->get('serializer')->serialize($entry, 'json'); return (new JsonResponse())->setJson($json); @@ -308,6 +312,9 @@ class WallabagRestController extends FOSRestController $this->validateAuthentication(); $this->validateUserAccess($entry->getUser()->getId()); + // entry deleted, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry)); + $em = $this->getDoctrine()->getManager(); $em->remove($entry); $em->flush(); diff --git a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php index b893ea295..aa7ff9144 100644 --- a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php +++ b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php @@ -9,6 +9,8 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractConsumer { @@ -17,11 +19,12 @@ abstract class AbstractConsumer protected $import; protected $logger; - public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null) + public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, EventDispatcherInterface $eventDispatcher, LoggerInterface $logger = null) { $this->em = $em; $this->userRepository = $userRepository; $this->import = $import; + $this->eventDispatcher = $eventDispatcher; $this->logger = $logger ?: new NullLogger(); } @@ -59,6 +62,9 @@ abstract class AbstractConsumer try { $this->em->flush(); + // entry saved, dispatch event about it! + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 764b390a8..1d4a6e27b 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractImport implements ImportInterface { protected $em; protected $logger; protected $contentProxy; + protected $eventDispatcher; protected $producer; protected $user; protected $markAsRead; @@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface protected $importedEntries = 0; protected $queuedEntries = 0; - public function __construct(EntityManager $em, ContentProxy $contentProxy) + public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher) { $this->em = $em; $this->logger = new NullLogger(); $this->contentProxy = $contentProxy; + $this->eventDispatcher = $eventDispatcher; } public function setLogger(LoggerInterface $logger) @@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ($this->markAsRead) { @@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface continue; } + // store each entry to be flushed so we can trigger the entry.saved event for each of them + // entry.saved needs the entry to be persisted in db because it needs it id to generate + // images (at least) + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); @@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 2ca1683b1..8bf7d92e6 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class BrowserImport extends AbstractImport { @@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ((array) $importedEntry !== $importedEntry) { @@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport continue; } + // @see AbstractImport + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; } ++$i; } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 327e25001..0c26aced4 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -16,13 +16,6 @@ class PocketImport extends AbstractImport const NB_ELEMENTS = 5000; - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->contentProxy = $contentProxy; - $this->logger = new NullLogger(); - } - /** * Only used for test purpose. * diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index 70b8a0d47..a5af52828 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -6,6 +6,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.readability: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -13,6 +14,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.instapaper: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -20,6 +22,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.instapaper.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -27,6 +30,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.wallabag_v2: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -34,6 +38,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.firefox: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -41,6 +46,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.firefox.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.chrome: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -48,4 +54,5 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.chrome.import" + - "@event_dispatcher" - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 0a81e1b55..5ced4c838 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -18,6 +18,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" + - "@event_dispatcher" - "@logger" # instapaper @@ -38,6 +39,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.instapaper.import" + - "@event_dispatcher" - "@logger" # pocket @@ -58,6 +60,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" + - "@event_dispatcher" - "@logger" # wallabag v1 @@ -78,6 +81,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" + - "@event_dispatcher" - "@logger" # wallabag v2 @@ -98,6 +102,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" + - "@event_dispatcher" - "@logger" # firefox @@ -118,6 +123,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.firefox.import" + - "@event_dispatcher" - "@logger" # chrome @@ -138,4 +144,5 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.chrome.import" + - "@event_dispatcher" - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index d600be0f8..64822963e 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -20,6 +20,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] @@ -31,6 +32,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -41,6 +43,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -51,6 +54,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -61,6 +65,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -71,6 +76,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -80,6 +86,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: diff --git a/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php index a3263771c..856954a64 100644 --- a/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php +++ b/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php @@ -112,10 +112,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn($entry); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->once()) + ->method('dispatch'); + $consumer = new AMQPEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $message = new AMQPMessage($body); @@ -157,10 +166,19 @@ JSON; ->disableOriginalConstructor() ->getMock(); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new AMQPEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $message = new AMQPMessage($body); @@ -212,10 +230,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn(null); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new AMQPEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $message = new AMQPMessage($body); diff --git a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php index 01a92ad2a..3b92f7596 100644 --- a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php +++ b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php @@ -111,10 +111,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn($entry); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->once()) + ->method('dispatch'); + $consumer = new RedisEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $res = $consumer->manage($body); @@ -156,10 +165,19 @@ JSON; ->disableOriginalConstructor() ->getMock(); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new RedisEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $res = $consumer->manage($body); @@ -211,10 +229,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn(null); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new RedisEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $res = $consumer->manage($body); diff --git a/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php index 1e52615c0..6b3adda4a 100644 --- a/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php @@ -18,7 +18,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getChromeImport($unsetUser = false) + private function getChromeImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new ChromeImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new ChromeImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $chromeImport = $this->getChromeImport(); + $chromeImport = $this->getChromeImport(false, 1); $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $chromeImport = $this->getChromeImport(); + $chromeImport = $this->getChromeImport(false, 1); $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php index 007dda6a2..b516fbc53 100644 --- a/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php @@ -18,7 +18,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getFirefoxImport($unsetUser = false) + private function getFirefoxImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new FirefoxImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new FirefoxImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $firefoxImport = $this->getFirefoxImport(); + $firefoxImport = $this->getFirefoxImport(false, 2); $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $firefoxImport = $this->getFirefoxImport(); + $firefoxImport = $this->getFirefoxImport(false, 1); $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php b/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php index 75900bd70..e262a8082 100644 --- a/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php @@ -18,7 +18,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getInstapaperImport($unsetUser = false) + private function getInstapaperImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $import = new InstapaperImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $import = new InstapaperImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $instapaperImport = $this->getInstapaperImport(); + $instapaperImport = $this->getInstapaperImport(false, 3); $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $instapaperImport = $this->getInstapaperImport(); + $instapaperImport = $this->getInstapaperImport(false, 1); $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 9ec7935c9..141ece36e 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -24,7 +24,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase protected $contentProxy; protected $logHandler; - private function getPocketImport($consumerKey = 'ConsumerKey') + private function getPocketImport($consumerKey = 'ConsumerKey', $dispatched = 0) { $this->user = new User(); @@ -55,10 +55,15 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase ->method('getScheduledEntityInsertions') ->willReturn([]); - $pocket = new PocketImport( - $this->em, - $this->contentProxy - ); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $pocket = new PocketImport($this->em, $this->contentProxy, $dispatcher); $pocket->setUser($this->user); $this->logHandler = new TestHandler(); @@ -252,7 +257,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $client->getEmitter()->attach($mock); - $pocketImport = $this->getPocketImport(); + $pocketImport = $this->getPocketImport('ConsumerKey', 1); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') ->disableOriginalConstructor() @@ -339,7 +344,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $client->getEmitter()->attach($mock); - $pocketImport = $this->getPocketImport(); + $pocketImport = $this->getPocketImport('ConsumerKey', 2); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') ->disableOriginalConstructor() @@ -591,7 +596,7 @@ JSON; $client->getEmitter()->attach($mock); - $pocketImport = $this->getPocketImport(); + $pocketImport = $this->getPocketImport('ConsumerKey', 1); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') ->disableOriginalConstructor() diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php index d98cd486a..d1bbe648b 100644 --- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php @@ -18,7 +18,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getReadabilityImport($unsetUser = false) + private function getReadabilityImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new ReadabilityImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new ReadabilityImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $readabilityImport = $this->getReadabilityImport(); + $readabilityImport = $this->getReadabilityImport(false, 24); $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $readabilityImport = $this->getReadabilityImport(); + $readabilityImport = $this->getReadabilityImport(false, 1); $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability-read.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php index 82dc4c7e1..4dbced604 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php @@ -18,7 +18,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getWallabagV1Import($unsetUser = false) + private function getWallabagV1Import($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -44,7 +44,15 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new WallabagV1Import($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new WallabagV1Import($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -68,7 +76,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $wallabagV1Import = $this->getWallabagV1Import(); + $wallabagV1Import = $this->getWallabagV1Import(false, 3); $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -101,7 +109,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $wallabagV1Import = $this->getWallabagV1Import(); + $wallabagV1Import = $this->getWallabagV1Import(false, 3); $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1-read.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php index bea89efbd..0e50b8b2c 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php @@ -18,7 +18,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getWallabagV2Import($unsetUser = false) + private function getWallabagV2Import($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -44,7 +44,15 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new WallabagV2Import($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new WallabagV2Import($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -68,7 +76,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import = $this->getWallabagV2Import(false, 2); $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -97,7 +105,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import = $this->getWallabagV2Import(false, 2); $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2-read.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -246,7 +254,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase public function testImportWithExceptionFromGraby() { - $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import = $this->getWallabagV2Import(false, 2); $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') From 1f66d79e6b5f54375e63799f10d0773203fe12f9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:10:57 +0100 Subject: [PATCH 15/17] Add more importer to wallabag:import command All importer available expect Pocket which require an oAuth login. --- .../ImportBundle/Command/ImportCommand.php | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 1df38295f..2f7a906e3 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -14,10 +14,10 @@ class ImportCommand extends ContainerAwareCommand { $this ->setName('wallabag:import') - ->setDescription('Import entries from a JSON export from a wallabag v1 instance') + ->setDescription('Import entries from a JSON export') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, readability, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -42,29 +42,35 @@ class ImportCommand extends ContainerAwareCommand switch ($input->getOption('importer')) { case 'v2': - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + $import = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); break; case 'firefox': - $wallabag = $this->getContainer()->get('wallabag_import.firefox.import'); + $import = $this->getContainer()->get('wallabag_import.firefox.import'); break; case 'chrome': - $wallabag = $this->getContainer()->get('wallabag_import.chrome.import'); + $import = $this->getContainer()->get('wallabag_import.chrome.import'); + break; + case 'readability': + $import = $this->getContainer()->get('wallabag_import.readability.import'); + break; + case 'instapaper': + $import = $this->getContainer()->get('wallabag_import.instapaper.import'); break; case 'v1': default: - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); + $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); break; } - $wallabag->setMarkAsRead($input->getOption('markAsRead')); - $wallabag->setUser($user); + $import->setMarkAsRead($input->getOption('markAsRead')); + $import->setUser($user); - $res = $wallabag + $res = $import ->setFilepath($input->getArgument('filepath')) ->import(); if (true === $res) { - $summary = $wallabag->getSummary(); + $summary = $import->getSummary(); $output->writeln(''.$summary['imported'].' imported'); $output->writeln(''.$summary['skipped'].' already saved'); } From e46fdf3096d55028611713b7a196401aa57d864b Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:26:14 +0100 Subject: [PATCH 16/17] CS --- src/Wallabag/ImportBundle/Import/PocketImport.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 0c26aced4..330934809 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,8 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; From ca08d02f2f17c0e4ec0152a701c540f62713a428 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Thu, 3 Nov 2016 15:59:18 +0100 Subject: [PATCH 17/17] Add warning message for import + download images When import isn't async and downloading images is enabled, we warn the user that import can fail. Also, added missing translations --- .../CoreBundle/Resources/translations/messages.da.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.de.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.en.yml | 2 ++ .../CoreBundle/Resources/translations/messages.es.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.fa.yml | 7 +++++++ .../CoreBundle/Resources/translations/messages.fr.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.it.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.oc.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.pl.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.ro.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.tr.yml | 7 +++++++ .../ImportBundle/Resources/views/Chrome/index.html.twig | 2 ++ .../ImportBundle/Resources/views/Firefox/index.html.twig | 2 ++ .../{_workerEnabled.html.twig => _information.html.twig} | 7 +++++++ .../ImportBundle/Resources/views/Import/index.html.twig | 2 ++ .../Resources/views/Instapaper/index.html.twig | 2 +- .../ImportBundle/Resources/views/Pocket/index.html.twig | 2 +- .../Resources/views/Readability/index.html.twig | 2 +- .../Resources/views/WallabagV1/index.html.twig | 2 +- 19 files changed, 81 insertions(+), 4 deletions(-) rename src/Wallabag/ImportBundle/Resources/views/Import/{_workerEnabled.html.twig => _information.html.twig} (55%) diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index c05955833..87083799d 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." # firefox: # page_title: 'Import > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index 0051da2f4..5e5dd996f 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -367,6 +367,7 @@ import: how_to: 'Bitte wähle deinen Readability Export aus und klicke den unteren Button für das Hochladen und Importieren dessen.' worker: enabled: "Der Import erfolgt asynchron. Sobald der Import gestartet ist, wird diese Aufgabe extern abgearbeitet. Der aktuelle Service dafür ist:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Aus Firefox importieren' description: "Dieser Import wird all deine Lesezeichen aus Firefox importieren. Gehe zu deinen Lesezeichen (Strg+Shift+O), dann auf \"Importen und Sichern\", wähle \"Sichern…\". Du erhälst eine .json Datei." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Neuer Client erstellt.' client_deleted: 'Client gelöscht' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 462be5562..ab56cdb97 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -367,6 +367,7 @@ import: how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Import > Firefox' description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -470,6 +471,7 @@ flashes: rss_updated: 'RSS information updated' tagging_rules_updated: 'Tagging rules updated' tagging_rules_deleted: 'Tagging rule deleted' + # user_added: 'User "%username%" added' rss_token_updated: 'RSS token updated' annotations_reset: Annotations reset tags_reset: Tags reset diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index cfabe09f2..c852e5b75 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nuevo cliente creado.' client_deleted: 'Cliente suprimido' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 07b5bee7c..cf0724509 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -284,6 +284,7 @@ quickstart: paragraph_2: 'ادامه دهید!' configure: title: 'برنامه را تنظیم کنید' + # description: 'In order to have an application which suits you, have a look into the configuration of wallabag.' language: 'زبان و نمای برنامه را تغییر دهید' rss: 'خوراک آر-اس-اس را فعال کنید' tagging_rules: 'قانون‌های برچسب‌گذاری خودکار مقاله‌هایتان را تعریف کنید' @@ -366,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'درون‌ریزی > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -503,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index db6f9f7e2..c4aa37515 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -367,6 +367,7 @@ import: how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer." worker: enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :" + download_images_warning: "Vous avez configuré le téléchagement des images pour vos articles. Combiné à l'import classique, cette opération peut être très très longue (voire échouer). Nous vous conseillons vivement d'activer les imports asynchrones." firefox: page_title: 'Import > Firefox' description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox. Ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json.

" @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nouveau client %name% créé' client_deleted: 'Client %name% supprimé' + user: + notice: + added: 'Utilisateur "%username%" ajouté' + updated: 'Utilisateur "%username%" mis à jour' + deleted: 'Utilisateur "%username%" supprimé' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index f1aff51a8..38c780dd3 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importa da > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nuovo client creato.' client_deleted: 'Client eliminato' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index e0567d7e1..103ca831e 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -367,6 +367,7 @@ import: how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." worker: enabled: "L'importacion se fa de manièra asincròna. Un còp l'importacion lançada, una aisina externa s'ocuparà dels messatges un per un. Lo servici actual es : " + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' description: "Aquesta aisina importarà totas vòstres favorits de Firefox. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Novèl client creat' client_deleted: 'Client suprimit' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index a2989dbda..1680dae0a 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -367,6 +367,7 @@ import: how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' worker: enabled: "Import jest wykonywany asynchronicznie. Od momentu rozpoczęcia importu, zewnętrzna usługa może zajmować się na raz tylko jednym zadaniem. Bieżącą usługą jest:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Import > Firefox' description: "Ten importer zaimportuje wszystkie twoje zakładki z Firefoksa. Idź do twoich zakładek (Ctrl+Shift+O), następnie w \"Import i kopie zapasowe\", wybierz \"Utwórz kopię zapasową...\". Uzyskasz plik .json." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nowy klient utworzony.' client_deleted: 'Klient usunięty' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 6e4813e58..16d39188b 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." # firefox: # page_title: 'Import > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 769031023..c9a269ecd 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -108,6 +108,7 @@ config: # if_label: 'if' # then_tag_as_label: 'then tag as' # delete_rule_label: 'delete' + # edit_rule_label: 'edit' rule_label: 'Kural' tags_label: 'Etiketler' faq: @@ -366,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'İçe Aktar > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -503,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig index ead828c6e..93b08540f 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} +
{{ import.description|trans|raw }}

{{ 'import.chrome.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig index f975da3fe..ced3f0088 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} +
{{ import.description|trans|raw }}

{{ 'import.firefox.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig similarity index 55% rename from src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig rename to src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig index 2390a41f8..48bbcfe7d 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig @@ -1,8 +1,15 @@ {% set redis = craue_setting('import_with_redis') %} {% set rabbit = craue_setting('import_with_rabbitmq') %} +{% set downloadImages = craue_setting('download_images_enabled') %} {% if redis or rabbit %}
{{ 'import.worker.enabled'|trans }} {% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}
{% endif %} + +{% if not redis and not rabbit and downloadImages %} +
+ {{ 'import.worker.download_images_warning'|trans|raw }} +
+{% endif %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index 6ea5e0f42..b1ec40a6b 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} + {{ 'import.page_description'|trans }}
    {% for import in imports %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig index 5789361f0..28165d190 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig index 6195fa073..536e3d1ae 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %} {% if not has_consumer_key %}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig index 74653b0f6..737b0adf4 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig index 0b19bc347..974b2c73e 100644 --- a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}