mirror of
https://github.com/wallabag/wallabag.git
synced 2024-05-20 01:58:06 +00:00
Download pictures successfully
Needs to rewrite them properly (get base url)
This commit is contained in:
Thomas Citharel
Jeremy Benoist
parent
4dface6670
commit
419214d722
|
|
@ -82,7 +82,8 @@
|
|||
"white-october/pagerfanta-bundle": "^1.0",
|
||||
"php-amqplib/rabbitmq-bundle": "^1.8",
|
||||
"predis/predis": "^1.0",
|
||||
"javibravo/simpleue": "^1.0"
|
||||
"javibravo/simpleue": "^1.0",
|
||||
"symfony/dom-crawler": "^3.1"
|
||||
},
|
||||
"require-dev": {
|
||||
"doctrine/doctrine-fixtures-bundle": "~2.2",
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ class ContentProxy
|
|||
|
||||
$entry->setUrl($content['url'] ?: $url);
|
||||
$entry->setTitle($title);
|
||||
$entry->setContent($html);
|
||||
|
||||
$entry->setLanguage($content['language']);
|
||||
$entry->setMimetype($content['content_type']);
|
||||
$entry->setReadingTime(Utils::getReadingTime($html));
|
||||
|
|
@ -75,6 +75,14 @@ class ContentProxy
|
|||
$entry->setDomainName($domainName);
|
||||
}
|
||||
|
||||
if (true) {
|
||||
$this->logger->log('debug','Starting to download images');
|
||||
$downloadImages = new DownloadImages($html, $url, $this->logger);
|
||||
$html = $downloadImages->process();
|
||||
}
|
||||
|
||||
$entry->setContent($html);
|
||||
|
||||
if (isset($content['open_graph']['og_image'])) {
|
||||
$entry->setPreviewPicture($content['open_graph']['og_image']);
|
||||
}
|
||||
|
|
@ -151,172 +159,4 @@ class ContentProxy
|
|||
{
|
||||
return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Changing pictures URL in article content.
|
||||
*/
|
||||
public static function filterPicture($content, $url, $id)
|
||||
{
|
||||
$matches = array();
|
||||
$processing_pictures = array(); // list of processing image to avoid processing the same pictures twice
|
||||
preg_match_all('#<\s*(img)[^>]+src="([^"]*)"[^>]*>#Si', $content, $matches, PREG_SET_ORDER);
|
||||
foreach ($matches as $i => $link) {
|
||||
$link[1] = trim($link[1]);
|
||||
if (!preg_match('#^(([a-z]+://)|(\#))#', $link[1])) {
|
||||
$absolute_path = self::_getAbsoluteLink($link[2], $url);
|
||||
$filename = basename(parse_url($absolute_path, PHP_URL_PATH));
|
||||
$directory = self::_createAssetsDirectory($id);
|
||||
$fullpath = $directory.'/'.$filename;
|
||||
|
||||
if (in_array($absolute_path, $processing_pictures) === true) {
|
||||
// replace picture's URL only if processing is OK : already processing -> go to next picture
|
||||
continue;
|
||||
}
|
||||
|
||||
if (self::_downloadPictures($absolute_path, $fullpath) === true) {
|
||||
$content = str_replace($matches[$i][2], Tools::getPocheUrl().$fullpath, $content);
|
||||
}
|
||||
|
||||
$processing_pictures[] = $absolute_path;
|
||||
}
|
||||
}
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get absolute URL.
|
||||
*/
|
||||
private static function _getAbsoluteLink($relativeLink, $url)
|
||||
{
|
||||
/* return if already absolute URL */
|
||||
if (parse_url($relativeLink, PHP_URL_SCHEME) != '') {
|
||||
return $relativeLink;
|
||||
}
|
||||
|
||||
/* queries and anchors */
|
||||
if ($relativeLink[0] == '#' || $relativeLink[0] == '?') {
|
||||
return $url.$relativeLink;
|
||||
}
|
||||
|
||||
/* parse base URL and convert to local variables:
|
||||
$scheme, $host, $path */
|
||||
extract(parse_url($url));
|
||||
|
||||
/* remove non-directory element from path */
|
||||
$path = preg_replace('#/[^/]*$#', '', $path);
|
||||
|
||||
/* destroy path if relative url points to root */
|
||||
if ($relativeLink[0] == '/') {
|
||||
$path = '';
|
||||
}
|
||||
|
||||
/* dirty absolute URL */
|
||||
$abs = $host.$path.'/'.$relativeLink;
|
||||
|
||||
/* replace '//' or '/./' or '/foo/../' with '/' */
|
||||
$re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#');
|
||||
for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) {
|
||||
}
|
||||
|
||||
/* absolute URL is ready! */
|
||||
return $scheme.'://'.$abs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloading pictures.
|
||||
*
|
||||
* @return bool true if the download and processing is OK, false else
|
||||
*/
|
||||
private static function _downloadPictures($absolute_path, $fullpath)
|
||||
{
|
||||
$rawdata = Tools::getFile($absolute_path);
|
||||
$fullpath = urldecode($fullpath);
|
||||
|
||||
if (file_exists($fullpath)) {
|
||||
unlink($fullpath);
|
||||
}
|
||||
|
||||
// check extension
|
||||
$file_ext = strrchr($fullpath, '.');
|
||||
$whitelist = array('.jpg', '.jpeg', '.gif', '.png');
|
||||
if (!(in_array($file_ext, $whitelist))) {
|
||||
Tools::logm('processed image with not allowed extension. Skipping '.$fullpath);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// check headers
|
||||
$imageinfo = getimagesize($absolute_path);
|
||||
if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') {
|
||||
Tools::logm('processed image with bad header. Skipping '.$fullpath);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// regenerate image
|
||||
$im = imagecreatefromstring($rawdata);
|
||||
if ($im === false) {
|
||||
Tools::logm('error while regenerating image '.$fullpath);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
switch ($imageinfo['mime']) {
|
||||
case 'image/gif':
|
||||
$result = imagegif($im, $fullpath);
|
||||
break;
|
||||
case 'image/jpeg':
|
||||
case 'image/jpg':
|
||||
$result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY);
|
||||
break;
|
||||
case 'image/png':
|
||||
$result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9));
|
||||
break;
|
||||
}
|
||||
imagedestroy($im);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a directory for an article.
|
||||
*
|
||||
* @param $id ID of the article
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
private static function _createAssetsDirectory($id)
|
||||
{
|
||||
$assets_path = ABS_PATH;
|
||||
if (!is_dir($assets_path)) {
|
||||
mkdir($assets_path, 0715);
|
||||
}
|
||||
|
||||
$article_directory = $assets_path.$id;
|
||||
if (!is_dir($article_directory)) {
|
||||
mkdir($article_directory, 0715);
|
||||
}
|
||||
|
||||
return $article_directory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the directory.
|
||||
*
|
||||
* @param $directory
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public static function removeDirectory($directory)
|
||||
{
|
||||
if (is_dir($directory)) {
|
||||
$files = array_diff(scandir($directory), array('.', '..'));
|
||||
foreach ($files as $file) {
|
||||
(is_dir("$directory/$file")) ? self::removeDirectory("$directory/$file") : unlink("$directory/$file");
|
||||
}
|
||||
|
||||
return rmdir($directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
147
src/Wallabag/CoreBundle/Helper/DownloadImages.php
Normal file
147
src/Wallabag/CoreBundle/Helper/DownloadImages.php
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
<?php
|
||||
|
||||
namespace Wallabag\CoreBundle\Helper;
|
||||
|
||||
use Psr\Log\LoggerInterface as Logger;
|
||||
use Symfony\Component\DomCrawler\Crawler;
|
||||
|
||||
define('REGENERATE_PICTURES_QUALITY', 75);
|
||||
|
||||
class DownloadImages {
|
||||
private $folder;
|
||||
private $url;
|
||||
private $html;
|
||||
private $fileName;
|
||||
private $logger;
|
||||
|
||||
public function __construct($html, $url, Logger $logger) {
|
||||
$this->html = $html;
|
||||
$this->url = $url;
|
||||
$this->setFolder();
|
||||
$this->logger = $logger;
|
||||
}
|
||||
|
||||
public function setFolder($folder = "assets/images") {
|
||||
// if folder doesn't exist, attempt to create one and store the folder name in property $folder
|
||||
if(!file_exists($folder)) {
|
||||
mkdir($folder);
|
||||
}
|
||||
$this->folder = $folder;
|
||||
}
|
||||
|
||||
public function process() {
|
||||
//instantiate the symfony DomCrawler Component
|
||||
$crawler = new Crawler($this->html);
|
||||
// create an array of all scrapped image links
|
||||
$this->logger->log('debug', 'Finding images inside document');
|
||||
$result = $crawler
|
||||
->filterXpath('//img')
|
||||
->extract(array('src'));
|
||||
|
||||
// download and save the image to the folder
|
||||
foreach ($result as $image) {
|
||||
$file = file_get_contents($image);
|
||||
|
||||
// Checks
|
||||
$absolute_path = self::getAbsoluteLink($image, $this->url);
|
||||
$filename = basename(parse_url($absolute_path, PHP_URL_PATH));
|
||||
$fullpath = $this->folder."/".$filename;
|
||||
self::checks($file, $fullpath, $absolute_path);
|
||||
$this->html = str_replace($image, $fullpath, $this->html);
|
||||
}
|
||||
|
||||
return $this->html;
|
||||
}
|
||||
|
||||
private function checks($rawdata, $fullpath, $absolute_path) {
|
||||
$fullpath = urldecode($fullpath);
|
||||
|
||||
if (file_exists($fullpath)) {
|
||||
unlink($fullpath);
|
||||
}
|
||||
|
||||
// check extension
|
||||
$this->logger->log('debug','Checking extension');
|
||||
|
||||
$file_ext = strrchr($fullpath, '.');
|
||||
$whitelist = array('.jpg', '.jpeg', '.gif', '.png');
|
||||
if (!(in_array($file_ext, $whitelist))) {
|
||||
$this->logger->log('debug','processed image with not allowed extension. Skipping '.$fullpath);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// check headers
|
||||
$this->logger->log('debug','Checking headers');
|
||||
$imageinfo = getimagesize($absolute_path);
|
||||
if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') {
|
||||
$this->logger->log('debug','processed image with bad header. Skipping '.$fullpath);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// regenerate image
|
||||
$this->logger->log('debug','regenerating image');
|
||||
$im = imagecreatefromstring($rawdata);
|
||||
if ($im === false) {
|
||||
$this->logger->log('error','error while regenerating image '.$fullpath);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
switch ($imageinfo['mime']) {
|
||||
case 'image/gif':
|
||||
$result = imagegif($im, $fullpath);
|
||||
$this->logger->log('debug','Re-creating gif');
|
||||
break;
|
||||
case 'image/jpeg':
|
||||
case 'image/jpg':
|
||||
$result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY);
|
||||
$this->logger->log('debug','Re-creating jpg');
|
||||
break;
|
||||
case 'image/png':
|
||||
$this->logger->log('debug','Re-creating png');
|
||||
$result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9));
|
||||
break;
|
||||
}
|
||||
imagedestroy($im);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private static function getAbsoluteLink($relativeLink, $url)
|
||||
{
|
||||
/* return if already absolute URL */
|
||||
if (parse_url($relativeLink, PHP_URL_SCHEME) != '') {
|
||||
return $relativeLink;
|
||||
}
|
||||
|
||||
/* queries and anchors */
|
||||
if ($relativeLink[0] == '#' || $relativeLink[0] == '?') {
|
||||
return $url.$relativeLink;
|
||||
}
|
||||
|
||||
/* parse base URL and convert to local variables:
|
||||
$scheme, $host, $path */
|
||||
extract(parse_url($url));
|
||||
|
||||
/* remove non-directory element from path */
|
||||
$path = preg_replace('#/[^/]*$#', '', $path);
|
||||
|
||||
/* destroy path if relative url points to root */
|
||||
if ($relativeLink[0] == '/') {
|
||||
$path = '';
|
||||
}
|
||||
|
||||
/* dirty absolute URL */
|
||||
$abs = $host.$path.'/'.$relativeLink;
|
||||
|
||||
/* replace '//' or '/./' or '/foo/../' with '/' */
|
||||
$re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#');
|
||||
for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) {
|
||||
}
|
||||
|
||||
/* absolute URL is ready! */
|
||||
return $scheme.'://'.$abs;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue