mirrors/wallabag
1
0
Fork 0
mirror of https://github.com/wallabag/wallabag.git synced 2024-11-29 20:41:03 +00:00
Code Issues Projects Releases Wiki Activity

Fix #47 - update Readability class (thx to http://code.fivefilters.org/php-readability/)

Browse source
This commit is contained in:
nicosomb 2013-04-23 15:57:23 +02:00
parent c1c9f252f7
commit 0ace6cab0b
1 changed files with 1099 additions and 1066 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

119
inc/Readability.php
View file

@ -2,6 +2,8 @@
/** /**
* Arc90's Readability ported to PHP for FiveFilters.org * Arc90's Readability ported to PHP for FiveFilters.org
* Based on readability.js version 1.7.1 (without multi-page support) * Based on readability.js version 1.7.1 (without multi-page support)
* Updated to allow HTML5 parsing with html5lib
* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
* ------------------------------------------------------ * ------------------------------------------------------
* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js * Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
* Arc90's project URL: http://lab.arc90.com/experiments/readability/ * Arc90's project URL: http://lab.arc90.com/experiments/readability/
@ -10,7 +12,7 @@
* More information: http://fivefilters.org/content-only/ * More information: http://fivefilters.org/content-only/
* License: Apache License, Version 2.0 * License: Apache License, Version 2.0
* Requires: PHP5 * Requires: PHP5
* Date: 2011-07-22 * Date: 2012-09-19
* *
* Differences between the PHP port and the original * Differences between the PHP port and the original
* ------------------------------------------------------ * ------------------------------------------------------
@ -62,14 +64,8 @@ $r->init();
echo $r->articleContent->innerHTML; echo $r->articleContent->innerHTML;
*/ */
class Readability class Readability
{ {
/* constants */
const FLAG_STRIP_UNLIKELYS = 1;
const FLAG_WEIGHT_CLASSES = 2;
const FLAG_CLEAN_CONDITIONALLY = 4;
public $version = '1.7.1-without-multi-page'; public $version = '1.7.1-without-multi-page';
public $convertLinksToFootnotes = false; public $convertLinksToFootnotes = false;
public $revertForcedParagraphElements = true; public $revertForcedParagraphElements = true;
@ -78,9 +74,10 @@ class Readability
public $dom; public $dom;
public $url = null; // optional - URL where HTML was retrieved public $url = null; // optional - URL where HTML was retrieved
public $debug = false; public $debug = false;
public $lightClean = true; // preserves more content (experimental) added 2012-09-19
protected $body = null; // protected $body = null; //
protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
protected $success = false; // indicates whether we were able to extract or not protected $success = false; // indicates whether we were able to extract or not
/** /**
@ -88,37 +85,47 @@ class Readability
* Defined up here so we don't instantiate them repeatedly in loops. * Defined up here so we don't instantiate them repeatedly in loops.
**/ **/
public $regexps = array( public $regexps = array(
'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i', 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i', 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i', 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i', 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
'replaceFonts' => '/<(\/?)font[^>]*>/i', 'replaceFonts' => '/<(\/?)font[^>]*>/i',
// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim() // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
'normalize' => '/\s{2,}/', 'normalize' => '/\s{2,}/',
'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/', 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i', 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i' 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
); );
/* constants */
const FLAG_STRIP_UNLIKELYS = 1;
const FLAG_WEIGHT_CLASSES = 2;
const FLAG_CLEAN_CONDITIONALLY = 4;
/** /**
* Create instance of Readability * Create instance of Readability
* @param string UTF-8 encoded string * @param string UTF-8 encoded string
* @param string (optional) URL associated with HTML (used for footnotes) * @param string (optional) URL associated with HTML (used for footnotes)
* @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
*/ */
function __construct($html, $url=null) function __construct($html, $url=null, $parser='libxml')
{ {
$this->url = $url;
/* Turn all double br's into p's */ /* Turn all double br's into p's */
$html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html); $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
$html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
if (trim($html) == '') $html = '<html></html>';
if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
// all good
} else {
$this->dom = new DOMDocument(); $this->dom = new DOMDocument();
$this->dom->preserveWhiteSpace = false; $this->dom->preserveWhiteSpace = false;
$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
if (trim($html) == '') $html = '<html></html>';
@$this->dom->loadHTML($html); @$this->dom->loadHTML($html);
$this->url = $url; }
$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
} }
/** /**
@ -185,7 +192,6 @@ class Readability
$articleContent = $this->dom->createElement('div'); $articleContent = $this->dom->createElement('div');
$articleContent->setAttribute('id', 'readability-content'); $articleContent->setAttribute('id', 'readability-content');
$articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>'; $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
return $this->success;
} }
$overlay->setAttribute('id', 'readOverlay'); $overlay->setAttribute('id', 'readOverlay');
@ -215,7 +221,7 @@ class Readability
* Debug * Debug
*/ */
protected function dbg($msg) { protected function dbg($msg) {
if ($this->debug) echo '* ',$msg, '<br />', "\n"; if ($this->debug) echo '* ',$msg, "\n";
} }
/** /**
@ -422,7 +428,7 @@ class Readability
* If there is only one h2, they are probably using it * If there is only one h2, they are probably using it
* as a header and not a subheader, so remove it since we already have a header. * as a header and not a subheader, so remove it since we already have a header.
***/ ***/
if ($articleContent->getElementsByTagName('h2')->length == 1) { if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
$this->clean($articleContent, 'h2'); $this->clean($articleContent, 'h2');
} }
$this->clean($articleContent, 'iframe'); $this->clean($articleContent, 'iframe');
@ -441,8 +447,9 @@ class Readability
$imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length; $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
$embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length; $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
$objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length; $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
$iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '') if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
{ {
$articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i)); $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
} }
@ -566,7 +573,7 @@ class Readability
} }
else else
{ {
// EXPERIMENTAL /* EXPERIMENTAL */
// TODO: change these p elements back to text nodes after processing // TODO: change these p elements back to text nodes after processing
for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) { for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
$childNode = $node->childNodes->item($i); $childNode = $node->childNodes->item($i);
@ -736,23 +743,6 @@ class Readability
} }
} }
/* Look for a special classname */
if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('class') && $siblingNode->getAttribute('class') != '')
{
if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('class'))) {
$append = true;
}
}
/* Look for a special classname */
if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('id') && $siblingNode->getAttribute('id') != '')
{
if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('id'))) {
$append = true;
}
}
if ($append) if ($append)
{ {
$this->dbg('Appending node: ' . $siblingNode->nodeName); $this->dbg('Appending node: ' . $siblingNode->nodeName);
@ -965,13 +955,15 @@ class Readability
* Clean a node of all elements of type "tag". * Clean a node of all elements of type "tag".
* (Unless it's a youtube/vimeo video. People love movies.) * (Unless it's a youtube/vimeo video. People love movies.)
* *
* Updated 2012-09-18 to preserve youtube/vimeo iframes
*
* @param DOMElement $e * @param DOMElement $e
* @param string $tag * @param string $tag
* @return void * @return void
*/ */
public function clean($e, $tag) { public function clean($e, $tag) {
$targetList = $e->getElementsByTagName($tag); $targetList = $e->getElementsByTagName($tag);
$isEmbed = ($tag == 'object' || $tag == 'embed'); $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
for ($y=$targetList->length-1; $y >= 0; $y--) { for ($y=$targetList->length-1; $y >= 0; $y--) {
/* Allow youtube and vimeo videos through as people usually want to see those. */ /* Allow youtube and vimeo videos through as people usually want to see those. */
@ -1036,6 +1028,7 @@ class Readability
$img = $tagsList->item($i)->getElementsByTagName('img')->length; $img = $tagsList->item($i)->getElementsByTagName('img')->length;
$li = $tagsList->item($i)->getElementsByTagName('li')->length-100; $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
$input = $tagsList->item($i)->getElementsByTagName('input')->length; $input = $tagsList->item($i)->getElementsByTagName('input')->length;
$a = $tagsList->item($i)->getElementsByTagName('a')->length;
$embedCount = 0; $embedCount = 0;
$embeds = $tagsList->item($i)->getElementsByTagName('embed'); $embeds = $tagsList->item($i)->getElementsByTagName('embed');
@ -1044,28 +1037,69 @@ class Readability
$embedCount++; $embedCount++;
} }
} }
$embeds = $tagsList->item($i)->getElementsByTagName('iframe');
for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
$embedCount++;
}
}
$linkDensity = $this->getLinkDensity($tagsList->item($i)); $linkDensity = $this->getLinkDensity($tagsList->item($i));
$contentLength = strlen($this->getInnerText($tagsList->item($i))); $contentLength = strlen($this->getInnerText($tagsList->item($i)));
$toRemove = false; $toRemove = false;
if ( $img > $p ) { if ($this->lightClean) {
$this->dbg('Light clean...');
if ( ($img > $p) && ($img > 4) ) {
$this->dbg(' more than 4 images and more image elements than paragraph elements');
$toRemove = true; $toRemove = true;
} else if ($li > $p && $tag != 'ul' && $tag != 'ol') { } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
$toRemove = true; $toRemove = true;
} else if ( $input > floor($p/3) ) { } else if ( $input > floor($p/3) ) {
$this->dbg(' too many <input> elements');
$toRemove = true; $toRemove = true;
} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) { } else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
$this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');
$toRemove = true; $toRemove = true;
} else if($weight < 25 && $linkDensity > 0.2) { } else if($weight < 25 && $linkDensity > 0.2) {
$this->dbg(' weight smaller than 25 and link density above 0.2');
$toRemove = true;
} else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
$this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
$toRemove = true;
} else if($embedCount > 3) {
$this->dbg(' more than 3 embeds');
$toRemove = true;
}
} else {
$this->dbg('Standard clean...');
if ( $img > $p ) {
$this->dbg(' more image elements than paragraph elements');
$toRemove = true;
} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
$toRemove = true;
} else if ( $input > floor($p/3) ) {
$this->dbg(' too many <input> elements');
$toRemove = true;
} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
$this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
$toRemove = true;
} else if($weight < 25 && $linkDensity > 0.2) {
$this->dbg(' weight smaller than 25 and link density above 0.2');
$toRemove = true; $toRemove = true;
} else if($weight >= 25 && $linkDensity > 0.5) { } else if($weight >= 25 && $linkDensity > 0.5) {
$this->dbg(' weight above 25 but link density greater than 0.5');
$toRemove = true; $toRemove = true;
} else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) { } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
$this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
$toRemove = true; $toRemove = true;
} }
}
if ($toRemove) { if ($toRemove) {
//$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
$tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
} }
} }
@ -1101,4 +1135,3 @@ class Readability
$this->flags = $this->flags & ~$flag; $this->flags = $this->flags & ~$flag;
} }
} }
?>