From 31a10069a52c2fd2aca3a835a7bdc1accae197f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 28 Feb 2014 20:36:32 +0100 Subject: [PATCH 1/3] [add] upload form for import --- inc/poche/Poche.class.php | 24 +++++++++++++++++------- inc/poche/Tools.class.php | 1 - index.php | 2 ++ themes/baggy/config.twig | 25 +++++++++++++++++++------ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/inc/poche/Poche.class.php b/inc/poche/Poche.class.php index 49651c529..026b0b4e1 100755 --- a/inc/poche/Poche.class.php +++ b/inc/poche/Poche.class.php @@ -1068,13 +1068,7 @@ class Poche Tools::redirect(); } - $targetDefinition = 'IMPORT_' . strtoupper($from) . '_FILE'; - $targetFile = constant($targetDefinition); - - if (! defined($targetDefinition)) { - $this->messages->add('e', _('Incomplete inc/poche/define.inc.php file, please define "' . $targetDefinition . '".')); - Tools::redirect(); - } + $targetFile = CACHE . '/' . constant(strtoupper($from) . '_FILE'); if (! file_exists($targetFile)) { $this->messages->add('e', _('Could not find required "' . $targetFile . '" import file.')); @@ -1084,6 +1078,22 @@ class Poche $this->$providers[$from]($targetFile); } + public function uploadFile() { + if(isset($_FILES['file'])) + { + $dir = CACHE . '/'; + $file = basename($_FILES['file']['name']); + if(move_uploaded_file($_FILES['file']['tmp_name'], $dir . $file)) { + $this->messages->add('s', _('File uploaded. You can now execute import.')); + } + else { + $this->messages->add('e', _('Error while importing file. Do you have access to upload it?')); + } + } + + Tools::redirect('?view=config'); + } + /** * export poche entries in json * @return json all poche entries diff --git a/inc/poche/Tools.class.php b/inc/poche/Tools.class.php index 4ed28ed1b..eed7afbdd 100644 --- a/inc/poche/Tools.class.php +++ b/inc/poche/Tools.class.php @@ -241,7 +241,6 @@ class Tools } } - public static function download_db() { header('Content-Disposition: attachment; filename="poche.sqlite.gz"'); self::status(200); diff --git a/index.php b/index.php index 9f5d0adee..ecae91198 100644 --- a/index.php +++ b/index.php @@ -75,6 +75,8 @@ if (isset($_GET['login'])) { $poche->updateTheme(); } elseif (isset($_GET['updatelanguage'])) { $poche->updateLanguage(); +} elseif (isset($_GET['uploadfile'])) { + $poche->uploadFile(); } elseif (isset($_GET['feed'])) { if (isset($_GET['action']) && $_GET['action'] == 'generate') { $poche->generateToken(); diff --git a/themes/baggy/config.twig b/themes/baggy/config.twig index b37ac115a..4026bf28a 100644 --- a/themes/baggy/config.twig +++ b/themes/baggy/config.twig @@ -103,13 +103,26 @@ {% endif %}

{% trans "Import" %}

-

{% trans "Please execute the import script locally as it can take a very long time." %}

-

{% trans "More info in the official documentation:" %} wallabag.org

+

1. {% trans "Select a file on your computer and upload it." %}

+
+
+
+ + +
+
+ +
+
+ + +
+

2. {% trans "Then, click on the right link below." %}

{% trans "Export your wallabag data" %}

From 53e3158dfe697ea59da1fa0e401e8da75ae13030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 28 Feb 2014 21:49:38 +0100 Subject: [PATCH 2/3] [add] cron to fetch content on imported entries --- cron.php | 53 +++++++++++++++++++++++ inc/poche/Database.class.php | 67 +++++++++++++++++++--------- inc/poche/Poche.class.php | 84 ++++++++---------------------------- inc/poche/Tools.class.php | 56 +++++++++++++++++++++++- themes/baggy/config.twig | 8 ++++ 5 files changed, 180 insertions(+), 88 deletions(-) create mode 100644 cron.php diff --git a/cron.php b/cron.php new file mode 100644 index 000000000..cc137ca28 --- /dev/null +++ b/cron.php @@ -0,0 +1,53 @@ +getConfigUser($user_id); + +if ($token != $config['token']) { + die(_('Uh, there is a problem with the cron.')); +} + +$items = $store->retrieveUnfetchedEntries($user_id, $limit); + +foreach ($items as $item) { + $url = new Url(base64_encode($item['url'])); + $content = Tools::getPageContent($url); + + $title = ($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled'); + $body = $content['rss']['channel']['item']['description']; + + // // clean content from prevent xss attack + // $config = HTMLPurifier_Config::createDefault(); + // $purifier = new HTMLPurifier($config); + // $title = $purifier->purify($title); + // $body = $purifier->purify($body); + + + $store->updateContentAndTitle($item['id'], $title, $body, $user_id); +} \ No newline at end of file diff --git a/inc/poche/Database.class.php b/inc/poche/Database.class.php index c998fe14d..edc775f59 100755 --- a/inc/poche/Database.class.php +++ b/inc/poche/Database.class.php @@ -230,8 +230,30 @@ class Database { } } + public function updateContentAndTitle($id, $title, $body, $user_id) { + $sql_action = 'UPDATE entries SET content = ?, title = ? WHERE id=? AND user_id=?'; + $params_action = array($body, $title, $id, $user_id); + $query = $this->executeQuery($sql_action, $params_action); + + return $query; + } + + public function retrieveUnfetchedEntries($user_id, $limit) { + + $sql_limit = "LIMIT 0,".$limit; + if (STORAGE == 'postgres') { + $sql_limit = "LIMIT ".$limit." OFFSET 0"; + } + + $sql = "SELECT * FROM entries WHERE (content = '' OR content IS NULL) AND user_id=? ORDER BY id " . $sql_limit; + $query = $this->executeQuery($sql, array($user_id)); + $entries = $query->fetchAll(); + + return $entries; + } + public function retrieveAll($user_id) { - $sql = "SELECT * FROM entries WHERE user_id=? ORDER BY id"; + $sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? ORDER BY id"; $query = $this->executeQuery($sql, array($user_id)); $entries = $query->fetchAll(); @@ -250,7 +272,7 @@ class Database { public function retrieveOneByURL($url, $user_id) { $entry = NULL; - $sql = "SELECT * FROM entries WHERE url=? AND user_id=?"; + $sql = "SELECT * FROM entries WHERE content <> '' AND url=? AND user_id=?"; $params = array($url, $user_id); $query = $this->executeQuery($sql, $params); $entry = $query->fetchAll(); @@ -267,21 +289,22 @@ class Database { public function getEntriesByView($view, $user_id, $limit = '', $tag_id = 0) { switch ($view) { case 'archive': - $sql = "SELECT * FROM entries WHERE user_id=? AND is_read=? "; + $sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? "; $params = array($user_id, 1); break; case 'fav' : - $sql = "SELECT * FROM entries WHERE user_id=? AND is_fav=? "; + $sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_fav=? "; $params = array($user_id, 1); break; case 'tag' : $sql = "SELECT entries.* FROM entries LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id - WHERE entries.user_id=? AND tags_entries.tag_id = ? "; + WHERE entries.content <> '' AND + entries.user_id=? AND tags_entries.tag_id = ? "; $params = array($user_id, $tag_id); break; default: - $sql = "SELECT * FROM entries WHERE user_id=? AND is_read=? "; + $sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? "; $params = array($user_id, 0); break; } @@ -294,24 +317,25 @@ class Database { return $entries; } - public function getEntriesByViewCount($view, $user_id, $tag_id = 0) { - switch ($view) { + public function getEntriesByViewCount($view, $user_id, $tag_id = 0) { + switch ($view) { case 'archive': - $sql = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? "; + $sql = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? "; $params = array($user_id, 1); break; case 'fav' : - $sql = "SELECT count(*) FROM entries WHERE user_id=? AND is_fav=? "; + $sql = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_fav=? "; $params = array($user_id, 1); break; - case 'tag' : - $sql = "SELECT count(*) FROM entries - LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id - WHERE entries.user_id=? AND tags_entries.tag_id = ? "; - $params = array($user_id, $tag_id); - break; + case 'tag' : + $sql = "SELECT count(*) FROM entries + LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id + WHERE entries.content <> '' AND + entries.user_id=? AND tags_entries.tag_id = ? "; + $params = array($user_id, $tag_id); + break; default: - $sql = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? "; + $sql = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? "; $params = array($user_id, 0); break; } @@ -319,7 +343,7 @@ class Database { $query = $this->executeQuery($sql, $params); list($count) = $query->fetch(); - return $count; + return $count; } public function updateContent($id, $content, $user_id) { @@ -369,7 +393,7 @@ class Database { $sql = "SELECT DISTINCT tags.* FROM tags LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id LEFT JOIN entries ON tags_entries.entry_id=entries.id - WHERE entries.user_id=?"; + WHERE entries.content <> '' AND entries.user_id=?"; $query = $this->executeQuery($sql, array($user_id)); $tags = $query->fetchAll(); @@ -381,7 +405,7 @@ class Database { $sql = "SELECT DISTINCT tags.* FROM tags LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id LEFT JOIN entries ON tags_entries.entry_id=entries.id - WHERE tags.id=? AND entries.user_id=?"; + WHERE entries.content <> '' AND tags.id=? AND entries.user_id=?"; $params = array(intval($id), $user_id); $query = $this->executeQuery($sql, $params); $tag = $query->fetchAll(); @@ -393,7 +417,8 @@ class Database { $sql = "SELECT entries.* FROM entries LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id - WHERE tags_entries.tag_id = ? AND entries.user_id=?"; + WHERE entries.content <> '' AND + tags_entries.tag_id = ? AND entries.user_id=?"; $query = $this->executeQuery($sql, array($tag_id, $user_id)); $entries = $query->fetchAll(); diff --git a/inc/poche/Poche.class.php b/inc/poche/Poche.class.php index 026b0b4e1..5a89a8d27 100755 --- a/inc/poche/Poche.class.php +++ b/inc/poche/Poche.class.php @@ -362,60 +362,6 @@ class Poche ); } - protected function getPageContent(Url $url) - { - // Saving and clearing context - $REAL = array(); - foreach( $GLOBALS as $key => $value ) { - if( $key != 'GLOBALS' && $key != '_SESSION' && $key != 'HTTP_SESSION_VARS' ) { - $GLOBALS[$key] = array(); - $REAL[$key] = $value; - } - } - // Saving and clearing session - $REAL_SESSION = array(); - foreach( $_SESSION as $key => $value ) { - $REAL_SESSION[$key] = $value; - unset($_SESSION[$key]); - } - - // Running code in different context - $scope = function() { - extract( func_get_arg(1) ); - $_GET = $_REQUEST = array( - "url" => $url->getUrl(), - "max" => 5, - "links" => "preserve", - "exc" => "", - "format" => "json", - "submit" => "Create Feed" - ); - ob_start(); - require func_get_arg(0); - $json = ob_get_flush(); - return $json; - }; - $json = $scope( "inc/3rdparty/makefulltextfeed.php", array("url" => $url) ); - - // Clearing and restoring context - foreach( $GLOBALS as $key => $value ) { - if( $key != "GLOBALS" && $key != "_SESSION" ) { - unset($GLOBALS[$key]); - } - } - foreach( $REAL as $key => $value ) { - $GLOBALS[$key] = $value; - } - // Clearing and restoring session - foreach( $_SESSION as $key => $value ) { - unset($_SESSION[$key]); - } - foreach( $REAL_SESSION as $key => $value ) { - $_SESSION[$key] = $value; - } - return json_decode($json, true); - } - /** * Call action (mark as fav, archive, delete, etc.) */ @@ -424,15 +370,21 @@ class Poche switch ($action) { case 'add': - $content = $this->getPageContent($url); - $title = ($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled'); - $body = $content['rss']['channel']['item']['description']; + if (!$import) { + $content = Tools::getPageContent($url); + $title = ($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled'); + $body = $content['rss']['channel']['item']['description']; - // clean content from prevent xss attack - $config = HTMLPurifier_Config::createDefault(); - $purifier = new HTMLPurifier($config); - $title = $purifier->purify($title); - $body = $purifier->purify($body); + // clean content from prevent xss attack + $config = HTMLPurifier_Config::createDefault(); + $purifier = new HTMLPurifier($config); + $title = $purifier->purify($title); + $body = $purifier->purify($body); + } + else { + $title = ''; + $body = ''; + } //search for possible duplicate if not in import mode if (!$import) { @@ -897,7 +849,7 @@ class Poche # the second
    is for read links $read = 1; } - $this->messages->add('s', _('import from instapaper completed')); + $this->messages->add('s', _('import from instapaper completed. You have to execute the cron to fetch content.')); Tools::logm('import from instapaper completed'); Tools::redirect(); } @@ -941,7 +893,7 @@ class Poche # the second
      is for read links $read = 1; } - $this->messages->add('s', _('import from pocket completed')); + $this->messages->add('s', _('import from pocket completed. You have to execute the cron to fetch content.')); Tools::logm('import from pocket completed'); Tools::redirect(); } @@ -997,7 +949,7 @@ class Poche } } } - $this->messages->add('s', _('import from Readability completed. ' . $count . ' new links.')); + $this->messages->add('s', _('import from Readability completed. You have to execute the cron to fetch content.')); Tools::logm('import from Readability completed'); Tools::redirect(); } @@ -1043,7 +995,7 @@ class Poche } } - $this->messages->add('s', _('import from Poche completed. ' . $count . ' new links.')); + $this->messages->add('s', _('import from Poche completed. You have to execute the cron to fetch content.')); Tools::logm('import from Poche completed'); Tools::redirect(); } diff --git a/inc/poche/Tools.class.php b/inc/poche/Tools.class.php index eed7afbdd..393a415dd 100644 --- a/inc/poche/Tools.class.php +++ b/inc/poche/Tools.class.php @@ -193,7 +193,7 @@ class Tools public static function logm($message) { - if (DEBUG_POCHE) { + if (DEBUG_POCHE && php_sapi_name() != 'cli') { $t = strval(date('Y/m/d_H:i:s')) . ' - ' . $_SERVER["REMOTE_ADDR"] . ' - ' . strval($message) . "\n"; file_put_contents(CACHE . '/log.txt', $t, FILE_APPEND); error_log('DEBUG POCHE : ' . $message); @@ -251,4 +251,58 @@ class Tools exit; } + + public static function getPageContent(Url $url) + { + // Saving and clearing context + $REAL = array(); + foreach( $GLOBALS as $key => $value ) { + if( $key != 'GLOBALS' && $key != '_SESSION' && $key != 'HTTP_SESSION_VARS' ) { + $GLOBALS[$key] = array(); + $REAL[$key] = $value; + } + } + // Saving and clearing session + $REAL_SESSION = array(); + foreach( $_SESSION as $key => $value ) { + $REAL_SESSION[$key] = $value; + unset($_SESSION[$key]); + } + + // Running code in different context + $scope = function() { + extract( func_get_arg(1) ); + $_GET = $_REQUEST = array( + "url" => $url->getUrl(), + "max" => 5, + "links" => "preserve", + "exc" => "", + "format" => "json", + "submit" => "Create Feed" + ); + ob_start(); + require func_get_arg(0); + $json = ob_get_flush(); + return $json; + }; + $json = $scope( "inc/3rdparty/makefulltextfeed.php", array("url" => $url) ); + + // Clearing and restoring context + foreach( $GLOBALS as $key => $value ) { + if( $key != "GLOBALS" && $key != "_SESSION" ) { + unset($GLOBALS[$key]); + } + } + foreach( $REAL as $key => $value ) { + $GLOBALS[$key] = $value; + } + // Clearing and restoring session + foreach( $_SESSION as $key => $value ) { + unset($_SESSION[$key]); + } + foreach( $REAL_SESSION as $key => $value ) { + $_SESSION[$key] = $value; + } + return json_decode($json, true); + } } diff --git a/themes/baggy/config.twig b/themes/baggy/config.twig index 4026bf28a..8f35d7978 100644 --- a/themes/baggy/config.twig +++ b/themes/baggy/config.twig @@ -125,6 +125,14 @@
    • {% trans "Import from wallabag" %} {{ '(after uploaded %s file)'|trans|format(constant('POCHE_FILE')) }}
    + {% if token == '' %} +

    {% trans "3. Your feed token is currently empty and must first be generated to fetch content. Click here to generate it." %}

    + {% else %} +

    3. {% trans "You can fetch content for imported items." %} Click here to fetch content for 10 articles.

    +

    {% trans "You can also create a cron task:" %}

    + 
    0 */4 * * *  cd /path/to/wallabag && php cron.php --limit=10 --user-id={{user_id}} --token={{token}} >/dev/null 2>&1
    + {% endif %} +

    {% trans "Export your wallabag data" %}

    {% if constant('STORAGE') == 'sqlite' %}

    {% trans "Click here" %} {% trans "to download your database." %}

    {% endif %} From f98373cc34f1826ba93fd93ae7f291726af1f933 Mon Sep 17 00:00:00 2001 From: Maryana Rozhankivska Date: Fri, 7 Mar 2014 12:54:08 +0200 Subject: [PATCH 3/3] getPageContent moved to Tools, fix of #426 --- inc/poche/Tools.class.php | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/inc/poche/Tools.class.php b/inc/poche/Tools.class.php index 393a415dd..eeb101b40 100644 --- a/inc/poche/Tools.class.php +++ b/inc/poche/Tools.class.php @@ -263,10 +263,12 @@ class Tools } } // Saving and clearing session - $REAL_SESSION = array(); - foreach( $_SESSION as $key => $value ) { - $REAL_SESSION[$key] = $value; - unset($_SESSION[$key]); + if ( isset($_SESSION) ) { + $REAL_SESSION = array(); + foreach( $_SESSION as $key => $value ) { + $REAL_SESSION[$key] = $value; + unset($_SESSION[$key]); + } } // Running code in different context @@ -282,7 +284,8 @@ class Tools ); ob_start(); require func_get_arg(0); - $json = ob_get_flush(); + $json = ob_get_contents(); + ob_end_clean(); return $json; }; $json = $scope( "inc/3rdparty/makefulltextfeed.php", array("url" => $url) ); @@ -297,12 +300,15 @@ class Tools $GLOBALS[$key] = $value; } // Clearing and restoring session - foreach( $_SESSION as $key => $value ) { - unset($_SESSION[$key]); - } - foreach( $REAL_SESSION as $key => $value ) { - $_SESSION[$key] = $value; + if ( isset($REAL_SESSION) ) { + foreach( $_SESSION as $key => $value ) { + unset($_SESSION[$key]); + } + foreach( $REAL_SESSION as $key => $value ) { + $_SESSION[$key] = $value; + } } + return json_decode($json, true); } }