Merge pull request #2317 from wallabag/restricted-access

Added authentication for restricted access articles

app/AppKernel.php

@@ -31,6 +31,7 @@ class AppKernel extends Kernel
             new Craue\ConfigBundle\CraueConfigBundle(),
             new WhiteOctober\PagerfantaBundle\WhiteOctoberPagerfantaBundle(),
             new FOS\JsRoutingBundle\FOSJsRoutingBundle(),
+            new BD\GuzzleSiteAuthenticatorBundle\BDGuzzleSiteAuthenticatorBundle(),
 
             // wallabag bundles
             new Wallabag\CoreBundle\WallabagCoreBundle(),

app/DoctrineMigrations/Version20161122144743.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Add the restricted_access internal setting for articles with paywall
+ */
+class Version20161122144743 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $this->addSql("INSERT INTO ".$this->getTable('craue_config_setting')." (name, value, section) VALUES ('restricted_access', 0, 'entry')");
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+        $this->addSql("DELETE FROM ".$this->getTable('craue_config_setting')." WHERE name = 'restricted_access';");
+    }
+}

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Aktiver demo-indstilling? (anvendes kun til wallabags offent
 demo_mode_username: "Demobruger"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Test-Modus aktivieren? (nur für die öffentliche wallabag-D
 demo_mode_username: "Test-Benutzer"
 share_public: Erlaube eine öffentliche URL für Einträge
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
 demo_mode_username: "Demo user"
 share_public: Allow public url for entries
 download_images_enabled: Download images locally
+restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Activar modo demo (sólo usado para la demo de wallabag)"
 demo_mode_username: "Nombre de usuario demo"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml

@@ -32,3 +32,4 @@ modify_settings: "اعمال"
 # demo_mode_username: "Demo user"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Activer le mode démo ? (utiliser uniquement pour la démo p
 demo_mode_username: "Utilisateur de la démo"
 share_public: Autoriser une URL publique pour les articles
 download_images_enabled: Télécharger les images en local
+restricted_access: Activer l'authentification pour les articles derrière un paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Abilita modalità demo ? (usato solo per la demo pubblica di
 demo_mode_username: "Utente Demo"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Activar lo mode demostracion ? (utilizar solament per la dem
 demo_mode_username: "Utilizaire de la demostracion"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Włacz tryb demo? (używany wyłącznie dla publicznej demon
 demo_mode_username: "Użytkownik Demonstracyjny"
 share_public: Zezwalaj na publiczny adres url dla wpisow
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pt.yml

@@ -32,3 +32,4 @@ demo_mode_enabled: "Habilitar modo demo? (somente usado para o demo público do
 demo_mode_username: "Usuário demo"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml

@@ -32,3 +32,4 @@ modify_settings: "aplică"
 # demo_mode_username: "Demo user"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml

@@ -32,3 +32,4 @@
 # demo_mode_username: "Demo user"
 # share_public: Allow public url for entries
 # download_images_enabled: Download images locally
+# restricted_access: Enable authentication for websites with paywall

app/config/parameters.yml.dist

@@ -56,3 +56,6 @@ parameters:
     redis_port: 6379
     redis_path: null
     redis_password: null
+
+    # sites credentials
+    sites_credentials: {}

composer.json

@@ -83,7 +83,8 @@
         "predis/predis": "^1.0",
         "javibravo/simpleue": "^1.0",
         "symfony/dom-crawler": "^3.1",
-        "friendsofsymfony/jsrouting-bundle": "^1.6"
+        "friendsofsymfony/jsrouting-bundle": "^1.6",
+        "bdunogier/guzzle-site-authenticator": "^1.0@beta"
     },
     "require-dev": {
         "doctrine/doctrine-fixtures-bundle": "~2.2",

docs/de/developer/paywall.rst

@@ -0,0 +1,56 @@
+Articles behind a paywall
+=========================
+
+wallabag can fetch articles from websites which use a paywall system.
+
+Enable paywall authentication
+-----------------------------
+
+In internal settings, in the **Article** section, enable authentication for websites with paywall (with the value 1).
+
+Configure credentials in wallabag
+---------------------------------
+
+Edit your ``app/config/parameters.yml`` file to edit credentials for each website with paywall. Here is an example for some french websites:
+
+.. code:: yaml
+
+    sites_credentials:
+        mediapart.fr: {username: "myMediapartLogin", password: "mypassword"}
+        arretsurimages.net: {username: "myASILogin", password: "mypassword"}
+
+.. note::
+
+    These credentials will be shared between each user of your wallabag instance.
+
+Parsing configuration files
+---------------------------
+
+.. note::
+
+    Read `this part of the documentation <http://doc.wallabag.org/en/master/user/errors_during_fetching.html>`_ to understand the configuration files.
+
+Each parsing configuration file needs to be improved by adding ``requires_login``, ``login_uri``,
+``login_username_field``, ``login_password_field`` and ``not_logged_in_xpath``.
+
+Be careful, the login form must be in the page content when wallabag loads it. It's impossible for wallabag to be authenticated
+on a website where the login form is loaded after the page (by ajax for example).
+
+``login_uri`` is the action URL of the form (``action`` attribute in the form).
+``login_username_field`` is the ``name`` attribute of the login field.
+``login_password_field`` is the ``name`` attribute of the password field.
+
+For example:
+
+.. code::
+
+    title://div[@id="titrage-contenu"]/h1[@class="title"]
+    body: //div[@class="contenu-html"]/div[@class="page-pane"]
+
+    requires_login: yes
+
+    login_uri: http://www.arretsurimages.net/forum/login.php
+    login_username_field: username
+    login_password_field: password
+
+    not_logged_in_xpath: //body[@class="not-logged-in"]

docs/de/index.rst

@@ -46,6 +46,7 @@ Die Dokumentation ist in anderen Sprachen verfügbar :
 
    developer/api
    developer/docker
+   developer/paywall
    developer/documentation
    developer/translate
    developer/asynchronous

docs/en/developer/paywall.rst

@@ -0,0 +1,56 @@
+Articles behind a paywall
+=========================
+
+wallabag can fetch articles from websites which use a paywall system.
+
+Enable paywall authentication
+-----------------------------
+
+In internal settings, in the **Article** section, enable authentication for websites with paywall (with the value 1).
+
+Configure credentials in wallabag
+---------------------------------
+
+Edit your ``app/config/parameters.yml`` file to edit credentials for each website with paywall. Here is an example for some french websites:
+
+.. code:: yaml
+
+    sites_credentials:
+        mediapart.fr: {username: "myMediapartLogin", password: "mypassword"}
+        arretsurimages.net: {username: "myASILogin", password: "mypassword"}
+
+.. note::
+
+    These credentials will be shared between each user of your wallabag instance.
+
+Parsing configuration files
+---------------------------
+
+.. note::
+
+    Read `this part of the documentation <http://doc.wallabag.org/en/master/user/errors_during_fetching.html>`_ to understand the configuration files.
+
+Each parsing configuration file needs to be improved by adding ``requires_login``, ``login_uri``,
+``login_username_field``, ``login_password_field`` and ``not_logged_in_xpath``.
+
+Be careful, the login form must be in the page content when wallabag loads it. It's impossible for wallabag to be authenticated
+on a website where the login form is loaded after the page (by ajax for example).
+
+``login_uri`` is the action URL of the form (``action`` attribute in the form).
+``login_username_field`` is the ``name`` attribute of the login field.
+``login_password_field`` is the ``name`` attribute of the password field.
+
+For example:
+
+.. code::
+
+    title://div[@id="titrage-contenu"]/h1[@class="title"]
+    body: //div[@class="contenu-html"]/div[@class="page-pane"]
+
+    requires_login: yes
+
+    login_uri: http://www.arretsurimages.net/forum/login.php
+    login_username_field: username
+    login_password_field: password
+
+    not_logged_in_xpath: //body[@class="not-logged-in"]

docs/en/index.rst

@@ -46,6 +46,7 @@ The documentation is available in other languages:
 
    developer/api
    developer/docker
+   developer/paywall
    developer/documentation
    developer/translate
    developer/asynchronous

docs/fr/developer/paywall.rst

@@ -0,0 +1,56 @@
+Articles derrière un paywall
+============================
+
+wallabag peut récupérer le contenu des articles des sites qui utilisent un système de paiement.
+
+Activer l'authentification pour les paywall
+-------------------------------------------
+
+Dans les paramètres internes, section **Article**, activez l'authentification pour les articles derrière un paywall (avec la valeur 1).
+
+Configurer les accès dans wallabag
+----------------------------------
+
+Éditez le fichier ``app/config/parameters.yml`` pour modifier les accès aux sites avec paywall. Voici un exemple pour certains sites :
+
+.. code:: yaml
+
+    sites_credentials:
+        mediapart.fr: {username: "myMediapartLogin", password: "mypassword"}
+        arretsurimages.net: {username: "myASILogin", password: "mypassword"}
+
+.. note::
+
+    Ces accès seront partagés entre chaque utilisateur de votre instance wallabag.
+
+Fichiers de configuration pour parser les articles
+--------------------------------------------------
+
+.. note::
+
+    Lisez `cette documentation <http://doc.wallabag.org/fr/master/user/errors_during_fetching.html>`_ pour en savoir plus sur ces fichiers de configuration.
+
+Chaque fichier de configuration doit être enrichi en ajoutant ``requires_login``, ``login_uri``,
+``login_username_field``, ``login_password_field`` et ``not_logged_in_xpath``.
+
+Attention, le formulaire de connexion doit se trouver dans le contenu de la page lors du chargement de celle-ci.
+Il sera impossible pour wallabag de se connecter à un site dont le formulaire de connexion est chargé après coup (en ajax par exemple).
+
+``login_uri`` correspond à l'URL à laquelle le formulaire est soumis (attribut ``action`` du formulaire).
+``login_username_field`` correspond à l'attribut ``name`` du champ de l'identifiant.
+``login_password_field`` correspond à l'attribut ``name`` du champ du mot de passe.
+
+Par exemple :
+
+.. code::
+
+    title://div[@id="titrage-contenu"]/h1[@class="title"]
+    body: //div[@class="contenu-html"]/div[@class="page-pane"]
+
+    requires_login: yes
+
+    login_uri: http://www.arretsurimages.net/forum/login.php
+    login_username_field: username
+    login_password_field: password
+
+    not_logged_in_xpath: //body[@class="not-logged-in"]

docs/fr/index.rst

@@ -47,6 +47,7 @@ La documentation est disponible dans d'autres langues :
 
    developer/api
    developer/docker
+   developer/paywall
    developer/documentation
    developer/translate
    developer/asynchronous

src/Wallabag/CoreBundle/Command/InstallCommand.php

@@ -432,6 +432,11 @@ class InstallCommand extends ContainerAwareCommand
                 'value' => '0',
                 'section' => 'misc',
             ],
+            [
+                'name' => 'restricted_access',
+                'value' => '0',
+                'section' => 'entry',
+            ],
         ];
 
         foreach ($settings as $setting) {

src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php

@@ -155,6 +155,11 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
                 'value' => '0',
                 'section' => 'misc',
             ],
+            [
+                'name' => 'restricted_access',
+                'value' => '0',
+                'section' => 'entry',
+            ],
         ];
 
         foreach ($settings as $setting) {

src/Wallabag/CoreBundle/GuzzleSiteAuthenticator/GrabySiteConfigBuilder.php

@@ -0,0 +1,68 @@
+<?php
+
+namespace Wallabag\CoreBundle\GuzzleSiteAuthenticator;
+
+use BD\GuzzleSiteAuthenticator\SiteConfig\SiteConfig;
+use BD\GuzzleSiteAuthenticator\SiteConfig\SiteConfigBuilder;
+use Graby\SiteConfig\ConfigBuilder;
+use OutOfRangeException;
+
+class GrabySiteConfigBuilder implements SiteConfigBuilder
+{
+    /**
+     * @var \Graby\SiteConfig\ConfigBuilder
+     */
+    private $grabyConfigBuilder;
+    /**
+     * @var array
+     */
+    private $credentials;
+
+    /**
+     * GrabySiteConfigBuilder constructor.
+     *
+     * @param \Graby\SiteConfig\ConfigBuilder $grabyConfigBuilder
+     * @param array                           $credentials
+     */
+    public function __construct(ConfigBuilder $grabyConfigBuilder, array $credentials = [])
+    {
+        $this->grabyConfigBuilder = $grabyConfigBuilder;
+        $this->credentials = $credentials;
+    }
+
+    /**
+     * Builds the SiteConfig for a host.
+     *
+     * @param string $host The "www." prefix is ignored
+     *
+     * @return SiteConfig
+     *
+     * @throws OutOfRangeException If there is no config for $host
+     */
+    public function buildForHost($host)
+    {
+        // required by credentials below
+        $host = strtolower($host);
+        if (substr($host, 0, 4) == 'www.') {
+            $host = substr($host, 4);
+        }
+
+        $config = $this->grabyConfigBuilder->buildForHost($host);
+        $parameters = [
+            'host' => $host,
+            'requiresLogin' => $config->requires_login ?: false,
+            'loginUri' => $config->login_uri ?: null,
+            'usernameField' => $config->login_username_field ?: null,
+            'passwordField' => $config->login_password_field ?: null,
+            'extraFields' => is_array($config->login_extra_fields) ? $config->login_extra_fields : [],
+            'notLoggedInXpath' => $config->not_logged_in_xpath ?: null,
+        ];
+
+        if (isset($this->credentials[$host])) {
+            $parameters['username'] = $this->credentials[$host]['username'];
+            $parameters['password'] = $this->credentials[$host]['password'];
+        }
+
+        return new SiteConfig($parameters);
+    }
+}

src/Wallabag/CoreBundle/Helper/HttpClientFactory.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace Wallabag\CoreBundle\Helper;
+
+use Graby\Ring\Client\SafeCurlHandler;
+use GuzzleHttp\Client;
+use GuzzleHttp\Cookie\CookieJar;
+use GuzzleHttp\Event\SubscriberInterface;
+
+/**
+ * Builds and configures the Guzzle HTTP client.
+ */
+class HttpClientFactory
+{
+    /** @var \GuzzleHttp\Event\SubscriberInterface */
+    private $authenticatorSubscriber;
+
+    /** @var \GuzzleHttp\Cookie\CookieJar */
+    private $cookieJar;
+
+    private $restrictedAccess;
+
+    /**
+     * HttpClientFactory constructor.
+     *
+     * @param \GuzzleHttp\Event\SubscriberInterface $authenticatorSubscriber
+     * @param \GuzzleHttp\Cookie\CookieJar          $cookieJar
+     * @param string                                $restrictedAccess        this param is a kind of boolean. Values: 0 or 1
+     */
+    public function __construct(SubscriberInterface $authenticatorSubscriber, CookieJar $cookieJar, $restrictedAccess)
+    {
+        $this->authenticatorSubscriber = $authenticatorSubscriber;
+        $this->cookieJar = $cookieJar;
+        $this->restrictedAccess = $restrictedAccess;
+    }
+
+    /**
+     * @return \GuzzleHttp\Client|null
+     */
+    public function buildHttpClient()
+    {
+        if (0 === (int) $this->restrictedAccess) {
+            return null;
+        }
+
+        // we clear the cookie to avoid websites who use cookies for analytics
+        $this->cookieJar->clear();
+        // need to set the (shared) cookie jar
+        $client = new Client(['handler' => new SafeCurlHandler(), 'defaults' => ['cookies' => $this->cookieJar]]);
+        $client->getEmitter()->attach($this->authenticatorSubscriber);
+
+        return $client;
+    }
+}

src/Wallabag/CoreBundle/Resources/config/services.yml

@@ -41,11 +41,44 @@ services:
         arguments:
             -
                 error_message: '%wallabag_core.fetching_error_message%'
+            - "@wallabag_core.guzzle.http_client"
+            - "@wallabag_core.graby.config_builder"
         calls:
             - [ setLogger, [ "@logger" ] ]
         tags:
             - { name: monolog.logger, channel: graby }
 
+    wallabag_core.graby.config_builder:
+        class: Graby\SiteConfig\ConfigBuilder
+        arguments:
+            - {}
+            - "@logger"
+
+    wallabag_core.guzzle.http_client:
+        class: GuzzleHttp\ClientInterface
+        factory: ["@wallabag_core.guzzle.http_client_factory", buildHttpClient]
+
+    wallabag_core.guzzle_authenticator.config_builder:
+        class: Wallabag\CoreBundle\GuzzleSiteAuthenticator\GrabySiteConfigBuilder
+        arguments:
+            - "@wallabag_core.graby.config_builder"
+            - "%sites_credentials%"
+
+    # service alias override
+    bd_guzzle_site_authenticator.site_config_builder:
+        alias: wallabag_core.guzzle_authenticator.config_builder
+
+    wallabag_core.guzzle.http_client_factory:
+        class: Wallabag\CoreBundle\Helper\HttpClientFactory
+        arguments:
+            - "@bd_guzzle_site_authenticator.authenticator_subscriber"
+            - "@wallabag_core.guzzle.cookie_jar"
+            - '@=service(''craue_config'').get(''restricted_access'')'
+
+    wallabag_core.guzzle.cookie_jar:
+        class: GuzzleHttp\Cookie\FileCookieJar
+        arguments: ["%kernel.cache_dir%/cookiejar.json"]
+
     wallabag_core.content_proxy:
         class: Wallabag\CoreBundle\Helper\ContentProxy
         arguments:

tests/Wallabag/CoreBundle/GuzzleSiteAuthenticator/GrabySiteConfigBuilderTest.php

@@ -0,0 +1,85 @@
+<?php
+
+namespace Tests\Wallabag\CoreBundle\GuzzleSiteAuthenticator;
+
+use BD\GuzzleSiteAuthenticator\SiteConfig\SiteConfig;
+use Graby\SiteConfig\SiteConfig as GrabySiteConfig;
+use PHPUnit_Framework_TestCase;
+use Wallabag\CoreBundle\GuzzleSiteAuthenticator\GrabySiteConfigBuilder;
+
+class GrabySiteConfigBuilderTest extends PHPUnit_Framework_TestCase
+{
+    /** @var \Wallabag\CoreBundle\GuzzleSiteAuthenticator\GrabySiteConfigBuilder */
+    protected $builder;
+
+    public function testBuildConfigExists()
+    {
+        /* @var \Graby\SiteConfig\ConfigBuilder|\PHPUnit_Framework_MockObject_MockObject */
+        $grabyConfigBuilderMock = $this->getMockBuilder('\Graby\SiteConfig\ConfigBuilder')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $grabySiteConfig = new GrabySiteConfig();
+        $grabySiteConfig->requires_login = true;
+        $grabySiteConfig->login_uri = 'http://example.com/login';
+        $grabySiteConfig->login_username_field = 'login';
+        $grabySiteConfig->login_password_field = 'password';
+        $grabySiteConfig->login_extra_fields = ['field' => 'value'];
+        $grabySiteConfig->not_logged_in_xpath = '//div[@class="need-login"]';
+
+        $grabyConfigBuilderMock
+            ->method('buildForHost')
+            ->with('example.com')
+            ->will($this->returnValue($grabySiteConfig));
+
+        $this->builder = new GrabySiteConfigBuilder(
+            $grabyConfigBuilderMock,
+            ['example.com' => ['username' => 'foo', 'password' => 'bar']]
+        );
+
+        $config = $this->builder->buildForHost('example.com');
+
+        self::assertEquals(
+            new SiteConfig([
+                'host' => 'example.com',
+                'requiresLogin' => true,
+                'loginUri' => 'http://example.com/login',
+                'usernameField' => 'login',
+                'passwordField' => 'password',
+                'extraFields' => ['field' => 'value'],
+                'notLoggedInXpath' => '//div[@class="need-login"]',
+                'username' => 'foo',
+                'password' => 'bar',
+            ]),
+            $config
+        );
+    }
+
+    public function testBuildConfigDoesntExist()
+    {
+        /* @var \Graby\SiteConfig\ConfigBuilder|\PHPUnit_Framework_MockObject_MockObject */
+        $grabyConfigBuilderMock = $this->getMockBuilder('\Graby\SiteConfig\ConfigBuilder')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $grabyConfigBuilderMock
+            ->method('buildForHost')
+            ->with('unknown.com')
+            ->will($this->returnValue(new GrabySiteConfig()));
+
+        $this->builder = new GrabySiteConfigBuilder($grabyConfigBuilderMock, []);
+
+        $config = $this->builder->buildForHost('unknown.com');
+
+        self::assertEquals(
+            new SiteConfig([
+                'host' => 'unknown.com',
+                'requiresLogin' => false,
+                'username' => null,
+                'password' => null,
+                'extraFields' => [],
+            ]),
+            $config
+        );
+    }
+}