Added internal setting to enable/disable headers storage

app/DoctrineMigrations/Version20171120163128.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Add store_article_headers in craue_config_setting.
+ */
+class Version20171120163128 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $storeArticleHeaders = $this->container
+            ->get('doctrine.orm.default_entity_manager')
+            ->getConnection()
+            ->fetchArray('SELECT * FROM ' . $this->getTable('craue_config_setting') . " WHERE name = 'store_article_headers'");
+
+        $this->skipIf(false !== $storeArticleHeaders, 'It seems that you already played this migration.');
+
+        $this->addSql('INSERT INTO ' . $this->getTable('craue_config_setting') . " (name, value, section) VALUES ('store_article_headers', '0', 'entry')");
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+        $this->addSql('DELETE FROM ' . $this->getTable('craue_config_setting') . " WHERE name = 'store_article_headers';");
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+}

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml

@@ -12,6 +12,8 @@ export_xml: Aktiver eksport til XML
 # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 # import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli-URL, hvis tjenesten er aktiv
+scuttle_url: Scuttle-URL, hvis tjenesten er aktiv
+unmark_url: Unmark-URL, hvis tjenesten er aktiv
 share_diaspora: Aktiver deling til Diaspora
 share_mail: Aktiver deling med email
 share_shaarli: Aktiver deling gennem Shaarli
@@ -34,3 +36,4 @@ demo_mode_username: "Demobruger"
 # download_images_enabled: Download images locally
 # restricted_access: Enable authentication for websites with paywall
 # api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml

@@ -12,6 +12,8 @@ export_xml: 'XML-Export aktivieren'
 import_with_rabbitmq: 'Aktiviere RabbitMQ, um Artikel asynchron zu importieren'
 import_with_redis: 'Aktiviere Redis, um Artikel asynchron zu importieren'
 shaarli_url: 'Shaarli-URL, sofern der Service aktiviert ist'
+scuttle_url: 'Scuttle-URL, sofern der Service aktiviert ist'
+unmark_url: 'Unmark-URL, sofern der Service aktiviert ist'
 share_diaspora: 'Teilen zu Diaspora aktiveren'
 share_mail: 'Teilen via E-Mail aktiveren'
 share_shaarli: 'Teilen zu Shaarli aktiveren'
@@ -34,3 +36,4 @@ share_public: 'Erlaube eine öffentliche URL für Einträge'
 download_images_enabled: 'Bilder lokal herunterladen'
 restricted_access: 'Authentifizierung für Webseiten mit Paywall aktivieren'
 api_user_registration: 'Registrierung eines Benutzers über die API ermöglichen'
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml

@@ -12,6 +12,8 @@ export_xml: Enable XML export
 import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, if the service is enabled
+scuttle_url: Scuttle URL, if the service is enabled
+unmark_url: Unmark URL, if the service is enabled
 share_diaspora: Enable share to diaspora*
 share_mail: Enable share by e-mail
 share_shaarli: Enable sharing to Shaarli
@@ -34,3 +36,4 @@ share_public: Allow public URL for entries
 download_images_enabled: Download images locally
 restricted_access: Enable authentication for paywalled websites
 api_user_registration: Enable user to be registered using the API
+store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml

@@ -12,6 +12,8 @@ export_xml: Activar exportación a XML
 import_with_rabbitmq: Activar RabbitMQ para importar datos de forma asíncrona
 import_with_redis: Activar Redis para importar datos de forma asíncrona
 shaarli_url: URL de Shaarli, si el servicio está activado
+scuttle_url: URL de Scuttle, si el servicio está activado
+unmark_url: URL de Unmark, si el servicio está activado
 share_diaspora: Activar compartir con Diaspora
 share_mail: Activar compartir con Email
 share_shaarli: Activar compartir con Shaarli
@@ -34,3 +36,4 @@ share_public: Permitir URL pública para los artículos
 download_images_enabled: Descargar imágenes localmente
 restricted_access: Activar autenticación para websites con paywall
 # api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml

@@ -12,6 +12,8 @@ export_xml:  فعال‌سازی برون‌سپاری به XML
 # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 # import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: نشانی Shaarli، اگر فعال بود
+scuttle_url: نشانی Scuttle، اگر فعال بود
+unmark_url: نشانی Unmark، اگر فعال بود
 share_diaspora: فعال‌سازی هم‌رسانی به Diaspora
 share_mail:  فعال‌سازی هم‌رسانی با ایمیل
 share_shaarli:  فعال‌سازی هم‌رسانی به Shaarli
@@ -34,3 +36,4 @@ modify_settings: "اعمال"
 # download_images_enabled: Download images locally
 # restricted_access: Enable authentication for websites with paywall
 # api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml

@@ -12,6 +12,8 @@ export_xml: Activer l'export XML
 import_with_rabbitmq: Activer RabbitMQ pour gérer les imports de façon asynchrone
 import_with_redis: Activer Redis pour gérer les imports de façon asynchrone
 shaarli_url: URL de Shaarli, si le service Shaarli est activé
+scuttle_url: URL de Scuttle, si le service Scuttle est activé
+unmark_url: URL de Unmark, si le service Unmark est activé
 share_diaspora: Activer le partage vers Diaspora
 share_mail: Activer le partage par email
 share_shaarli: Activer le partage vers Shaarli
@@ -34,3 +36,4 @@ share_public: Autoriser une URL publique pour les articles
 download_images_enabled: Télécharger les images en local
 restricted_access: Activer l'authentification pour les articles derrière un paywall
 api_user_registration: Activer la création de compte depuis l'API
+store_article_headers: Activer le stockage des en-têtes HTTP de chaque article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml

@@ -12,6 +12,8 @@ export_xml: Abilita esportazione XML
 # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 # import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, se il servizio è abilitato
+scuttle_url: Scuttle URL, se il servizio è abilitato
+unmark_url: Unmark URL, se il servizio è abilitato
 share_diaspora: Abilita la condivisione con Diaspora
 share_mail: Abilita la condivisione per email
 share_shaarli: Abilita la condivisione con Shaarli
@@ -34,3 +36,4 @@ demo_mode_username: "Utente Demo"
 # download_images_enabled: Download images locally
 # restricted_access: Enable authentication for websites with paywall
 api_user_registration: Abilita la registrazione dell'utente attraverso l'API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml

@@ -12,6 +12,8 @@ export_xml: Activar l'expòrt XML
 import_with_rabbitmq: Activar RabbitMQ per importar de donadas de manièra asincròna
 import_with_redis: Activar Redis per importar de donadas de manièra asincròna
 shaarli_url: URL de Shaarli, se lo servici Shaarli es activat
+scuttle_url: URL de Scuttle, se lo servici Scuttle es activat
+unmark_url: URL de Unmark, se lo servici Scuttle es activat
 share_diaspora: Activar lo partatge cap a Diaspora*
 share_mail: Activar lo partatge per corrièl
 share_shaarli: Activar lo partatge cap a Shaarli
@@ -34,3 +36,4 @@ share_public: Autorizar una URL publica pels articles
 download_images_enabled: Telecargar los imatges en local
 restricted_access: Activar l'autenticacion pels sites amb peatge
 api_user_registration: Autorizar los utilizaires a se marcar amb l'API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml

@@ -12,6 +12,7 @@ export_xml: Włącz eksport do XML
 import_with_rabbitmq: Włącz RabbitMQ dla asynchronicznego importu danych
 import_with_redis: Włącz Redis dla asynchronicznego importu danych
 shaarli_url: Adress URL Shaarli, jeżeli usługa jest włączona
+scuttle_url: Adress URL Scuttle, jeżeli usługa jest włączona
 share_diaspora: Włącz udostępnianie dla Diaspora
 share_mail: Włącz udostępnianie przez email
 share_shaarli: Włącz udostępnianie dla Shaarli
@@ -34,3 +35,4 @@ share_public: Zezwalaj na publiczny adres url dla wpisow
 download_images_enabled: Pobierz obrazy lokalnie
 restricted_access: Włącz autoryzację dla stron z paywallem
 api_user_registration: Włącz rejestrację użytkownika przy użyciu API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pt.yml

@@ -12,6 +12,8 @@ export_xml: Habilita exportação para XML
 # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 # import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: URL Shaarli, se o serviço está habilitado
+scuttle_url: URL Scuttle, se o serviço está habilitado
+unmark_url: URL Unmark, se o serviço está habilitado
 share_diaspora: Habilitar compartilhamento para o Diaspora
 share_mail: Habilitar compartilhamento por e-mail
 share_shaarli: Habilitar compartilhamento para o Shaarli
@@ -34,3 +36,4 @@ demo_mode_username: "Usuário demo"
 # download_images_enabled: Download images locally
 # restricted_access: Enable authentication for websites with paywall
 # api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml

@@ -12,6 +12,8 @@ export_xml: Permite exportare XML
 # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 # import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, dacă serviciul este permis
+scuttle_url: Scuttle URL, dacă serviciul este permis
+unmark_url: Unmark URL, dacă serviciul este permis
 share_diaspora: Permite share către Diaspora
 share_mail: Permite share prin email
 share_shaarli: Permite share către Shaarli
@@ -34,3 +36,4 @@ modify_settings: "aplică"
 # download_images_enabled: Download images locally
 # restricted_access: Enable authentication for websites with paywall
 # api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ru.yml

@@ -12,6 +12,8 @@ export_xml: "Включить XML экспорт"
 import_with_rabbitmq: "Включить RabbitMQ для импорта данных(асинхронно)"
 import_with_redis: "Включить Redis для импорта данных(асинхронно)"
 shaarli_url: "Shaarli URL, если сервис включен"
+scuttle_url: "Scuttle URL, если сервис включен"
+unmark_url: "Unmark URL, если сервис включен"
 share_diaspora: "Включить возможность поделиться в соц.сети Diaspora"
 share_mail: "Включить возможность поделиться по email"
 share_shaarli: "Включить возможность поделиться в Shaarli"
@@ -33,3 +35,5 @@ demo_mode_username: "Демо пользователь"
 share_public: "Разрешить публичные ссылки на записи"
 download_images_enabled: "Скачивать изображения локально"
 restricted_access: "Включить авторизацию на сайте с помощью paywall"
+# api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml

@@ -12,6 +12,8 @@
 # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
 # import_with_redis: Enable Redis to import data asynchronously
 # shaarli_url: Shaarli URL, if the service is enabled
+# scuttle_url: Scuttle URL, if the service is enabled
+# unmark_url: Unmark URL, if the service is enabled
 # share_diaspora: Enable share to Diaspora
 # share_mail: Enable share by email
 # share_shaarli: Enable share to Shaarli
@@ -35,3 +37,4 @@
 # download_images_enabled: Download images locally
 # restricted_access: Enable authentication for websites with paywall
 # api_user_registration: Enable user to be registered using the API
+# store_article_headers: Enable if wallabag stores HTTP headers for each article

app/config/wallabag.yml

@@ -153,6 +153,10 @@ wallabag_core:
             name: api_user_registration
             value: 0
             section: api
+        -
+            name: store_article_headers
+            value: 0
+            section: entry
 
 wallabag_user:
     registration_enabled: "%fosuser_registration%"

src/Wallabag/CoreBundle/Helper/ContentProxy.php

@@ -24,8 +24,9 @@ class ContentProxy
     protected $mimeGuesser;
     protected $fetchingErrorMessage;
     protected $eventDispatcher;
+    protected $storeArticleHeaders;
 
-    public function __construct(Graby $graby, RuleBasedTagger $tagger, ValidatorInterface $validator, LoggerInterface $logger, $fetchingErrorMessage)
+    public function __construct(Graby $graby, RuleBasedTagger $tagger, ValidatorInterface $validator, LoggerInterface $logger, $fetchingErrorMessage, $storeArticleHeaders)
     {
         $this->graby = $graby;
         $this->tagger = $tagger;
@@ -33,6 +34,7 @@ class ContentProxy
         $this->logger = $logger;
         $this->mimeGuesser = new MimeTypeExtensionGuesser();
         $this->fetchingErrorMessage = $fetchingErrorMessage;
+        $this->storeArticleHeaders = $storeArticleHeaders;
     }
 
     /**
@@ -185,7 +187,7 @@ class ContentProxy
             $entry->setPublishedBy($content['authors']);
         }
 
-        if (!empty($content['all_headers'])) {
+        if (!empty($content['all_headers']) && $this->storeArticleHeaders) {
             $entry->setHeaders($content['all_headers']);
         }
 

src/Wallabag/CoreBundle/Resources/config/services.yml

@@ -94,6 +94,7 @@ services:
             - "@validator"
             - "@logger"
             - '%wallabag_core.fetching_error_message%'
+            - '@=service(''craue_config'').get(''store_article_headers'')'
 
     wallabag_core.tags_assigner:
         class: Wallabag\CoreBundle\Helper\TagsAssigner

tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php

@@ -137,6 +137,8 @@ class EntryControllerTest extends WallabagCoreTestCase
         $this->logInAs('admin');
         $client = $this->getClient();
 
+        $client->getContainer()->get('craue_config')->set('store_article_headers', 1);
+
         $crawler = $client->request('GET', '/new');
 
         $this->assertSame(200, $client->getResponse()->getStatusCode());
@@ -165,6 +167,7 @@ class EntryControllerTest extends WallabagCoreTestCase
         $this->assertSame('2015-03-28 11:43:19', $content->getPublishedAt()->format('Y-m-d H:i:s'));
         $this->assertSame('Morgane Tual', $author[0]);
         $this->assertArrayHasKey('x-varnish1', $content->getHeaders());
+        $client->getContainer()->get('craue_config')->set('store_article_headers', 0);
     }
 
     public function testPostWithMultipleAuthors()

tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php

@@ -40,7 +40,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 'language' => '',
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://user@:80');
 
@@ -75,7 +75,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 'language' => '',
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://0.0.0.0');
 
@@ -115,7 +115,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 ],
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://domain.io');
 
@@ -157,7 +157,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 ],
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://0.0.0.0');
 
@@ -199,7 +199,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 ],
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://0.0.0.0');
 
@@ -241,7 +241,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 'status' => '200',
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $validator, $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $validator, $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://0.0.0.0');
 
@@ -290,7 +290,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 ],
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $validator, $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $validator, $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://0.0.0.0');
 
@@ -311,7 +311,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
         $tagger->expects($this->once())
             ->method('tag');
 
-        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry(
             $entry,
@@ -353,7 +353,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
         $logHandler = new TestHandler();
         $logger = new Logger('test', [$logHandler]);
 
-        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $logger, $this->fetchingErrorMessage);
+        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $logger, $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry(
             $entry,
@@ -388,7 +388,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
         $handler = new TestHandler();
         $logger->pushHandler($handler);
 
-        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $logger, $this->fetchingErrorMessage);
+        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $logger, $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry(
             $entry,
@@ -425,7 +425,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
             ->method('tag')
             ->will($this->throwException(new \Exception()));
 
-        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry(
             $entry,
@@ -465,7 +465,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
         $tagger->expects($this->once())
             ->method('tag');
 
-        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy((new Graby()), $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry(
             $entry,
@@ -517,7 +517,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
                 'open_graph' => [],
             ]);
 
-        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage);
+        $proxy = new ContentProxy($graby, $tagger, $this->getValidator(), $this->getLogger(), $this->fetchingErrorMessage, false);
         $entry = new Entry(new User());
         $proxy->updateEntry($entry, 'http://0.0.0.0');