Merge pull request #1941 from wallabag/v2-asynchronous-jobs

Use asynchronous jobs for imports

.travis.yml

@@ -1,5 +1,9 @@
 language: php
 
+services:
+  - rabbitmq
+  - redis
+
 # faster builds on docker-container setup
 sudo: false
 

app/AppKernel.php

@@ -38,6 +38,7 @@ class AppKernel extends Kernel
             new Wallabag\UserBundle\WallabagUserBundle(),
             new Wallabag\ImportBundle\WallabagImportBundle(),
             new Wallabag\AnnotationBundle\WallabagAnnotationBundle(),
+            new OldSound\RabbitMqBundle\OldSoundRabbitMqBundle(),
         ];
 
         if (in_array($this->getEnvironment(), ['dev', 'test'], true)) {

app/DoctrineMigrations/Version20160911214952.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+class Version20160911214952 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_redis\', \'0\', \'import\')');
+        $this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_rabbitmq\', \'0\', \'import\')');
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+    }
+}

app/DoctrineMigrations/Version20160916201049.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+class Version20160916201049 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $this->addSql('ALTER TABLE '.$this->getTable('config').' ADD pocket_consumer_key VARCHAR(255) DEFAULT NULL');
+        $this->addSql("DELETE FROM `".$this->getTable('craue_config_setting')."` WHERE `name` = 'pocket_consumer_key';");
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.');
+
+        $this->addSql('ALTER TABLE `'.$this->getTable('config').'` DROP pocket_consumer_key');
+        $this->addSql("INSERT INTO `".$this->getTable('craue_config_setting')."` (`name`, `value`, `section`) VALUES ('pocket_consumer_key', NULL, 'import')");
+    }
+}

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml

@@ -8,7 +8,8 @@ export_csv: Aktiver eksport til CSV
 export_json: Aktiver eksport til JSON
 export_txt: Aktiver eksport til TXT
 export_xml: Aktiver eksport til XML
-pocket_consumer_key: Brugers nøgle til Pocket for at importere materialer (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli-URL, hvis tjenesten er aktiv
 share_diaspora: Aktiver deling til Diaspora
 share_mail: Aktiver deling med email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml

@@ -8,7 +8,8 @@ export_csv: CSV-Export aktivieren
 export_json: JSON-Export aktivieren
 export_txt: TXT-Export aktivieren
 export_xml: XML-Export aktivieren
-pocket_consumer_key: Consumer-Key für Pocket, um Inhalte zu importieren (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli-URL, sofern der Service aktiviert ist
 share_diaspora: Teilen zu Diaspora aktiveren
 share_mail: Teilen via E-Mail aktiveren

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml

@@ -8,7 +8,8 @@ export_csv: Enable CSV export
 export_json: Enable JSON export
 export_txt: Enable TXT export
 export_xml: Enable XML export
-pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication)
+import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, if the service is enabled
 share_diaspora: Enable share to Diaspora
 share_mail: Enable share by email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml

@@ -8,7 +8,8 @@ export_csv: Activar exportación a CSV
 export_json: Activar exportación a JSON
 export_txt: Activar exportación a TXT
 export_xml: Activar exportación a XML
-pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, si el servicio está activado
 share_diaspora: Activar compartir con Diaspora
 share_mail: Activar compartir con email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml

@@ -8,7 +8,8 @@ export_csv:  فعال‌سازی برون‌سپاری به CSV
 export_json:  فعال‌سازی برون‌سپاری به JSON
 export_txt:  فعال‌سازی برون‌سپاری به TXT
 export_xml:  فعال‌سازی برون‌سپاری به XML
-pocket_consumer_key: کلید کاربری Pocket برای درون‌ریزی مطالب (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: نشانی Shaarli، اگر فعال بود
 share_diaspora: فعال‌سازی هم‌رسانی به Diaspora
 share_mail:  فعال‌سازی هم‌رسانی با ایمیل

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml

@@ -8,7 +8,8 @@ export_csv: Activer l'export CSV
 export_json: Activer l'export JSON
 export_txt: Activer l'export TXT
 export_xml: Activer l'export XML
-pocket_consumer_key: Clé d'authentification Pocket pour importer les données (https://getpocket.com/developer/docs/authentication)
+import_with_rabbitmq: Activer RabbitMQ pour gérer les imports de façon asynchrone
+import_with_redis: Activer Redis pour gérer les imports de façon asynchrone
 shaarli_url: URL de Shaarli, si le service Shaarli est activé
 share_diaspora: Activer le partage vers Diaspora
 share_mail: Activer le partage par email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml

@@ -8,7 +8,8 @@ export_csv: Abilita esportazione CSV
 export_json: Abilita esportazione JSON
 export_txt: Abilita esportazione TXT
 export_xml: Abilita esportazione XML
-pocket_consumer_key: Consumer key per Pocket per importare i contenuti (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, se il servizio è abilitato
 share_diaspora: Abilita la condivisione con Diaspora
 share_mail: Abilita la condivisione per email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml

@@ -8,7 +8,8 @@ export_csv: Activar l'expòrt CSV
 export_json: Activar l'expòrt JSON
 export_txt: Activar l'expòrt TXT
 export_xml: Activar l'expòrt XML
-pocket_consumer_key: Clau d'autentificacion Pocket per importar las donadas (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: URL de Shaarli, se lo servici Shaarli es activat
 share_diaspora: Activar lo partatge cap a Diaspora
 share_mail: Activar lo partatge per corrièl

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml

@@ -8,7 +8,8 @@ export_csv: Włącz eksport do CSV
 export_json: Włącz eksport do JSON
 export_txt: Włącz eksport do TXT
 export_xml: Włącz eksport do XML
-pocket_consumer_key: Klucz klienta Pocket do importu zawartości (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Adress URL Shaarli, jeżeli usługa jest włączona
 share_diaspora: Włącz udostępnianie dla Diaspora
 share_mail: Włącz udostępnianie przez email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml

@@ -8,7 +8,8 @@ export_csv: Permite exportare CSV
 export_json: Permite exportare JSON
 export_txt: Permite exportare TXT
 export_xml: Permite exportare XML
-pocket_consumer_key: Cheie consumator pentru importarea contentului din Pocket (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, dacă serviciul este permis
 share_diaspora: Permite share către Diaspora
 share_mail: Permite share prin email

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml

@@ -0,0 +1,31 @@
+# download_pictures: Download pictures on your server
+# carrot: Enable share to Carrot
+# diaspora_url: Diaspora URL, if the service is enabled
+# export_epub: Enable ePub export
+# export_mobi: Enable .mobi export
+# export_pdf: Enable PDF export
+# export_csv: Enable CSV export
+# export_json: Enable JSON export
+# export_txt: Enable TXT export
+# export_xml: Enable XML export
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
+# shaarli_url: Shaarli URL, if the service is enabled
+# share_diaspora: Enable share to Diaspora
+# share_mail: Enable share by email
+# share_shaarli: Enable share to Shaarli
+# share_twitter: Enable share to Twitter
+# show_printlink: Display a link to print content
+# wallabag_support_url: Support URL for wallabag
+# wallabag_url: URL of *your* wallabag instance
+# entry: "article"
+# export: "export"
+# import: "import"
+# misc: "misc"
+# modify_settings: "apply"
+# piwik_host: Host of your website in Piwik
+# piwik_site_id: ID of your website in Piwik
+# piwik_enabled: Enable Piwik
+# demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
+# demo_mode_username: "Demo user"
+# share_public: Allow public url for entries

app/config/config.yml

@@ -215,3 +215,67 @@ lexik_maintenance:
     response:
         code: 503
         status: "wallabag Service Temporarily Unavailable"
+
+old_sound_rabbit_mq:
+    connections:
+        default:
+            host:     "%rabbitmq_host%"
+            port:     "%rabbitmq_port%"
+            user:     "%rabbitmq_user%"
+            password: "%rabbitmq_password%"
+            vhost:    /
+            lazy:     true
+    producers:
+        import_pocket:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.pocket'
+                type: topic
+        import_readability:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.readability'
+                type: topic
+        import_wallabag_v1:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v1'
+                type: topic
+        import_wallabag_v2:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v2'
+                type: topic
+    consumers:
+        import_pocket:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.pocket'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.pocket'
+            callback: wallabag_import.consumer.amqp.pocket
+        import_readability:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.readability'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.readability'
+            callback: wallabag_import.consumer.amqp.readability
+        import_wallabag_v1:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v1'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.wallabag_v1'
+            callback: wallabag_import.consumer.amqp.wallabag_v1
+        import_wallabag_v2:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v2'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.wallabag_v2'
+            callback: wallabag_import.consumer.amqp.wallabag_v2

app/config/parameters.yml.dist

@@ -38,3 +38,15 @@ parameters:
     fosuser_confirmation: true
 
     from_email: no-reply@wallabag.org
+
+    rss_limit: 50
+
+    # RabbitMQ processing
+    rabbitmq_host: localhost
+    rabbitmq_port: 5672
+    rabbitmq_user: guest
+    rabbitmq_password: guest
+
+    # Redis processing
+    redis_host: localhost
+    redis_port: 6379

app/config/parameters_test.yml

@@ -5,4 +5,4 @@ parameters:
     test_database_name: null
     test_database_user: null
     test_database_password: null
-    test_database_path: '%kernel.root_dir%/../data/db/wallabag_testYO.sqlite'
+    test_database_path: '%kernel.root_dir%/../data/db/wallabag_test.sqlite'

composer.json

@@ -81,7 +81,10 @@
         "lexik/maintenance-bundle": "~2.1",
         "ocramius/proxy-manager": "1.*",
         "white-october/pagerfanta-bundle": "^1.0",
-        "mouf/nodejs-installer": "~1.0"
+        "mouf/nodejs-installer": "~1.0",
+        "php-amqplib/rabbitmq-bundle": "^1.8",
+        "predis/predis": "^1.0",
+        "javibravo/simpleue": "^1.0"
     },
     "require-dev": {
         "doctrine/doctrine-fixtures-bundle": "~2.2",
@@ -89,7 +92,8 @@
         "sensio/generator-bundle": "^3.0",
         "phpunit/phpunit": "~5.0",
         "symfony/phpunit-bridge": "^3.0",
-        "friendsofphp/php-cs-fixer": "~1.9"
+        "friendsofphp/php-cs-fixer": "~1.9",
+        "m6web/redis-mock": "^2.0"
     },
     "scripts": {
         "post-cmd": [

docker-compose.yml

@@ -11,6 +11,7 @@ services:
         links:
             - php:php
         command: nginx -c /nginx.conf
+
     php:
         build:
             context: docker/php
@@ -30,6 +31,7 @@ services:
             # If all DBMS are commented out, sqlite will be used as default
             # - ./docker/postgres/env
             # - ./docker/mariadb/env
+
     #postgres:
     #    image: postgres:9
     #    ports:
@@ -38,6 +40,7 @@ services:
     #        - ./docker/data/pgsql:/var/lib/postgresql/data
     #    env_file:
     #        - ./docker/postgres/env
+
     #mariadb:
     #    image: mariadb:10
     #    ports:
@@ -46,3 +49,13 @@ services:
     #        - ./docker/data/mariadb:/var/lib/mysql
     #    env_file:
     #        - ./docker/mariadb/env
+
+    rabbitmq:
+        image: rabbitmq:3-management
+        ports:
+            - "15672:15672"
+
+    redis:
+        image: redis
+        ports:
+            - "6379:6379"

docs/en/developer/rabbitmq.rst

@@ -0,0 +1,67 @@
+Install RabbitMQ for asynchronous tasks
+=======================================
+
+In order to launch asynchronous tasks (useful for huge imports for example), we can use RabbitMQ.
+
+Requirements
+------------
+
+You need to have RabbitMQ installed on your server.
+
+Installation
+~~~~~~~~~~~~
+
+.. code:: bash
+
+  wget https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+  apt-key add rabbitmq-signing-key-public.asc
+  apt-get update
+  apt-get install rabbitmq-server
+
+Configuration and launch
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  rabbitmq-plugins enable rabbitmq_management # (useful to have a web interface, available at http://localhost:15672/ (guest/guest)
+  rabbitmq-server -detached
+
+Stop RabbitMQ
+~~~~~~~~~~~~~
+
+.. code:: bash
+
+  rabbitmqctl stop
+
+
+Configure RabbitMQ in wallabag
+------------------------------
+
+Edit your ``parameters.yml`` file to edit RabbitMQ configuration. The default one should be ok:
+
+.. code:: yaml
+
+    rabbitmq_host: localhost
+    rabbitmq_port: 5672
+    rabbitmq_user: guest
+    rabbitmq_password: guest
+
+
+Launch RabbitMQ consumer
+------------------------
+
+Depending on which service you want to import from you need to enable one (or many if you want to support many) cron job:
+
+.. code:: bash
+
+  # for Pocket import
+  bin/console rabbitmq:consumer import_pocket -w
+
+  # for Readbility import
+  bin/console rabbitmq:consumer import_readability -w
+
+  # for wallabag v1 import
+  bin/console rabbitmq:consumer import_wallabag_v1 -w
+
+  # for wallabag v2 import
+  bin/console rabbitmq:consumer import_wallabag_v2 -w

docs/en/developer/redis.rst

@@ -0,0 +1,62 @@
+Install Redis for asynchronous tasks
+=======================================
+
+In order to launch asynchronous tasks (useful for huge imports for example), we can use Redis.
+
+Requirements
+------------
+
+You need to have Redis installed on your server.
+
+Installation
+~~~~~~~~~~~~
+
+.. code:: bash
+
+  apt-get install redis-server
+
+Launch
+~~~~~~
+
+The server might be already running after installing, if not you can launch it using:
+
+.. code:: bash
+
+  redis-server
+
+
+Configure Redis in wallabag
+---------------------------
+
+Edit your ``parameters.yml`` file to edit Redis configuration. The default one should be ok:
+
+.. code:: yaml
+
+    redis_host: localhost
+    redis_port: 6379
+
+
+Launch Redis consumer
+------------------------
+
+Depending on which service you want to import from you need to enable one (or many if you want to support many) cron job:
+
+.. code:: bash
+
+  # for Pocket import
+  bin/console wallabag:import:redis-worker pocket -vv >> /path/to/wallabag/var/logs/redis-pocket.log
+
+  # for Readbility import
+  bin/console wallabag:import:redis-worker readability -vv >> /path/to/wallabag/var/logs/redis-readability.log
+
+  # for wallabag v1 import
+  bin/console wallabag:import:redis-worker wallabag_v1 -vv >> /path/to/wallabag/var/logs/redis-wallabag_v1.log
+
+  # for wallabag v2 import
+  bin/console wallabag:import:redis-worker wallabag_v2 -vv >> /path/to/wallabag/var/logs/redis-wallabag_v2.log
+
+If you want to launch the import only for some messages and not all, you can specify this number (here 12) and the worker will stop right after the 12th message :
+
+.. code:: bash
+
+  bin/console wallabag:import:redis-worker pocket -vv --maxIterations=12

src/Wallabag/CoreBundle/Command/InstallCommand.php

@@ -317,8 +317,13 @@ class InstallCommand extends ContainerAwareCommand
                 'section' => 'export',
             ],
             [
-                'name' => 'pocket_consumer_key',
+                'name' => 'import_with_redis',
-                'value' => null,
+                'value' => '0',
+                'section' => 'import',
+            ],
+            [
+                'name' => 'import_with_rabbitmq',
+                'value' => '0',
                 'section' => 'import',
             ],
             [

src/Wallabag/CoreBundle/Controller/EntryController.php

@@ -17,26 +17,35 @@ use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache;
 class EntryController extends Controller
 {
     /**
-     * @param Entry $entry
+     * Fetch content and update entry.
+     * In case it fails, entry will return to avod loosing the data.
+     *
+     * @param Entry  $entry
+     * @param string $prefixMessage Should be the translation key: entry_saved or entry_reloaded
+     *
+     * @return Entry
      */
-    private function updateEntry(Entry $entry)
+    private function updateEntry(Entry $entry, $prefixMessage = 'entry_saved')
     {
+        // put default title in case of fetching content failed
+        $entry->setTitle('No title found');
+
+        $message = 'flashes.entry.notice.'.$prefixMessage;
+
         try {
             $entry = $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl());
-
-            $em = $this->getDoctrine()->getManager();
-            $em->persist($entry);
-            $em->flush();
         } catch (\Exception $e) {
             $this->get('logger')->error('Error while saving an entry', [
                 'exception' => $e,
                 'entry' => $entry,
             ]);
 
-            return false;
+            $message = 'flashes.entry.notice.'.$prefixMessage.'_failed';
         }
 
-        return true;
+        $this->get('session')->getFlashBag()->add('notice', $message);
+
+        return $entry;
     }
 
     /**
@@ -66,12 +75,11 @@ class EntryController extends Controller
                 return $this->redirect($this->generateUrl('view', ['id' => $existingEntry->getId()]));
             }
 
-            $message = 'flashes.entry.notice.entry_saved';
+            $this->updateEntry($entry);
-            if (false === $this->updateEntry($entry)) {
-                $message = 'flashes.entry.notice.entry_saved_failed';
-            }
 
-            $this->get('session')->getFlashBag()->add('notice', $message);
+            $em = $this->getDoctrine()->getManager();
+            $em->persist($entry);
+            $em->flush();
 
             return $this->redirect($this->generateUrl('homepage'));
         }
@@ -95,6 +103,10 @@ class EntryController extends Controller
 
         if (false === $this->checkIfEntryAlreadyExists($entry)) {
             $this->updateEntry($entry);
+
+            $em = $this->getDoctrine()->getManager();
+            $em->persist($entry);
+            $em->flush();
         }
 
         return $this->redirect($this->generateUrl('homepage'));
@@ -316,15 +328,11 @@ class EntryController extends Controller
     {
         $this->checkUserAction($entry);
 
-        $message = 'flashes.entry.notice.entry_reloaded';
+        $this->updateEntry($entry, 'entry_reloaded');
-        if (false === $this->updateEntry($entry)) {
-            $message = 'flashes.entry.notice.entry_reload_failed';
-        }
 
-        $this->get('session')->getFlashBag()->add(
+        $em = $this->getDoctrine()->getManager();
-            'notice',
+        $em->persist($entry);
-            $message
+        $em->flush();
-        );
 
         return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()]));
     }

src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php

@@ -20,6 +20,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
         $adminConfig->setItemsPerPage(30);
         $adminConfig->setReadingSpeed(1);
         $adminConfig->setLanguage('en');
+        $adminConfig->setPocketConsumerKey('xxxxx');
 
         $manager->persist($adminConfig);
 
@@ -30,6 +31,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
         $bobConfig->setItemsPerPage(10);
         $bobConfig->setReadingSpeed(1);
         $bobConfig->setLanguage('fr');
+        $bobConfig->setPocketConsumerKey(null);
 
         $manager->persist($bobConfig);
 
@@ -40,6 +42,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
         $emptyConfig->setItemsPerPage(10);
         $emptyConfig->setReadingSpeed(1);
         $emptyConfig->setLanguage('en');
+        $emptyConfig->setPocketConsumerKey(null);
 
         $manager->persist($emptyConfig);
 

src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php

@@ -91,8 +91,13 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
                 'section' => 'export',
             ],
             [
-                'name' => 'pocket_consumer_key',
+                'name' => 'import_with_redis',
-                'value' => null,
+                'value' => '0',
+                'section' => 'import',
+            ],
+            [
+                'name' => 'import_with_rabbitmq',
+                'value' => '0',
                 'section' => 'import',
             ],
             [

src/Wallabag/CoreBundle/Entity/Config.php

@@ -80,6 +80,13 @@ class Config
      */
     private $readingSpeed;
 
+    /**
+     * @var string
+     *
+     * @ORM\Column(name="pocket_consumer_key", type="string", nullable=true)
+     */
+    private $pocketConsumerKey;
+
     /**
      * @ORM\OneToOne(targetEntity="Wallabag\UserBundle\Entity\User", inversedBy="config")
      */
@@ -278,6 +285,30 @@ class Config
         return $this->readingSpeed;
     }
 
+    /**
+     * Set pocketConsumerKey.
+     *
+     * @param string $pocketConsumerKey
+     *
+     * @return Config
+     */
+    public function setPocketConsumerKey($pocketConsumerKey)
+    {
+        $this->pocketConsumerKey = $pocketConsumerKey;
+
+        return $this;
+    }
+
+    /**
+     * Get pocketConsumerKey.
+     *
+     * @return string
+     */
+    public function getPocketConsumerKey()
+    {
+        return $this->pocketConsumerKey;
+    }
+
     /**
      * @param TaggingRule $rule
      *

src/Wallabag/CoreBundle/Entity/Entry.php

@@ -97,7 +97,7 @@ class Entry
     private $content;
 
     /**
-     * @var date
+     * @var \DateTime
      *
      * @ORM\Column(name="created_at", type="datetime")
      *
@@ -106,7 +106,7 @@ class Entry
     private $createdAt;
 
     /**
-     * @var date
+     * @var \DateTime
      *
      * @ORM\Column(name="updated_at", type="datetime")
      *
@@ -410,7 +410,22 @@ class Entry
     }
 
     /**
-     * @return string
+     * Set created_at.
+     * Only used when importing data from an other service.
+     *
+     * @param \DateTime $createdAt
+     *
+     * @return Entry
+     */
+    public function setCreatedAt(\DateTime $createdAt)
+    {
+        $this->createdAt = $createdAt;
+
+        return $this;
+    }
+
+    /**
+     * @return \DateTime
      */
     public function getCreatedAt()
     {
@@ -418,7 +433,7 @@ class Entry
     }
 
     /**
-     * @return string
+     * @return \DateTime
      */
     public function getUpdatedAt()
     {

src/Wallabag/CoreBundle/Form/Type/ConfigType.php

@@ -52,6 +52,9 @@ class ConfigType extends AbstractType
                 'choices' => array_flip($this->languages),
                 'label' => 'config.form_settings.language_label',
             ])
+            ->add('pocket_consumer_key', null, [
+                'label' => 'config.form_settings.pocket_consumer_key_label',
+            ])
             ->add('save', SubmitType::class, [
                 'label' => 'config.form.save',
             ])

src/Wallabag/CoreBundle/Resources/config/services.yml

@@ -125,3 +125,11 @@ services:
         arguments:
             - "@security.token_storage"
             - "@router"
+
+    wallabag_core.redis.client:
+        class: Predis\Client
+        arguments:
+            -
+                host: '%redis_host%'
+                port: '%redis_port%'
+                schema: tcp

src/Wallabag/CoreBundle/Resources/translations/messages.da.yml

@@ -68,6 +68,7 @@ config:
             # 200_word: 'I read ~200 words per minute'
             # 300_word: 'I read ~300 words per minute'
             # 400_word: 'I read ~400 words per minute'
+        pocket_consumer_key_label: Brugers nøgle til Pocket for at importere materialer
     form_rss:
         description: 'RSS-feeds fra wallabag gør det muligt at læse de artikler, der gemmes i wallabag, med din RSS-læser. Det kræver, at du genererer et token først.'
         token_label: 'RSS-Token'
@@ -346,6 +347,8 @@ import:
     #     page_title: 'Import > Readability'
     #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
     #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     # page_title: 'Developer'
@@ -411,10 +414,10 @@ flashes:
         notice:
             # entry_already_saved: 'Entry already saved on %date%'
             # entry_saved: 'Entry saved'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             # entry_updated: 'Entry updated'
             # entry_reloaded: 'Entry reloaded'
-            # entry_reload_failed: 'Failed to reload entry'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Artikel arkiveret'
             entry_unarchived: 'Artikel ikke længere arkiveret'
             entry_starred: 'Artikel markeret som favorit'
@@ -428,6 +431,7 @@ flashes:
             # failed: 'Import failed, please try again.'
             # failed_on_file: 'Error while processing import. Please verify your import file.'
             # summary: 'Import summary: %imported% imported, %skipped% already saved.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             # client_created: 'New client created.'

src/Wallabag/CoreBundle/Resources/translations/messages.de.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'Ich lese ~200 Wörter pro Minute'
             300_word: 'Ich lese ~300 Wörter pro Minute'
             400_word: 'Ich lese ~400 Wörter pro Minute'
+        pocket_consumer_key_label: Consumer-Key für Pocket, um Inhalte zu importieren
     form_rss:
         description: 'Die RSS-Feeds von wallabag erlauben es dir, deine gespeicherten Artikel mit deinem bevorzugten RSS-Reader zu lesen. Vorher musst du jedoch einen Token erstellen.'
         token_label: 'RSS-token'
@@ -346,6 +347,8 @@ import:
         page_title: 'Aus Readability importieren'
     #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
     #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     page_title: 'Entwickler'
@@ -411,10 +414,10 @@ flashes:
         notice:
             entry_already_saved: 'Eintrag bereits am %date% gespeichert'
             entry_saved: 'Eintrag gespeichert'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'Eintrag aktualisiert'
             entry_reloaded: 'Eintrag neugeladen'
-            entry_reload_failed: 'Neuladen des Eintrags fehlgeschlagen'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Artikel archiviert'
             entry_unarchived: 'Artikel dearchiviert'
             entry_starred: 'Artikel favorisiert'
@@ -428,6 +431,7 @@ flashes:
             failed: 'Import fehlgeschlagen, bitte erneut probieren.'
             failed_on_file: 'Fehler während des Imports. Bitte überprüfe deine Import-Datei.'
             summary: 'Import-Zusammenfassung: %imported% importiert, %skipped% bereits gespeichert.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             client_created: 'Neuer Client erstellt.'

src/Wallabag/CoreBundle/Resources/translations/messages.en.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'I read ~200 words per minute'
             300_word: 'I read ~300 words per minute'
             400_word: 'I read ~400 words per minute'
+        pocket_consumer_key_label: Consumer key for Pocket to import contents
     form_rss:
         description: 'RSS feeds provided by wallabag allow you to read your saved articles with your favourite RSS reader. You need to generate a token first.'
         token_label: 'RSS token'
@@ -346,6 +347,8 @@ import:
         page_title: 'Import > Readability'
         description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
         how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     page_title: 'Developer'
@@ -413,10 +416,10 @@ flashes:
         notice:
             entry_already_saved: 'Entry already saved on %date%'
             entry_saved: 'Entry saved'
-            entry_saved_failed: 'Failed to save entry'
+            entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'Entry updated'
             entry_reloaded: 'Entry reloaded'
-            entry_reload_failed: 'Failed to reload entry'
+            entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Entry archived'
             entry_unarchived: 'Entry unarchived'
             entry_starred: 'Entry starred'
@@ -430,6 +433,7 @@ flashes:
             failed: 'Import failed, please try again.'
             failed_on_file: 'Error while processing import. Please verify your import file.'
             summary: 'Import summary: %imported% imported, %skipped% already saved.'
+            summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             client_created: 'New client %name% created.'

src/Wallabag/CoreBundle/Resources/translations/messages.es.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'Leo ~200 palabras por minuto'
             300_word: 'Leo ~300 palabras por minuto'
             400_word: 'Leo ~400 palabras por minuto'
+        # pocket_consumer_key_label: Consumer key for Pocket to import contents
     form_rss:
         description: 'Los feeds RSS de wallabag permiten leer los artículos guardados con su lector RSS favorito. Necesita generar un token primero'
         token_label: 'RSS token'
@@ -346,6 +347,8 @@ import:
         page_title: 'Importar > Readability'
     #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
     #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     page_title: 'Promotor'
@@ -411,10 +414,10 @@ flashes:
         notice:
             entry_already_saved: 'Entrada ya guardada por %fecha%'
             entry_saved: 'Entrada guardada'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'Entrada actualizada'
             entry_reloaded: 'Entrada recargada'
-            entry_reload_failed: 'Entrada recargada reprobada'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Artículo archivado'
             entry_unarchived: 'Artículo desarchivado'
             entry_starred: 'Artículo guardado en los favoritos'
@@ -425,9 +428,10 @@ flashes:
             tag_added: 'Etiqueta añadida'
     import:
         notice:
-             failed: 'Importación reprobada, por favor inténtelo de nuevo.'
+            failed: 'Importación reprobada, por favor inténtelo de nuevo.'
-             failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.'
+            failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.'
-             summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.'
+            summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             client_created: 'Nuevo cliente creado.'

src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'من تقریباً ۲۰۰ واژه را در دقیقه می‌خوانم'
             300_word: 'من تقریباً ۳۰۰ واژه را در دقیقه می‌خوانم'
             400_word: 'من تقریباً ۴۰۰ واژه را در دقیقه می‌خوانم'
+        pocket_consumer_key_label: کلید کاربری Pocket برای درون‌ریزی مطالب
     form_rss:
         description: 'با خوراک آر-اس-اس که wallabag در اختیارتان می‌گذارد، می‌توانید مقاله‌های ذخیره‌شده را در نرم‌افزار آر-اس-اس دلخواه خود بخوانید. برای این کار نخست باید یک کد بسازید.'
         token_label: 'کد آر-اس-اس'
@@ -344,8 +345,10 @@ import:
         description: 'این برنامه همهٔ داده‌های شما را در نسخهٔ ۲ wallabag درون‌ریزی می‌کند. به بخش «همهٔ مقاله‌ها» بروید و در بخش «برون‌ریزی» روی "JSON" کلیک کنید. با این کار شما پرونده‌ای به شکل "All articles.json" دریافت خواهید کرد.'
     readability:
         page_title: 'درون‌ریزی > Readability'
-    #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
+        # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
-    #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+        # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     # page_title: 'Developer'
@@ -411,10 +414,10 @@ flashes:
         notice:
             entry_already_saved: 'این مقاله در تاریخ %date% ذخیره شده بود'
             entry_saved: 'مقاله ذخیره شد'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'مقاله به‌روز شد'
             entry_reloaded: 'مقاله به‌روز شد'
-            entry_reload_failed: 'به‌روزرسانی مقاله شکست خورد'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'مقاله بایگانی شد'
             entry_unarchived: 'مقاله از بایگانی درآمد'
             entry_starred: 'مقاله برگزیده شد'
@@ -428,6 +431,7 @@ flashes:
             failed: 'درون‌ریزی شکست خورد. لطفاً دوباره تلاش کنید.'
             failed_on_file: 'خطا هنگام پردازش پروندهٔ ورودی. آیا پروندهٔ درون‌ریزی شده سالم است؟'
             summary: 'گزارش درون‌ریزی: %imported% وارد شد, %skipped% از قبل ذخیره شده بود.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             # client_created: 'New client created.'

src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'Je lis environ 200 mots par minute'
             300_word: 'Je lis environ 300 mots par minute'
             400_word: 'Je lis environ 400 mots par minute'
+        pocket_consumer_key_label: Clé d'authentification Pocket pour importer les données
     form_rss:
         description: "Les flux RSS fournis par wallabag vous permettent de lire vos articles sauvegardés dans votre lecteur de flux préféré. Pour pouvoir les utiliser, vous devez d'abord créer un jeton."
         token_label: 'Jeton RSS'
@@ -346,6 +347,8 @@ import:
         page_title: 'Importer > Readability'
         description: 'Cet outil va importer toutes vos données de Readability. Sur la page des outils (https://www.readability.com/tools/), cliquez sur "Export your data" dans la section "Data Export". Vous allez recevoir un email avec un lien pour télécharger le json.'
         how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer."
+    worker:
+        enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :"
 
 developer:
     page_title: 'Développeur'
@@ -413,10 +416,10 @@ flashes:
         notice:
             entry_already_saved: 'Article déjà sauvergardé le %date%'
             entry_saved: 'Article enregistré'
-            entry_saved_failed: "L'enregistrement a échoué"
+            entry_saved_failed: 'Article enregistré mais impossible de récupérer le contenu'
             entry_updated: 'Article mis à jour'
             entry_reloaded: 'Article rechargé'
-            entry_reload_failed: "Le rechargement de l'article a échoué"
+            entry_reload_failed: "Article mis à jour mais impossible de récupérer le contenu"
             entry_archived: 'Article marqué comme lu'
             entry_unarchived: 'Article marqué comme non lu'
             entry_starred: 'Article ajouté dans les favoris'
@@ -430,6 +433,7 @@ flashes:
             failed: "L'import a échoué, veuillez ré-essayer"
             failed_on_file: "Erreur lors du traitement de l'import. Vérifier votre fichier."
             summary: "Rapport d'import: %imported% importés, %skipped% déjà présent."
+            summary_with_queue: "Rapport d'import: %queued% en cours de traitement."
     developer:
         notice:
             client_created: 'Nouveau client %name% créé'

src/Wallabag/CoreBundle/Resources/translations/messages.it.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'Leggo ~200 parole al minuto'
             300_word: 'Leggo ~300 parole al minuto'
             400_word: 'Leggo ~400 parole al minuto'
+        pocket_consumer_key_label: Consumer key per Pocket per importare i contenuti
     form_rss:
         description: 'I feed RSS generati da wallabag ti permettono di leggere i tuoi contenuti salvati con il tuo lettore di RSS preferito. Prima, devi generare un token.'
         token_label: 'RSS token'
@@ -343,8 +344,10 @@ import:
         description: 'Questo importatore copierà tutti i tuoi dati da un wallabag v2. Vai in "Tutti i contenuti", e, nella sidebar di esportazione, clicca su "JSON". Otterrai un file "Tutti i contenuti.json".'
     readability:
         page_title: 'Importa da > Readability'
-    #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
+        # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
-    #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+        # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     page_title: 'Sviluppatori'
@@ -410,10 +413,10 @@ flashes:
         notice:
             entry_already_saved: 'Contenuto già salvato in data %date%'
             entry_saved: 'Contenuto salvato'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'Contenuto aggiornato'
             entry_reloaded: 'Contenuto ricaricato'
-            entry_reload_failed: 'Errore nel ricaricamento del contenuto'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Contenuto archiviato'
             entry_unarchived: 'Contenuto dis-archiviato'
             entry_starred: 'Contenuto segnato come preferito'
@@ -427,6 +430,7 @@ flashes:
             failed: 'Importazione fallita, riprova.'
             failed_on_file: 'Errore durante la processazione dei dati da importare. Verifica il tuo file di import.'
             summary: 'Sommario di importazione: %imported% importati, %skipped% già salvati.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             client_created: 'Nuovo client creato.'

src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml

@@ -68,6 +68,7 @@ config:
             200_word: "Legissi a l'entorn de 200 mots per minuta"
             300_word: "Legissi a l'entorn de 300 mots per minuta"
             400_word: "Legissi a l'entorn de 400 mots per minuta"
+        pocket_consumer_key_label: Clau d'autentificacion Pocket per importar las donadas
     form_rss:
         description: "Los fluxes RSS fornits per wallabag vos permeton de legir vòstres articles salvagardats dins vòstre lector de fluxes preferit. Per los poder emplegar, vos cal, d'en primièr crear un geton."
         token_label: 'Geton RSS'
@@ -346,6 +347,8 @@ import:
         page_title: 'Importer > Readability'
         description: "Aquesta aisina importarà totas vòstres articles de Readability. Sus la pagina de l'aisina (https://www.readability.com/tools/), clicatz sus \"Export your data\" dins la seccion \"Data Export\". Recebretz un corrièl per telecargar un json (qu'acaba pas amb un .json de fach)."
         how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar."
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     page_title: 'Desvolopador'
@@ -411,10 +414,10 @@ flashes:
         notice:
             entry_already_saved: 'Article ja salvargardat lo %date%'
             entry_saved: 'Article enregistrat'
-            entry_saved_failed: "Fracàs de l'enregistrament de l'entrada"
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'Article mes a jorn'
             entry_reloaded: 'Article recargat'
-            entry_reload_failed: "Fracàs de l'actualizacion de l'article"
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Article marcat coma legit'
             entry_unarchived: 'Article marcat coma pas legit'
             entry_starred: 'Article apondut dins los favorits'
@@ -428,6 +431,7 @@ flashes:
             failed: "L'importacion a fracassat, mercés de tornar ensajar"
             failed_on_file: "Errorr pendent du tractament de l'import. Mercés de verificar vòstre fichièr."
             summary: "Rapòrt d'import: %imported% importats, %skipped% ja presents."
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             client_created: 'Novèl client creat'

src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml

@@ -68,6 +68,7 @@ config:
             200_word: 'Czytam ~200 słów na minutę'
             300_word: 'Czytam ~300 słów na minutę'
             400_word: 'Czytam ~400 słów na minutę'
+        pocket_consumer_key_label: Klucz klienta Pocket do importu zawartości
     form_rss:
         description: 'Kanały RSS prowadzone przez wallabag pozwalają Ci na czytanie twoich zapisanych artykułów w twoium ulubionym czytniku RSS. Musisz najpierw wynegenerować tokena.‌'
         token_label: 'Token RSS'
@@ -346,6 +347,8 @@ import:
         page_title: 'Import > Readability'
         description: 'Ten importer, zaimportuje wszystkie twoje artykuły z Readability. Na stronie narzędzi (https://www.readability.com/tools/), kliknij na "Export your data" w sekcji "Data Export". Otrzymach email z plikiem JSON (plik nie będzie zawierał rozszerzenia .json).'
         how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     page_title: 'Deweloper'
@@ -411,10 +414,10 @@ flashes:
         notice:
             entry_already_saved: 'Wpis już został dodany %date%'
             entry_saved: 'Wpis zapisany'
-            entry_saved_failed: 'Zapis artykułu się nie powiódł'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             entry_updated: 'Wpis zaktualizowany'
             entry_reloaded: 'Wpis ponownie załadowany'
-            entry_reload_failed: 'Błąd ponownego załadowania'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Wpis dodany do archiwum'
             entry_unarchived: 'Wpis usunięty z archiwum'
             entry_starred: 'Wpis oznaczony gwiazdką'
@@ -428,6 +431,7 @@ flashes:
             failed: 'Nieudany import, prosimy spróbować ponownie.'
             failed_on_file: 'Błąd podczas ptrzetwarzania pliku. Sprawdż swój importowany plik.'
             summary: 'Podsumowanie importu: %imported% zaimportowane, %skipped% już zapisane.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             client_created: 'Nowy klient utworzony.'

src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml

@@ -68,6 +68,7 @@ config:
             # 200_word: 'I read ~200 words per minute'
             # 300_word: 'I read ~300 words per minute'
             # 400_word: 'I read ~400 words per minute'
+        pocket_consumer_key_label: Cheie consumator pentru importarea contentului din Pocket
     form_rss:
         description: 'Feed-urile RSS oferite de wallabag îți permit să-ți citești articolele salvate în reader-ul tău preferat RSS.'
         token_label: 'RSS-Token'
@@ -346,6 +347,8 @@ import:
     #     page_title: 'Import > Readability'
     #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
     #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     # page_title: 'Developer'
@@ -411,10 +414,10 @@ flashes:
         notice:
             # entry_already_saved: 'Entry already saved on %date%'
             # entry_saved: 'Entry saved'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             # entry_updated: 'Entry updated'
             # entry_reloaded: 'Entry reloaded'
-            # entry_reload_failed: 'Failed to reload entry'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Articol arhivat'
             entry_unarchived: 'Articol dezarhivat'
             entry_starred: 'Articol adăugat la favorite'
@@ -428,6 +431,7 @@ flashes:
             # failed: 'Import failed, please try again.'
             # failed_on_file: 'Error while processing import. Please verify your import file.'
             # summary: 'Import summary: %imported% imported, %skipped% already saved.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             # client_created: 'New client created.'

src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml

@@ -68,6 +68,7 @@ config:
             # 200_word: 'I read ~200 words per minute'
             # 300_word: 'I read ~300 words per minute'
             # 400_word: 'I read ~400 words per minute'
+        # pocket_consumer_key_label: Consumer key for Pocket to import contents
     form_rss:
         description: 'wallabag RSS akışı kaydetmiş olduğunuz makalelerini favori RSS okuyucunuzda görüntülemenizi sağlar. Bunu yapabilmek için öncelikle belirteç (token) oluşturmalısınız.'
         token_label: 'RSS belirteci (token)'
@@ -344,8 +345,10 @@ import:
         # description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.'
     readability:
         page_title: 'İçe Aktar > Readability'
-    #     description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
+        # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
-    #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+        # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
+    worker:
+        # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
 
 developer:
     # page_title: 'Developer'
@@ -411,10 +414,10 @@ flashes:
         notice:
             entry_already_saved: 'Entry already saved on %date%'
             entry_saved: 'Makale kaydedildi'
-            # entry_saved_failed: 'Failed to save entry'
+            # entry_saved_failed: 'Entry saved but fetching content failed'
             # entry_updated: 'Entry updated'
             entry_reloaded: 'Makale içeriği yenilendi'
-            # entry_reload_failed: 'Failed to reload entry'
+            # entry_reload_failed: 'Entry reloaded but fetching content failed'
             entry_archived: 'Makale arşivlendi'
             entry_unarchived: 'Makale arşivden çıkartıldı'
             entry_starred: 'Makale favorilere eklendi'
@@ -428,6 +431,7 @@ flashes:
             # failed: 'Import failed, please try again.'
             # failed_on_file: 'Error while processing import. Please verify your import file.'
             # summary: 'Import summary: %imported% imported, %skipped% already saved.'
+            # summary_with_queue: 'Import summary: %queued% queued.'
     developer:
         notice:
             # client_created: 'New client created.'

src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig

@@ -44,6 +44,18 @@
             </div>
         </fieldset>
 
+        <fieldset class="w500p inline">
+            <div class="row">
+                {{ form_label(form.config.pocket_consumer_key) }}
+                {{ form_errors(form.config.pocket_consumer_key) }}
+                {{ form_widget(form.config.pocket_consumer_key) }}
+                <p>
+                    &raquo;
+                    <a href="https://getpocket.com/developer/docs/authentication">https://getpocket.com/developer/docs/authentication</a>
+                </p>
+            </div>
+        </fieldset>
+
         {{ form_rest(form.config) }}
     </form>
 

src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig

@@ -62,6 +62,18 @@
                                 </div>
                             </div>
 
+                            <div class="row">
+                                <div class="input-field col s12">
+                                    {{ form_label(form.config.pocket_consumer_key) }}
+                                    {{ form_errors(form.config.pocket_consumer_key) }}
+                                    {{ form_widget(form.config.pocket_consumer_key) }}
+                                    <p>
+                                        &raquo;
+                                        <a href="https://getpocket.com/developer/docs/authentication">https://getpocket.com/developer/docs/authentication</a>
+                                    </p>
+                                </div>
+                            </div>
+
                             {{ form_widget(form.config.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }}
                             {{ form_rest(form.config) }}
                         </form>

src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig

@@ -19,6 +19,8 @@
             Materialize.toast('{{ flashMessage|trans }}', 4000);
         </script>
     {% endfor %}
+
+    {{ render(controller("WallabagImportBundle:Import:checkQueue")) }}
 {% endblock %}
 
 {% block menu %}

src/Wallabag/ImportBundle/Command/ImportCommand.php

@@ -26,6 +26,10 @@ class ImportCommand extends ContainerAwareCommand
     {
         $output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---');
 
+        if (!file_exists($input->getArgument('filepath'))) {
+            throw new Exception(sprintf('File "%s" not found', $input->getArgument('filepath')));
+        }
+
         $em = $this->getContainer()->get('doctrine')->getManager();
         // Turning off doctrine default logs queries for saving memory
         $em->getConnection()->getConfiguration()->setSQLLogger(null);
@@ -43,9 +47,9 @@ class ImportCommand extends ContainerAwareCommand
         }
 
         $wallabag->setMarkAsRead($input->getOption('markAsRead'));
+        $wallabag->setUser($user);
 
         $res = $wallabag
-            ->setUser($user)
             ->setFilepath($input->getArgument('filepath'))
             ->import();
 

src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Wallabag\ImportBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Config\Definition\Exception\Exception;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Simpleue\Worker\QueueWorker;
+
+class RedisWorkerCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('wallabag:import:redis-worker')
+            ->setDescription('Launch Redis worker')
+            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
+            ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false)
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
+        $output->writeln('Waiting for message ...');
+
+        $serviceName = $input->getArgument('serviceName');
+
+        if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
+            throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
+        }
+
+        $worker = new QueueWorker(
+            $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
+            $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName),
+            $input->getOption('maxIterations')
+        );
+
+        $worker->start();
+    }
+}

src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Wallabag\ImportBundle\Consumer;
+
+use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
+use PhpAmqpLib\Message\AMQPMessage;
+
+class AMQPEntryConsumer extends AbstractConsumer implements ConsumerInterface
+{
+    /**
+     * {@inheritdoc}
+     */
+    public function execute(AMQPMessage $msg)
+    {
+        return $this->handleMessage($msg->body);
+    }
+}

src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace Wallabag\ImportBundle\Consumer;
+
+use Doctrine\ORM\EntityManager;
+use Wallabag\ImportBundle\Import\AbstractImport;
+use Wallabag\UserBundle\Repository\UserRepository;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Entity\Tag;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+
+abstract class AbstractConsumer
+{
+    protected $em;
+    protected $userRepository;
+    protected $import;
+    protected $logger;
+
+    public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
+    {
+        $this->em = $em;
+        $this->userRepository = $userRepository;
+        $this->import = $import;
+        $this->logger = $logger ?: new NullLogger();
+    }
+
+    /**
+     * Handle a message and save it.
+     *
+     * @param string $body Message from the queue (in json)
+     *
+     * @return bool
+     */
+    protected function handleMessage($body)
+    {
+        $storedEntry = json_decode($body, true);
+
+        $user = $this->userRepository->find($storedEntry['userId']);
+
+        // no user? Drop message
+        if (null === $user) {
+            $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
+
+            return false;
+        }
+
+        $this->import->setUser($user);
+
+        $entry = $this->import->parseEntry($storedEntry);
+
+        if (null === $entry) {
+            $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
+
+            return false;
+        }
+
+        try {
+            $this->em->flush();
+
+            // clear only affected entities
+            $this->em->clear(Entry::class);
+            $this->em->clear(Tag::class);
+        } catch (\Exception $e) {
+            $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
+
+            return false;
+        }
+
+        $this->logger->info('Content with url imported! ('.$entry->getUrl().')');
+
+        return true;
+    }
+}

src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Wallabag\ImportBundle\Consumer;
+
+use Simpleue\Job\Job;
+
+class RedisEntryConsumer extends AbstractConsumer implements Job
+{
+    /**
+     * Handle one message by one message.
+     *
+     * @param string $job Content of the message (directly from Redis)
+     *
+     * @return bool
+     */
+    public function manage($job)
+    {
+        return $this->handleMessage($job);
+    }
+
+    /**
+     * Should tell if the given job will kill the worker.
+     * We don't want to stop it :).
+     */
+    public function isStopJob($job)
+    {
+        return false;
+    }
+}

src/Wallabag/ImportBundle/Controller/ImportController.php

@@ -16,4 +16,66 @@ class ImportController extends Controller
             'imports' => $this->get('wallabag_import.chain')->getAll(),
         ]);
     }
+
+    /**
+     * Display how many messages are queue (both in Redis and RabbitMQ).
+     * Only for admins.
+     */
+    public function checkQueueAction()
+    {
+        $nbRedisMessages = null;
+        $nbRabbitMessages = null;
+
+        if (!$this->get('security.authorization_checker')->isGranted('ROLE_SUPER_ADMIN')) {
+            return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
+                'nbRedisMessages' => $nbRedisMessages,
+                'nbRabbitMessages' => $nbRabbitMessages,
+            ]);
+        }
+
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket')
+                + $this->getTotalMessageInRabbitQueue('readability')
+                + $this->getTotalMessageInRabbitQueue('wallabag_v1')
+                + $this->getTotalMessageInRabbitQueue('wallabag_v2')
+            ;
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $redis = $this->get('wallabag_core.redis.client');
+
+            $nbRedisMessages = $redis->llen('wallabag.import.pocket')
+                + $redis->llen('wallabag.import.readability')
+                + $redis->llen('wallabag.import.wallabag_v1')
+                + $redis->llen('wallabag.import.wallabag_v2')
+            ;
+        }
+
+        return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
+            'nbRedisMessages' => $nbRedisMessages,
+            'nbRabbitMessages' => $nbRabbitMessages,
+        ]);
+    }
+
+    /**
+     * Count message in RabbitMQ queue.
+     * It get one message without acking it (so it'll stay in the queue)
+     * which will include the total of *other* messages in the queue.
+     * Adding one to that messages will result in the full total message.
+     *
+     * @param string $importService The import service related: pocket, readability, wallabag_v1 or wallabag_v2
+     *
+     * @return int
+     */
+    private function getTotalMessageInRabbitQueue($importService)
+    {
+        $message = $this
+            ->get('old_sound_rabbit_mq.import_'.$importService.'_consumer')
+            ->getChannel()
+            ->basic_get('wallabag.import.'.$importService);
+
+        if (null === $message) {
+            return 0;
+        }
+
+        return $message->delivery_info['message_count'] + 1;
+    }
 }

src/Wallabag/ImportBundle/Controller/PocketController.php

@@ -10,12 +10,31 @@ use Symfony\Component\Form\Extension\Core\Type\CheckboxType;
 
 class PocketController extends Controller
 {
+    /**
+     * Return Pocket Import Service with or without RabbitMQ enabled.
+     *
+     * @return \Wallabag\ImportBundle\Import\PocketImport
+     */
+    private function getPocketImportService()
+    {
+        $pocket = $this->get('wallabag_import.pocket.import');
+        $pocket->setUser($this->getUser());
+
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
+        }
+
+        return $pocket;
+    }
+
     /**
      * @Route("/pocket", name="import_pocket")
      */
     public function indexAction()
     {
-        $pocket = $this->get('wallabag_import.pocket.import');
+        $pocket = $this->getPocketImportService();
         $form = $this->createFormBuilder($pocket)
             ->add('mark_as_read', CheckboxType::class, [
                 'label' => 'import.form.mark_as_read_label',
@@ -24,8 +43,8 @@ class PocketController extends Controller
             ->getForm();
 
         return $this->render('WallabagImportBundle:Pocket:index.html.twig', [
-            'import' => $this->get('wallabag_import.pocket.import'),
+            'import' => $this->getPocketImportService(),
-            'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true,
+            'has_consumer_key' => '' === trim($this->getUser()->getConfig()->getPocketConsumerKey()) ? false : true,
             'form' => $form->createView(),
         ]);
     }
@@ -35,7 +54,7 @@ class PocketController extends Controller
      */
     public function authAction(Request $request)
     {
-        $requestToken = $this->get('wallabag_import.pocket.import')
+        $requestToken = $this->getPocketImportService()
             ->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL));
 
         if (false === $requestToken) {
@@ -62,7 +81,7 @@ class PocketController extends Controller
     public function callbackAction()
     {
         $message = 'flashes.import.notice.failed';
-        $pocket = $this->get('wallabag_import.pocket.import');
+        $pocket = $this->getPocketImportService();
 
         $markAsRead = $this->get('session')->get('mark_as_read');
         $this->get('session')->remove('mark_as_read');
@@ -83,6 +102,12 @@ class PocketController extends Controller
                 '%imported%' => $summary['imported'],
                 '%skipped%' => $summary['skipped'],
             ]);
+
+            if (0 < $summary['queued']) {
+                $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
+                    '%queued%' => $summary['queued'],
+                ]);
+            }
         }
 
         $this->get('session')->getFlashBag()->add(

src/Wallabag/ImportBundle/Controller/ReadabilityController.php

@@ -18,15 +18,21 @@ class ReadabilityController extends Controller
         $form->handleRequest($request);
 
         $readability = $this->get('wallabag_import.readability.import');
+        $readability->setUser($this->getUser());
+
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
+        }
 
         if ($form->isValid()) {
             $file = $form->get('file')->getData();
             $markAsRead = $form->get('mark_as_read')->getData();
             $name = 'readability_'.$this->getUser()->getId().'.json';
 
-            if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+            if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
                 $res = $readability
-                    ->setUser($this->getUser())
                     ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
                     ->setMarkAsRead($markAsRead)
                     ->import();
@@ -40,6 +46,12 @@ class ReadabilityController extends Controller
                         '%skipped%' => $summary['skipped'],
                     ]);
 
+                    if (0 < $summary['queued']) {
+                        $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
+                            '%queued%' => $summary['queued'],
+                        ]);
+                    }
+
                     unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
                 }
 

src/Wallabag/ImportBundle/Controller/WallabagController.php

@@ -38,15 +38,15 @@ abstract class WallabagController extends Controller
         $form->handleRequest($request);
 
         $wallabag = $this->getImportService();
+        $wallabag->setUser($this->getUser());
 
         if ($form->isValid()) {
             $file = $form->get('file')->getData();
             $markAsRead = $form->get('mark_as_read')->getData();
             $name = $this->getUser()->getId().'.json';
 
-            if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+            if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
                 $res = $wallabag
-                    ->setUser($this->getUser())
                     ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
                     ->setMarkAsRead($markAsRead)
                     ->import();
@@ -60,6 +60,12 @@ abstract class WallabagController extends Controller
                         '%skipped%' => $summary['skipped'],
                     ]);
 
+                    if (0 < $summary['queued']) {
+                        $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
+                            '%queued%' => $summary['queued'],
+                        ]);
+                    }
+
                     unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
                 }
 

src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php

@@ -12,7 +12,15 @@ class WallabagV1Controller extends WallabagController
      */
     protected function getImportService()
     {
-        return $this->get('wallabag_import.wallabag_v1.import');
+        $service = $this->get('wallabag_import.wallabag_v1.import');
+
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
+        }
+
+        return $service;
     }
 
     /**

src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php

@@ -12,7 +12,15 @@ class WallabagV2Controller extends WallabagController
      */
     protected function getImportService()
     {
-        return $this->get('wallabag_import.wallabag_v2.import');
+        $service = $this->get('wallabag_import.wallabag_v2.import');
+
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
+        }
+
+        return $service;
     }
 
     /**

src/Wallabag/ImportBundle/Form/Type/UploadImportType.php

@@ -15,6 +15,7 @@ class UploadImportType extends AbstractType
         $builder
             ->add('file', FileType::class, [
                 'label' => 'import.form.file_label',
+                'required' => true,
             ])
             ->add('mark_as_read', CheckboxType::class, [
                 'label' => 'import.form.mark_as_read_label',

src/Wallabag/ImportBundle/Import/AbstractImport.php

@@ -7,12 +7,21 @@ use Psr\Log\NullLogger;
 use Doctrine\ORM\EntityManager;
 use Wallabag\CoreBundle\Helper\ContentProxy;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Entity\Tag;
+use Wallabag\UserBundle\Entity\User;
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
 
 abstract class AbstractImport implements ImportInterface
 {
     protected $em;
     protected $logger;
     protected $contentProxy;
+    protected $producer;
+    protected $user;
+    protected $markAsRead;
+    protected $skippedEntries = 0;
+    protected $importedEntries = 0;
+    protected $queuedEntries = 0;
 
     public function __construct(EntityManager $em, ContentProxy $contentProxy)
     {
@@ -26,22 +35,151 @@ abstract class AbstractImport implements ImportInterface
         $this->logger = $logger;
     }
 
+    /**
+     * Set RabbitMQ/Redis Producer to send each entry to a queue.
+     * This method should be called when user has enabled RabbitMQ.
+     *
+     * @param ProducerInterface $producer
+     */
+    public function setProducer(ProducerInterface $producer)
+    {
+        $this->producer = $producer;
+    }
+
+    /**
+     * Set current user.
+     * Could the current *connected* user or one retrieve by the consumer.
+     *
+     * @param User $user
+     */
+    public function setUser(User $user)
+    {
+        $this->user = $user;
+    }
+
+    /**
+     * Set whether articles must be all marked as read.
+     *
+     * @param bool $markAsRead
+     */
+    public function setMarkAsRead($markAsRead)
+    {
+        $this->markAsRead = $markAsRead;
+
+        return $this;
+    }
+
+    /**
+     * Get whether articles must be all marked as read.
+     */
+    public function getMarkAsRead()
+    {
+        return $this->markAsRead;
+    }
+
     /**
      * Fetch content from the ContentProxy (using graby).
-     * If it fails return false instead of the updated entry.
+     * If it fails return the given entry to be saved in all case (to avoid user to loose the content).
      *
      * @param Entry  $entry   Entry to update
      * @param string $url     Url to grab content for
      * @param array  $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
      *
-     * @return Entry|false
+     * @return Entry
      */
     protected function fetchContent(Entry $entry, $url, array $content = [])
     {
         try {
             return $this->contentProxy->updateEntry($entry, $url, $content);
         } catch (\Exception $e) {
-            return false;
+            return $entry;
         }
     }
+
+    /**
+     * Parse and insert all given entries.
+     *
+     * @param $entries
+     */
+    protected function parseEntries($entries)
+    {
+        $i = 1;
+
+        foreach ($entries as $importedEntry) {
+            $entry = $this->parseEntry($importedEntry);
+
+            if (null === $entry) {
+                continue;
+            }
+
+            // flush every 20 entries
+            if (($i % 20) === 0) {
+                $this->em->flush();
+
+                // clear only affected entities
+                $this->em->clear(Entry::class);
+                $this->em->clear(Tag::class);
+            }
+            ++$i;
+        }
+
+        $this->em->flush();
+    }
+
+    /**
+     * Parse entries and send them to the queue.
+     * It should just be a simple loop on all item, no call to the database should be done
+     * to speedup queuing.
+     *
+     * Faster parse entries for Producer.
+     * We don't care to make check at this time. They'll be done by the consumer.
+     *
+     * @param array $entries
+     */
+    protected function parseEntriesForProducer(array $entries)
+    {
+        foreach ($entries as $importedEntry) {
+            // set userId for the producer (it won't know which user is connected)
+            $importedEntry['userId'] = $this->user->getId();
+
+            if ($this->markAsRead) {
+                $importedEntry = $this->setEntryAsRead($importedEntry);
+            }
+
+            ++$this->queuedEntries;
+
+            $this->producer->publish(json_encode($importedEntry));
+        }
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSummary()
+    {
+        return [
+            'skipped' => $this->skippedEntries,
+            'imported' => $this->importedEntries,
+            'queued' => $this->queuedEntries,
+        ];
+    }
+
+    /**
+     * Parse one entry.
+     *
+     * @param array $importedEntry
+     *
+     * @return Entry
+     */
+    abstract public function parseEntry(array $importedEntry);
+
+    /**
+     * Set current imported entry to archived / read.
+     * Implementation is different accross all imports.
+     *
+     * @param array $importedEntry
+     *
+     * @return array
+     */
+    abstract protected function setEntryAsRead(array $importedEntry);
 }

src/Wallabag/ImportBundle/Import/PocketImport.php

@@ -6,30 +6,33 @@ use Psr\Log\NullLogger;
 use Doctrine\ORM\EntityManager;
 use GuzzleHttp\Client;
 use GuzzleHttp\Exception\RequestException;
-use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Helper\ContentProxy;
-use Craue\ConfigBundle\Util\Config;
 
 class PocketImport extends AbstractImport
 {
-    private $user;
     private $client;
-    private $consumerKey;
+    private $accessToken;
-    private $skippedEntries = 0;
-    private $importedEntries = 0;
-    private $markAsRead;
-    protected $accessToken;
 
-    public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
+    const NB_ELEMENTS = 5000;
+
+    public function __construct(EntityManager $em, ContentProxy $contentProxy)
     {
-        $this->user = $tokenStorage->getToken()->getUser();
         $this->em = $em;
         $this->contentProxy = $contentProxy;
-        $this->consumerKey = $craueConfig->get('pocket_consumer_key');
         $this->logger = new NullLogger();
     }
 
+    /**
+     * Only used for test purpose.
+     *
+     * @return string
+     */
+    public function getAccessToken()
+    {
+        return $this->accessToken;
+    }
+
     /**
      * {@inheritdoc}
      */
@@ -66,7 +69,7 @@ class PocketImport extends AbstractImport
         $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
             [
                 'body' => json_encode([
-                    'consumer_key' => $this->consumerKey,
+                    'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
                     'redirect_uri' => $redirectUri,
                 ]),
             ]
@@ -96,7 +99,7 @@ class PocketImport extends AbstractImport
         $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
             [
                 'body' => json_encode([
-                    'consumer_key' => $this->consumerKey,
+                    'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
                     'code' => $code,
                 ]),
             ]
@@ -115,39 +118,23 @@ class PocketImport extends AbstractImport
         return true;
     }
 
-    /**
-     * Set whether articles must be all marked as read.
-     *
-     * @param bool $markAsRead
-     */
-    public function setMarkAsRead($markAsRead)
-    {
-        $this->markAsRead = $markAsRead;
-
-        return $this;
-    }
-
-    /**
-     * Get whether articles must be all marked as read.
-     */
-    public function getMarkAsRead()
-    {
-        return $this->markAsRead;
-    }
-
     /**
      * {@inheritdoc}
      */
-    public function import()
+    public function import($offset = 0)
     {
+        static $run = 0;
+
         $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
             [
                 'body' => json_encode([
-                    'consumer_key' => $this->consumerKey,
+                    'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
                     'access_token' => $this->accessToken,
                     'detailType' => 'complete',
                     'state' => 'all',
-                    'sort' => 'oldest',
+                    'sort' => 'newest',
+                    'count' => self::NB_ELEMENTS,
+                    'offset' => $offset,
                 ]),
             ]
         );
@@ -162,22 +149,26 @@ class PocketImport extends AbstractImport
 
         $entries = $response->json();
 
-        $this->parseEntries($entries['list']);
+        if ($this->producer) {
+            $this->parseEntriesForProducer($entries['list']);
+        } else {
+            $this->parseEntries($entries['list']);
+        }
+
+        // if we retrieve exactly the amount of items requested it means we can get more
+        // re-call import and offset item by the amount previous received:
+        //  - first call get 5k offset 0
+        //  - second call get 5k offset 5k
+        //  - and so on
+        if (count($entries['list']) === self::NB_ELEMENTS) {
+            ++$run;
+
+            return $this->import(self::NB_ELEMENTS * $run);
+        }
 
         return true;
     }
 
-    /**
-     * {@inheritdoc}
-     */
-    public function getSummary()
-    {
-        return [
-            'skipped' => $this->skippedEntries,
-            'imported' => $this->importedEntries,
-        ];
-    }
-
     /**
      * Set the Guzzle client.
      *
@@ -189,77 +180,74 @@ class PocketImport extends AbstractImport
     }
 
     /**
-     * @see https://getpocket.com/developer/docs/v3/retrieve
+     * {@inheritdoc}
      *
-     * @param $entries
+     * @see https://getpocket.com/developer/docs/v3/retrieve
      */
-    private function parseEntries($entries)
+    public function parseEntry(array $importedEntry)
     {
-        $i = 1;
+        $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
 
-        foreach ($entries as $pocketEntry) {
+        $existingEntry = $this->em
-            $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($url, $this->user->getId());
 
-            $existingEntry = $this->em
+        if (false !== $existingEntry) {
-                ->getRepository('WallabagCoreBundle:Entry')
+            ++$this->skippedEntries;
-                ->findByUrlAndUserId($url, $this->user->getId());
 
-            if (false !== $existingEntry) {
+            return;
-                ++$this->skippedEntries;
-                continue;
-            }
-
-            $entry = new Entry($this->user);
-            $entry = $this->fetchContent($entry, $url);
-
-            // jump to next entry in case of problem while getting content
-            if (false === $entry) {
-                ++$this->skippedEntries;
-                continue;
-            }
-
-            // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
-            if ($pocketEntry['status'] == 1 || $this->markAsRead) {
-                $entry->setArchived(true);
-            }
-
-            // 0 or 1 - 1 If the item is starred
-            if ($pocketEntry['favorite'] == 1) {
-                $entry->setStarred(true);
-            }
-
-            $title = 'Untitled';
-            if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
-                $title = $pocketEntry['resolved_title'];
-            } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
-                $title = $pocketEntry['given_title'];
-            }
-
-            $entry->setTitle($title);
-
-            // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
-            if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
-                $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
-            }
-
-            if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
-                $this->contentProxy->assignTagsToEntry(
-                    $entry,
-                    array_keys($pocketEntry['tags'])
-                );
-            }
-
-            $this->em->persist($entry);
-            ++$this->importedEntries;
-
-            // flush every 20 entries
-            if (($i % 20) === 0) {
-                $this->em->flush();
-            }
-            ++$i;
         }
 
-        $this->em->flush();
+        $entry = new Entry($this->user);
-        $this->em->clear();
+        $entry->setUrl($url);
+
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $entry = $this->fetchContent($entry, $url);
+
+        // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
+        $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
+
+        // 0 or 1 - 1 If the item is starred
+        $entry->setStarred($importedEntry['favorite'] == 1);
+
+        $title = 'Untitled';
+        if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
+            $title = $importedEntry['resolved_title'];
+        } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
+            $title = $importedEntry['given_title'];
+        }
+
+        $entry->setTitle($title);
+
+        // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
+        if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
+            $entry->setPreviewPicture($importedEntry['images'][1]['src']);
+        }
+
+        if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
+            $this->contentProxy->assignTagsToEntry(
+                $entry,
+                array_keys($importedEntry['tags'])
+            );
+        }
+
+        if (!empty($importedEntry['time_added'])) {
+            $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
+        }
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
+
+        return $entry;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function setEntryAsRead(array $importedEntry)
+    {
+        $importedEntry['status'] = '1';
+
+        return $importedEntry;
     }
 }

src/Wallabag/ImportBundle/Import/ReadabilityImport.php

@@ -3,28 +3,10 @@
 namespace Wallabag\ImportBundle\Import;
 
 use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\UserBundle\Entity\User;
 
 class ReadabilityImport extends AbstractImport
 {
-    private $user;
-    private $skippedEntries = 0;
-    private $importedEntries = 0;
     private $filepath;
-    private $markAsRead;
-
-    /**
-     * We define the user in a custom call because on the import command there is no logged in user.
-     * So we can't retrieve user from the `security.token_storage` service.
-     *
-     * @param User $user
-     */
-    public function setUser(User $user)
-    {
-        $this->user = $user;
-
-        return $this;
-    }
 
     /**
      * {@inheritdoc}
@@ -62,37 +44,6 @@ class ReadabilityImport extends AbstractImport
         return $this;
     }
 
-    /**
-     * Set whether articles must be all marked as read.
-     *
-     * @param bool $markAsRead
-     */
-    public function setMarkAsRead($markAsRead)
-    {
-        $this->markAsRead = $markAsRead;
-
-        return $this;
-    }
-
-    /**
-     * Get whether articles must be all marked as read.
-     */
-    public function getMarkAsRead()
-    {
-        return $this->markAsRead;
-    }
-
-    /**
-     * {@inheritdoc}
-     */
-    public function getSummary()
-    {
-        return [
-            'skipped' => $this->skippedEntries,
-            'imported' => $this->importedEntries,
-        ];
-    }
-
     /**
      * {@inheritdoc}
      */
@@ -116,64 +67,66 @@ class ReadabilityImport extends AbstractImport
             return false;
         }
 
+        if ($this->producer) {
+            $this->parseEntriesForProducer($data['bookmarks']);
+
+            return true;
+        }
+
         $this->parseEntries($data['bookmarks']);
 
         return true;
     }
 
     /**
-     * Parse and insert all given entries.
+     * {@inheritdoc}
-     *
-     * @param $entries
      */
-    protected function parseEntries($entries)
+    public function parseEntry(array $importedEntry)
     {
-        $i = 1;
+        $existingEntry = $this->em
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
 
-        foreach ($entries as $importedEntry) {
+        if (false !== $existingEntry) {
-            $existingEntry = $this->em
+            ++$this->skippedEntries;
-                ->getRepository('WallabagCoreBundle:Entry')
-                ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
 
-            if (false !== $existingEntry) {
+            return;
-                ++$this->skippedEntries;
-                continue;
-            }
-
-            $data = [
-                'title' => $importedEntry['article__title'],
-                'url' => $importedEntry['article__url'],
-                'content_type' => '',
-                'language' => '',
-                'is_archived' => $importedEntry['archive'] || $this->markAsRead,
-                'is_starred' => $importedEntry['favorite'],
-            ];
-
-            $entry = $this->fetchContent(
-                new Entry($this->user),
-                $data['url'],
-                $data
-            );
-
-            // jump to next entry in case of problem while getting content
-            if (false === $entry) {
-                ++$this->skippedEntries;
-                continue;
-            }
-            $entry->setArchived($data['is_archived']);
-            $entry->setStarred($data['is_starred']);
-
-            $this->em->persist($entry);
-            ++$this->importedEntries;
-
-            // flush every 20 entries
-            if (($i % 20) === 0) {
-                $this->em->flush();
-            }
-            ++$i;
         }
 
-        $this->em->flush();
+        $data = [
-        $this->em->clear();
+            'title' => $importedEntry['article__title'],
+            'url' => $importedEntry['article__url'],
+            'content_type' => '',
+            'language' => '',
+            'is_archived' => $importedEntry['archive'] || $this->markAsRead,
+            'is_starred' => $importedEntry['favorite'],
+            'created_at' => $importedEntry['date_added'],
+        ];
+
+        $entry = new Entry($this->user);
+        $entry->setUrl($data['url']);
+        $entry->setTitle($data['title']);
+
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $entry = $this->fetchContent($entry, $data['url'], $data);
+
+        $entry->setArchived($data['is_archived']);
+        $entry->setStarred($data['is_starred']);
+        $entry->setCreatedAt(new \DateTime($data['created_at']));
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
+
+        return $entry;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function setEntryAsRead(array $importedEntry)
+    {
+        $importedEntry['archive'] = 1;
+
+        return $importedEntry;
     }
 }

src/Wallabag/ImportBundle/Import/WallabagImport.php

@@ -3,15 +3,10 @@
 namespace Wallabag\ImportBundle\Import;
 
 use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\UserBundle\Entity\User;
 
 abstract class WallabagImport extends AbstractImport
 {
-    protected $user;
-    protected $skippedEntries = 0;
-    protected $importedEntries = 0;
     protected $filepath;
-    protected $markAsRead;
     // untitled in all languages from v1
     protected $untitled = [
         'Untitled',
@@ -28,19 +23,6 @@ abstract class WallabagImport extends AbstractImport
         '',
     ];
 
-    /**
-     * We define the user in a custom call because on the import command there is no logged in user.
-     * So we can't retrieve user from the `security.token_storage` service.
-     *
-     * @param User $user
-     */
-    public function setUser(User $user)
-    {
-        $this->user = $user;
-
-        return $this;
-    }
-
     /**
      * {@inheritdoc}
      */
@@ -79,22 +61,17 @@ abstract class WallabagImport extends AbstractImport
             return false;
         }
 
+        if ($this->producer) {
+            $this->parseEntriesForProducer($data);
+
+            return true;
+        }
+
         $this->parseEntries($data);
 
         return true;
     }
 
-    /**
-     * {@inheritdoc}
-     */
-    public function getSummary()
-    {
-        return [
-            'skipped' => $this->skippedEntries,
-            'imported' => $this->importedEntries,
-        ];
-    }
-
     /**
      * Set file path to the json file.
      *
@@ -108,85 +85,59 @@ abstract class WallabagImport extends AbstractImport
     }
 
     /**
-     * Set whether articles must be all marked as read.
+     * {@inheritdoc}
-     *
-     * @param bool $markAsRead
      */
-    public function setMarkAsRead($markAsRead)
+    public function parseEntry(array $importedEntry)
     {
-        $this->markAsRead = $markAsRead;
+        $existingEntry = $this->em
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
 
-        return $this;
+        if (false !== $existingEntry) {
-    }
+            ++$this->skippedEntries;
 
-    /**
+            return;
-     * Parse and insert all given entries.
-     *
-     * @param $entries
-     */
-    protected function parseEntries($entries)
-    {
-        $i = 1;
-
-        foreach ($entries as $importedEntry) {
-            $existingEntry = $this->em
-                ->getRepository('WallabagCoreBundle:Entry')
-                ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
-
-            if (false !== $existingEntry) {
-                ++$this->skippedEntries;
-                continue;
-            }
-
-            $data = $this->prepareEntry($importedEntry, $this->markAsRead);
-
-            $entry = $this->fetchContent(
-                new Entry($this->user),
-                $importedEntry['url'],
-                $data
-            );
-
-            // jump to next entry in case of problem while getting content
-            if (false === $entry) {
-                ++$this->skippedEntries;
-                continue;
-            }
-
-            if (array_key_exists('tags', $data)) {
-                $this->contentProxy->assignTagsToEntry(
-                    $entry,
-                    $data['tags']
-                );
-            }
-
-            if (isset($importedEntry['preview_picture'])) {
-                $entry->setPreviewPicture($importedEntry['preview_picture']);
-            }
-
-            $entry->setArchived($data['is_archived']);
-            $entry->setStarred($data['is_starred']);
-
-            $this->em->persist($entry);
-            ++$this->importedEntries;
-
-            // flush every 20 entries
-            if (($i % 20) === 0) {
-                $this->em->flush();
-            }
-            ++$i;
         }
 
-        $this->em->flush();
+        $data = $this->prepareEntry($importedEntry);
-        $this->em->clear();
+
+        $entry = new Entry($this->user);
+        $entry->setUrl($data['url']);
+        $entry->setTitle($data['title']);
+
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $entry = $this->fetchContent($entry, $data['url'], $data);
+
+        if (array_key_exists('tags', $data)) {
+            $this->contentProxy->assignTagsToEntry(
+                $entry,
+                $data['tags']
+            );
+        }
+
+        if (isset($importedEntry['preview_picture'])) {
+            $entry->setPreviewPicture($importedEntry['preview_picture']);
+        }
+
+        $entry->setArchived($data['is_archived']);
+        $entry->setStarred($data['is_starred']);
+
+        if (!empty($data['created_at'])) {
+            $entry->setCreatedAt(new \DateTime($data['created_at']));
+        }
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
+
+        return $entry;
     }
 
     /**
      * This should return a cleaned array for a given entry to be given to `updateEntry`.
      *
-     * @param array $entry      Data from the imported file
+     * @param array $entry Data from the imported file
-     * @param bool  $markAsRead Should we mark as read content?
      *
      * @return array
      */
-    abstract protected function prepareEntry($entry = [], $markAsRead = false);
+    abstract protected function prepareEntry($entry = []);
 }

src/Wallabag/ImportBundle/Import/WallabagV1Import.php

@@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport
     /**
      * {@inheritdoc}
      */
-    protected function prepareEntry($entry = [], $markAsRead = false)
+    protected function prepareEntry($entry = [])
     {
         $data = [
             'title' => $entry['title'],
@@ -39,9 +39,10 @@ class WallabagV1Import extends WallabagImport
             'url' => $entry['url'],
             'content_type' => '',
             'language' => '',
-            'is_archived' => $entry['is_read'] || $markAsRead,
+            'is_archived' => $entry['is_read'] || $this->markAsRead,
             'is_starred' => $entry['is_fav'],
             'tags' => '',
+            'created_at' => '',
         ];
 
         // force content to be refreshed in case on bad fetch in the v1 installation
@@ -56,4 +57,14 @@ class WallabagV1Import extends WallabagImport
 
         return $data;
     }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function setEntryAsRead(array $importedEntry)
+    {
+        $importedEntry['is_read'] = 1;
+
+        return $importedEntry;
+    }
 }

src/Wallabag/ImportBundle/Import/WallabagV2Import.php

@@ -31,12 +31,22 @@ class WallabagV2Import extends WallabagImport
     /**
      * {@inheritdoc}
      */
-    protected function prepareEntry($entry = [], $markAsRead = false)
+    protected function prepareEntry($entry = [])
     {
         return [
             'html' => $entry['content'],
             'content_type' => $entry['mimetype'],
-            'is_archived' => ($entry['is_archived'] || $markAsRead),
+            'is_archived' => ($entry['is_archived'] || $this->markAsRead),
         ] + $entry;
     }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function setEntryAsRead(array $importedEntry)
+    {
+        $importedEntry['is_archived'] = 1;
+
+        return $importedEntry;
+    }
 }

src/Wallabag/ImportBundle/Redis/Producer.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Wallabag\ImportBundle\Redis;
+
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
+use Simpleue\Queue\RedisQueue;
+
+/**
+ * This is a proxy class for "Simpleue\Queue\RedisQueue".
+ * It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
+ *
+ * It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
+ * kind of class which implements the same interface.
+ * So we can inject either a RabbitMQ producer or a Redis producer with the same signature
+ */
+class Producer implements ProducerInterface
+{
+    private $queue;
+
+    public function __construct(RedisQueue $queue)
+    {
+        $this->queue = $queue;
+    }
+
+    /**
+     * Publish a message in the Redis queue.
+     *
+     * @param string $msgBody
+     * @param string $routingKey           NOT USED
+     * @param array  $additionalProperties NOT USED
+     */
+    public function publish($msgBody, $routingKey = '', $additionalProperties = array())
+    {
+        $this->queue->sendJob($msgBody);
+    }
+}

src/Wallabag/ImportBundle/Resources/config/rabbit.yml

@@ -0,0 +1,30 @@
+# RabbitMQ stuff
+services:
+    wallabag_import.consumer.amqp.pocket:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+    wallabag_import.consumer.amqp.readability:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+    wallabag_import.consumer.amqp.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+    wallabag_import.consumer.amqp.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"

src/Wallabag/ImportBundle/Resources/config/redis.yml

@@ -0,0 +1,81 @@
+# Redis stuff
+services:
+    # readability
+    wallabag_import.queue.redis.readability:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.readability"
+
+    wallabag_import.producer.redis.readability:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.readability"
+
+    wallabag_import.consumer.redis.readability:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+
+    # pocket
+    wallabag_import.queue.redis.pocket:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.pocket"
+
+    wallabag_import.producer.redis.pocket:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.pocket"
+
+    wallabag_import.consumer.redis.pocket:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+
+    # wallabag v1
+    wallabag_import.queue.redis.wallabag_v1:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v1"
+
+    wallabag_import.producer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v1"
+
+    wallabag_import.consumer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+
+    # wallabag v2
+    wallabag_import.queue.redis.wallabag_v2:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v2"
+
+    wallabag_import.producer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v2"
+
+    wallabag_import.consumer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -1,3 +1,7 @@
+imports:
+    - { resource: rabbit.yml }
+    - { resource: redis.yml }
+
 services:
     wallabag_import.chain:
         class: Wallabag\ImportBundle\Import\ImportChain
@@ -14,7 +18,6 @@ services:
     wallabag_import.pocket.import:
         class: Wallabag\ImportBundle\Import\PocketImport
         arguments:
-            - "@security.token_storage"
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
             - "@craue_config"

src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig

@@ -0,0 +1,8 @@
+{% set redis = craue_setting('import_with_redis') %}
+{% set rabbit = craue_setting('import_with_rabbitmq') %}
+
+{% if redis or rabbit %}
+    <div class="card-panel yellow darken-1 black-text">
+        {{ 'import.worker.enabled'|trans }} <strong>{% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}</strong>
+    </div>
+{% endif %}

src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig

@@ -0,0 +1,11 @@
+{% if nbRedisMessages > 0 %}
+    <script>
+        Materialize.toast('Messages in queue: {{ nbRedisMessages }}', 4000);
+    </script>
+{% endif %}
+
+{% if nbRabbitMessages > 0 %}
+    <script>
+        Materialize.toast('Messages in queue: {{ nbRabbitMessages }}', 4000);
+    </script>
+{% endif %}

src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig

@@ -6,15 +6,13 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
             {% if not has_consumer_key %}
-                <div class="card-panel red darken-1">
+                <div class="card-panel red white-text">
                     {{ 'import.pocket.config_missing.description'|trans }}
 
-                    {% if is_granted('ROLE_SUPER_ADMIN') %}
+                    {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('config') ~ '">', '%keyurle%':'</a>'})|raw }}
-                        {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('craue_config_settings_modify') ~ '#set-import">', '%keyurle%':'</a>'})|raw }}
-                    {% else %}
-                        {{ 'import.pocket.config_missing.user_message'|trans }}
-                    {% endif %}
                 </div>
             {% endif %}
 
@@ -29,7 +27,7 @@
                         {{ form_label(form.mark_as_read) }}
                     </div>
                 </div>
-                <button class="btn waves-effect waves-light" type="submit" name="action">
+                <button class="btn waves-effect waves-light" type="submit" name="action" {% if not has_consumer_key %}disabled="disabled"{% endif %}>
                     {{ 'import.pocket.connect_to_pocket'|trans }}
                 </button>
             </form>

src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig

@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>
                 <p>{{ 'import.readability.how_to'|trans }}</p>

src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig

@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>
                 <p>{{ 'import.wallabag_v1.how_to'|trans }}</p>

src/Wallabag/UserBundle/Resources/config/services.yml

@@ -14,3 +14,9 @@ services:
             - "@router"
         tags:
             - { name: kernel.event_subscriber }
+
+    wallabag_user.user_repository:
+        class: Wallabag\UserBundle\Repository\UserRepository
+        factory: [ "@doctrine.orm.default_entity_manager", getRepository ]
+        arguments:
+            - WallabagUserBundle:User

tests/Wallabag/ImportBundle/Command/ImportCommandTest.php

@@ -0,0 +1,86 @@
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Console\Application;
+use Symfony\Component\Console\Tester\CommandTester;
+use Wallabag\ImportBundle\Command\ImportCommand;
+use Tests\Wallabag\CoreBundle\WallabagCoreTestCase;
+use M6Web\Component\RedisMock\RedisMockFactory;
+
+class ImportCommandTest extends WallabagCoreTestCase
+{
+    /**
+     * @expectedException Symfony\Component\Console\Exception\RuntimeException
+     * @expectedExceptionMessage Not enough arguments
+     */
+    public function testRunImportCommandWithoutArguments()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new ImportCommand());
+
+        $command = $application->find('wallabag:import');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+        ]);
+    }
+
+    /**
+     * @expectedException Symfony\Component\Config\Definition\Exception\Exception
+     * @expectedExceptionMessage not found
+     */
+    public function testRunImportCommandWithoutFilepath()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new ImportCommand());
+
+        $command = $application->find('wallabag:import');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+            'userId' => 1,
+            'filepath' => 1,
+        ]);
+    }
+
+    /**
+     * @expectedException Symfony\Component\Config\Definition\Exception\Exception
+     * @expectedExceptionMessage User with id
+     */
+    public function testRunImportCommandWithoutUserId()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new ImportCommand());
+
+        $command = $application->find('wallabag:import');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+            'userId' => 0,
+            'filepath' => './',
+        ]);
+    }
+
+    public function testRunImportCommand()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new ImportCommand());
+
+        $command = $application->find('wallabag:import');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+            'userId' => 1,
+            'filepath' => $application->getKernel()->getContainer()->getParameter('kernel.root_dir').'/../tests/Wallabag/ImportBundle/fixtures/wallabag-v2-read.json',
+            '--importer' => 'v2',
+        ]);
+
+        $this->assertContains('imported', $tester->getDisplay());
+        $this->assertContains('already saved', $tester->getDisplay());
+    }
+}

tests/Wallabag/ImportBundle/Command/RedisWorkerCommandTest.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Console\Application;
+use Symfony\Component\Console\Tester\CommandTester;
+use Wallabag\ImportBundle\Command\RedisWorkerCommand;
+use Tests\Wallabag\CoreBundle\WallabagCoreTestCase;
+use M6Web\Component\RedisMock\RedisMockFactory;
+
+class RedisWorkerCommandTest extends WallabagCoreTestCase
+{
+    /**
+     * @expectedException Symfony\Component\Console\Exception\RuntimeException
+     * @expectedExceptionMessage Not enough arguments (missing: "serviceName")
+     */
+    public function testRunRedisWorkerCommandWithoutArguments()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new RedisWorkerCommand());
+
+        $command = $application->find('wallabag:import:redis-worker');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+        ]);
+    }
+
+    /**
+     * @expectedException Symfony\Component\Config\Definition\Exception\Exception
+     * @expectedExceptionMessage No queue or consumer found for service name
+     */
+    public function testRunRedisWorkerCommandWithBadService()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new RedisWorkerCommand());
+
+        $command = $application->find('wallabag:import:redis-worker');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+            'serviceName' => 'YOMONSERVICE',
+        ]);
+    }
+
+    public function testRunRedisWorkerCommand()
+    {
+        $application = new Application($this->getClient()->getKernel());
+        $application->add(new RedisWorkerCommand());
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $application->getKernel()->getContainer()->set('wallabag_core.redis.client', $redisMock);
+
+        // put a fake message in the queue so the worker will stop after reading that message
+        // instead of waiting for others
+        $redisMock->lpush('wallabag.import.readability', '{}');
+
+        $command = $application->find('wallabag:import:redis-worker');
+
+        $tester = new CommandTester($command);
+        $tester->execute([
+            'command' => $command->getName(),
+            'serviceName' => 'readability',
+            '--maxIterations' => 1,
+        ]);
+
+        $this->assertContains('Worker started at', $tester->getDisplay());
+        $this->assertContains('Waiting for message', $tester->getDisplay());
+    }
+}

tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php

@@ -0,0 +1,225 @@
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
+
+use Wallabag\ImportBundle\Consumer\AMQPEntryConsumer;
+use PhpAmqpLib\Message\AMQPMessage;
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\CoreBundle\Entity\Entry;
+
+class AMQPEntryConsumerTest extends \PHPUnit_Framework_TestCase
+{
+    public function testMessageOk()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->once())
+            ->method('flush');
+
+        $em
+            ->expects($this->exactly(2))
+            ->method('clear');
+
+        $body = <<<'JSON'
+{
+    "item_id": "1402935436",
+    "resolved_id": "1402935436",
+    "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+    "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+    "favorite": "0",
+    "status": "0",
+    "time_added": "1473020899",
+    "time_updated": "1473020899",
+    "time_read": "0",
+    "time_favorited": "0",
+    "sort_id": 0,
+    "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+    "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+    "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings.  It all started with a mic drop ...",
+    "is_article": "1",
+    "is_index": "0",
+    "has_video": "0",
+    "has_image": "1",
+    "word_count": "200",
+    "tags": {
+        "ifttt": {
+            "item_id": "1402935436",
+            "tag": "ifttt"
+        },
+        "mashable": {
+            "item_id": "1402935436",
+            "tag": "mashable"
+        }
+    },
+    "authors": {
+        "2484273": {
+            "item_id": "1402935436",
+            "author_id": "2484273",
+            "name": "Adam Rosenberg",
+            "url": "http://mashable.com/author/adam-rosenberg/"
+        }
+    },
+    "image": {
+        "item_id": "1402935436",
+        "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+        "width": "0",
+        "height": "0"
+    },
+    "images": {
+        "1": {
+            "item_id": "1402935436",
+            "image_id": "1",
+            "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+            "width": "0",
+            "height": "0",
+            "credit": "Image:  Steve Eichner/NameFace/Sipa USA",
+            "caption": ""
+        }
+    },
+    "userId": 1
+}
+JSON;
+
+        $user = new User();
+        $entry = new Entry($user);
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(1)
+            ->willReturn($user);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $import
+            ->expects($this->once())
+            ->method('setUser')
+            ->with($user);
+
+        $import
+            ->expects($this->once())
+            ->method('parseEntry')
+            ->with(json_decode($body, true))
+            ->willReturn($entry);
+
+        $consumer = new AMQPEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $message = new AMQPMessage($body);
+
+        $consumer->execute($message);
+    }
+
+    public function testMessageWithBadUser()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->never())
+            ->method('flush');
+
+        $em
+            ->expects($this->never())
+            ->method('clear');
+
+        $body = '{ "userId": 123 }';
+
+        $user = new User();
+        $entry = new Entry($user);
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(123)
+            ->willReturn(null);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $consumer = new AMQPEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $message = new AMQPMessage($body);
+
+        $consumer->execute($message);
+    }
+
+    public function testMessageWithEntryProcessed()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->never())
+            ->method('flush');
+
+        $em
+            ->expects($this->never())
+            ->method('clear');
+
+        $body = '{ "userId": 123 }';
+
+        $user = new User();
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(123)
+            ->willReturn($user);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $import
+            ->expects($this->once())
+            ->method('setUser')
+            ->with($user);
+
+        $import
+            ->expects($this->once())
+            ->method('parseEntry')
+            ->with(json_decode($body, true))
+            ->willReturn(null);
+
+        $consumer = new AMQPEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $message = new AMQPMessage($body);
+
+        $consumer->execute($message);
+    }
+}

tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php

@@ -0,0 +1,225 @@
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
+
+use Wallabag\ImportBundle\Consumer\RedisEntryConsumer;
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\CoreBundle\Entity\Entry;
+
+class RedisEntryConsumerTest extends \PHPUnit_Framework_TestCase
+{
+    public function testMessageOk()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->once())
+            ->method('flush');
+
+        $em
+            ->expects($this->exactly(2))
+            ->method('clear');
+
+        $body = <<<'JSON'
+{
+    "item_id": "1402935436",
+    "resolved_id": "1402935436",
+    "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+    "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+    "favorite": "0",
+    "status": "0",
+    "time_added": "1473020899",
+    "time_updated": "1473020899",
+    "time_read": "0",
+    "time_favorited": "0",
+    "sort_id": 0,
+    "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+    "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+    "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings.  It all started with a mic drop ...",
+    "is_article": "1",
+    "is_index": "0",
+    "has_video": "0",
+    "has_image": "1",
+    "word_count": "200",
+    "tags": {
+        "ifttt": {
+            "item_id": "1402935436",
+            "tag": "ifttt"
+        },
+        "mashable": {
+            "item_id": "1402935436",
+            "tag": "mashable"
+        }
+    },
+    "authors": {
+        "2484273": {
+            "item_id": "1402935436",
+            "author_id": "2484273",
+            "name": "Adam Rosenberg",
+            "url": "http://mashable.com/author/adam-rosenberg/"
+        }
+    },
+    "image": {
+        "item_id": "1402935436",
+        "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+        "width": "0",
+        "height": "0"
+    },
+    "images": {
+        "1": {
+            "item_id": "1402935436",
+            "image_id": "1",
+            "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+            "width": "0",
+            "height": "0",
+            "credit": "Image:  Steve Eichner/NameFace/Sipa USA",
+            "caption": ""
+        }
+    },
+    "userId": 1
+}
+JSON;
+
+        $user = new User();
+        $entry = new Entry($user);
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(1)
+            ->willReturn($user);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $import
+            ->expects($this->once())
+            ->method('setUser')
+            ->with($user);
+
+        $import
+            ->expects($this->once())
+            ->method('parseEntry')
+            ->with(json_decode($body, true))
+            ->willReturn($entry);
+
+        $consumer = new RedisEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $res = $consumer->manage($body);
+
+        $this->assertTrue($res);
+    }
+
+    public function testMessageWithBadUser()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->never())
+            ->method('flush');
+
+        $em
+            ->expects($this->never())
+            ->method('clear');
+
+        $body = '{ "userId": 123 }';
+
+        $user = new User();
+        $entry = new Entry($user);
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(123)
+            ->willReturn(null);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $consumer = new RedisEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $res = $consumer->manage($body);
+
+        $this->assertFalse($res);
+    }
+
+    public function testMessageWithEntryProcessed()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->never())
+            ->method('flush');
+
+        $em
+            ->expects($this->never())
+            ->method('clear');
+
+        $body = '{ "userId": 123 }';
+
+        $user = new User();
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(123)
+            ->willReturn($user);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $import
+            ->expects($this->once())
+            ->method('setUser')
+            ->with($user);
+
+        $import
+            ->expects($this->once())
+            ->method('parseEntry')
+            ->with(json_decode($body, true))
+            ->willReturn(null);
+
+        $consumer = new RedisEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $res = $consumer->manage($body);
+
+        $this->assertFalse($res);
+        $this->assertFalse($consumer->isStopJob($body));
+    }
+}

tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php

@@ -17,6 +17,36 @@ class PocketControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
     }
 
+    public function testImportPocketWithRabbitEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
+
+        $crawler = $client->request('GET', '/import/pocket');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportPocketWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/pocket');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
+    }
+
     public function testImportPocketAuthBadToken()
     {
         $this->logInAs('admin');

tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php

@@ -19,6 +19,74 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
     }
 
+    public function testImportReadabilityWithRabbitEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
+
+        $crawler = $client->request('GET', '/import/readability');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportReadabilityBadFile()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $crawler = $client->request('GET', '/import/readability');
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $data = [
+            'upload_import_file[file]' => '',
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+    }
+
+    public function testImportReadabilityWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/readability');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $file = new UploadedFile(__DIR__.'/../fixtures/readability.json', 'readability.json');
+
+        $data = [
+            'upload_import_file[file]' => $file,
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $crawler = $client->followRedirect();
+
+        $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+        $this->assertContains('flashes.import.notice.summary', $body[0]);
+
+        $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.readability'));
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
+    }
+
     public function testImportReadabilityWithFile()
     {
         $this->logInAs('admin');
@@ -49,6 +117,13 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
 
         $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
         $this->assertContains('flashes.import.notice.summary', $body[0]);
+
+        $this->assertNotEmpty($content->getMimetype());
+        $this->assertNotEmpty($content->getPreviewPicture());
+        $this->assertNotEmpty($content->getLanguage());
+        $this->assertEquals(0, count($content->getTags()));
+        $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt());
+        $this->assertEquals('2016-08-25', $content->getCreatedAt()->format('Y-m-d'));
     }
 
     public function testImportReadabilityWithFileAndMarkAllAsRead()

tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php

@@ -19,6 +19,74 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
     }
 
+    public function testImportWallabagWithRabbitEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
+
+        $crawler = $client->request('GET', '/import/wallabag-v1');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportWallabagBadFile()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $crawler = $client->request('GET', '/import/wallabag-v1');
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $data = [
+            'upload_import_file[file]' => '',
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+    }
+
+    public function testImportWallabagWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/wallabag-v1');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v1.json', 'wallabag-v1.json');
+
+        $data = [
+            'upload_import_file[file]' => $file,
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $crawler = $client->followRedirect();
+
+        $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+        $this->assertContains('flashes.import.notice.summary', $body[0]);
+
+        $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.wallabag_v1'));
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
+    }
+
     public function testImportWallabagWithFile()
     {
         $this->logInAs('admin');
@@ -56,6 +124,12 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
 
         $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
         $this->assertContains('flashes.import.notice.summary', $body[0]);
+
+        $this->assertEmpty($content->getMimetype());
+        $this->assertEmpty($content->getPreviewPicture());
+        $this->assertEmpty($content->getLanguage());
+        $this->assertEquals(1, count($content->getTags()));
+        $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt());
     }
 
     public function testImportWallabagWithFileAndMarkAllAsRead()

tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php

@@ -19,6 +19,74 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
     }
 
+    public function testImportWallabagWithRabbitEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportWallabagBadFile()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $data = [
+            'upload_import_file[file]' => '',
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+    }
+
+    public function testImportWallabagWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v2.json', 'wallabag-v2.json');
+
+        $data = [
+            'upload_import_file[file]' => $file,
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $crawler = $client->followRedirect();
+
+        $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+        $this->assertContains('flashes.import.notice.summary', $body[0]);
+
+        $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.wallabag_v2'));
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
+    }
+
     public function testImportWallabagWithFile()
     {
         $this->logInAs('admin');
@@ -50,9 +118,9 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
                 $this->getLoggedInUserId()
             );
 
-        $this->assertEmpty($content->getMimetype());
+        $this->assertNotEmpty($content->getMimetype());
-        $this->assertEmpty($content->getPreviewPicture());
+        $this->assertNotEmpty($content->getPreviewPicture());
-        $this->assertEmpty($content->getLanguage());
+        $this->assertNotEmpty($content->getLanguage());
         $this->assertEquals(0, count($content->getTags()));
 
         $content = $client->getContainer()
@@ -67,6 +135,8 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
         $this->assertNotEmpty($content->getPreviewPicture());
         $this->assertNotEmpty($content->getLanguage());
         $this->assertEquals(2, count($content->getTags()));
+        $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt());
+        $this->assertEquals('2016-09-08', $content->getCreatedAt()->format('Y-m-d'));
     }
 
     public function testImportWallabagWithEmptyFile()

tests/Wallabag/ImportBundle/Import/PocketImportTest.php

@@ -4,21 +4,17 @@ namespace Tests\Wallabag\ImportBundle\Import;
 
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Entity\Config;
 use Wallabag\ImportBundle\Import\PocketImport;
 use GuzzleHttp\Client;
 use GuzzleHttp\Subscriber\Mock;
 use GuzzleHttp\Message\Response;
 use GuzzleHttp\Stream\Stream;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
-
+use Simpleue\Queue\RedisQueue;
-class PocketImportMock extends PocketImport
+use M6Web\Component\RedisMock\RedisMockFactory;
-{
-    public function getAccessToken()
-    {
-        return $this->accessToken;
-    }
-}
 
 class PocketImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -32,45 +28,24 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
     {
         $this->user = new User();
 
-        $this->tokenStorage = $this->getMockBuilder('Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface')
+        $config = new Config($this->user);
-            ->disableOriginalConstructor()
+        $config->setPocketConsumerKey('xxx');
-            ->getMock();
 
-        $token = $this->getMockBuilder('Symfony\Component\Security\Core\Authentication\Token\TokenInterface')
+        $this->user->setConfig($config);
-            ->disableOriginalConstructor()
-            ->getMock();
 
         $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy')
             ->disableOriginalConstructor()
             ->getMock();
 
-        $token->expects($this->once())
-            ->method('getUser')
-            ->willReturn($this->user);
-
-        $this->tokenStorage->expects($this->once())
-            ->method('getToken')
-            ->willReturn($token);
-
         $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
             ->disableOriginalConstructor()
             ->getMock();
 
-        $config = $this->getMockBuilder('Craue\ConfigBundle\Util\Config')
+        $pocket = new PocketImport(
-            ->disableOriginalConstructor()
-            ->getMock();
-
-        $config->expects($this->any())
-            ->method('get')
-            ->with('pocket_consumer_key')
-            ->willReturn($consumerKey);
-
-        $pocket = new PocketImportMock(
-            $this->tokenStorage,
             $this->em,
-            $this->contentProxy,
+            $this->contentProxy
-            $config
         );
+        $pocket->setUser($this->user);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -189,10 +164,16 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
                             "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
                             "favorite": "1",
                             "status": "1",
+                            "time_added": "1473020899",
+                            "time_updated": "1473020899",
+                            "time_read": "0",
+                            "time_favorited": "0",
+                            "sort_id": 0,
                             "resolved_title": "The Massive Ryder Cup Preview",
                             "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
                             "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
                             "is_article": "1",
+                            "is_index": "0",
                             "has_video": "1",
                             "has_image": "1",
                             "word_count": "3197",
@@ -236,10 +217,16 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
                             "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
                             "favorite": "1",
                             "status": "1",
+                            "time_added": "1473020899",
+                            "time_updated": "1473020899",
+                            "time_read": "0",
+                            "time_favorited": "0",
+                            "sort_id": 1,
                             "resolved_title": "The Massive Ryder Cup Preview",
                             "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
                             "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
                             "is_article": "1",
+                            "is_index": "0",
                             "has_video": "0",
                             "has_image": "0",
                             "word_count": "3197"
@@ -279,7 +266,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
         $res = $pocketImport->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 1, 'imported' => 1], $pocketImport->getSummary());
+        $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $pocketImport->getSummary());
     }
 
     /**
@@ -302,6 +289,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
                             "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
                             "favorite": "1",
                             "status": "1",
+                            "time_added": "1473020899",
+                            "time_updated": "1473020899",
+                            "time_read": "0",
+                            "time_favorited": "0",
+                            "sort_id": 0,
                             "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
                             "is_article": "1",
                             "has_video": "1",
@@ -315,6 +307,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
                             "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
                             "favorite": "1",
                             "status": "0",
+                            "time_added": "1473020899",
+                            "time_updated": "1473020899",
+                            "time_read": "0",
+                            "time_favorited": "0",
+                            "sort_id": 1,
                             "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
                             "is_article": "1",
                             "has_video": "0",
@@ -364,7 +361,174 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
         $res = $pocketImport->setMarkAsRead(true)->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 0, 'imported' => 2], $pocketImport->getSummary());
+        $this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $pocketImport->getSummary());
+    }
+
+    /**
+     * Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
+     */
+    public function testImportWithRabbit()
+    {
+        $client = new Client();
+
+        $body = <<<'JSON'
+{
+    "item_id": "229279689",
+    "resolved_id": "229279689",
+    "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+    "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
+    "favorite": "1",
+    "status": "1",
+    "time_added": "1473020899",
+    "time_updated": "1473020899",
+    "time_read": "0",
+    "time_favorited": "0",
+    "sort_id": 0,
+    "resolved_title": "The Massive Ryder Cup Preview",
+    "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+    "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
+    "is_article": "1",
+    "has_video": "0",
+    "has_image": "0",
+    "word_count": "3197"
+}
+JSON;
+
+        $mock = new Mock([
+            new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
+            new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
+                {
+                    "status": 1,
+                    "list": {
+                        "229279690": '.$body.'
+                    }
+                }
+            ')),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $pocketImport = $this->getPocketImport();
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = new Entry($this->user);
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $bodyAsArray = json_decode($body, true);
+        // because with just use `new User()` so it doesn't have an id
+        $bodyAsArray['userId'] = null;
+
+        $producer
+            ->expects($this->once())
+            ->method('publish')
+            ->with(json_encode($bodyAsArray));
+
+        $pocketImport->setClient($client);
+        $pocketImport->setProducer($producer);
+        $pocketImport->authorize('wunderbar_code');
+
+        $res = $pocketImport->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $pocketImport->getSummary());
+    }
+
+    /**
+     * Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
+     */
+    public function testImportWithRedis()
+    {
+        $client = new Client();
+
+        $body = <<<'JSON'
+{
+    "item_id": "229279689",
+    "resolved_id": "229279689",
+    "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+    "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
+    "favorite": "1",
+    "status": "1",
+    "time_added": "1473020899",
+    "time_updated": "1473020899",
+    "time_read": "0",
+    "time_favorited": "0",
+    "sort_id": 0,
+    "resolved_title": "The Massive Ryder Cup Preview",
+    "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+    "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
+    "is_article": "1",
+    "has_video": "0",
+    "has_image": "0",
+    "word_count": "3197"
+}
+JSON;
+
+        $mock = new Mock([
+            new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
+            new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
+                {
+                    "status": 1,
+                    "list": {
+                        "229279690": '.$body.'
+                    }
+                }
+            ')),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $pocketImport = $this->getPocketImport();
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = new Entry($this->user);
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'pocket');
+        $producer = new Producer($queue);
+
+        $pocketImport->setClient($client);
+        $pocketImport->setProducer($producer);
+        $pocketImport->authorize('wunderbar_code');
+
+        $res = $pocketImport->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $pocketImport->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('pocket'));
     }
 
     public function testImportBadResponse()
@@ -402,6 +566,8 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
                     "status": 1,
                     "list": {
                         "229279689": {
+                            "status": "1",
+                            "favorite": "1",
                             "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview"
                         }
                     }
@@ -439,6 +605,6 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
         $res = $pocketImport->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 1, 'imported' => 0], $pocketImport->getSummary());
+        $this->assertEquals(['skipped' => 0, 'imported' => 1, 'queued' => 0], $pocketImport->getSummary());
     }
 }

tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php

@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
 use Wallabag\ImportBundle\Import\ReadabilityImport;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -58,9 +61,9 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $entryRepo->expects($this->exactly(2))
+        $entryRepo->expects($this->exactly(24))
             ->method('findByUrlAndUserId')
-            ->will($this->onConsecutiveCalls(false, true));
+            ->willReturn(false);
 
         $this->em
             ->expects($this->any())
@@ -72,14 +75,14 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
             ->getMock();
 
         $this->contentProxy
-            ->expects($this->exactly(1))
+            ->expects($this->exactly(24))
             ->method('updateEntry')
             ->willReturn($entry);
 
         $res = $readabilityImport->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 1, 'imported' => 1], $readabilityImport->getSummary());
+        $this->assertEquals(['skipped' => 0, 'imported' => 24, 'queued' => 0], $readabilityImport->getSummary());
     }
 
     public function testImportAndMarkAllAsRead()
@@ -93,7 +96,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
 
         $entryRepo->expects($this->exactly(2))
             ->method('findByUrlAndUserId')
-            ->will($this->onConsecutiveCalls(false, false));
+            ->will($this->onConsecutiveCalls(false, true));
 
         $this->em
             ->expects($this->any())
@@ -101,7 +104,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
             ->willReturn($entryRepo);
 
         $this->contentProxy
-            ->expects($this->exactly(2))
+            ->expects($this->exactly(1))
             ->method('updateEntry')
             ->willReturn(new Entry($this->user));
 
@@ -117,7 +120,87 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
 
         $this->assertTrue($res);
 
-        $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
+        $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $readabilityImport->getSummary());
+    }
+
+    public function testImportWithRabbit()
+    {
+        $readabilityImport = $this->getReadabilityImport();
+        $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $producer
+            ->expects($this->exactly(24))
+            ->method('publish');
+
+        $readabilityImport->setProducer($producer);
+
+        $res = $readabilityImport->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $readabilityImport->getSummary());
+    }
+
+    public function testImportWithRedis()
+    {
+        $readabilityImport = $this->getReadabilityImport();
+        $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'readability');
+        $producer = new Producer($queue);
+
+        $readabilityImport->setProducer($producer);
+
+        $res = $readabilityImport->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $readabilityImport->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('readability'));
     }
 
     public function testImportBadFile()

tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php

@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
 use Wallabag\ImportBundle\Import\WallabagV1Import;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -79,7 +82,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
         $res = $wallabagV1Import->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 1, 'imported' => 3], $wallabagV1Import->getSummary());
+        $this->assertEquals(['skipped' => 1, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary());
     }
 
     public function testImportAndMarkAllAsRead()
@@ -117,7 +120,87 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
 
         $this->assertTrue($res);
 
-        $this->assertEquals(['skipped' => 0, 'imported' => 3], $wallabagV1Import->getSummary());
+        $this->assertEquals(['skipped' => 0, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary());
+    }
+
+    public function testImportWithRabbit()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $producer
+            ->expects($this->exactly(4))
+            ->method('publish');
+
+        $wallabagV1Import->setProducer($producer);
+
+        $res = $wallabagV1Import->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary());
+    }
+
+    public function testImportWithRedis()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'wallabag_v1');
+        $producer = new Producer($queue);
+
+        $wallabagV1Import->setProducer($producer);
+
+        $res = $wallabagV1Import->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
     }
 
     public function testImportBadFile()

tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php

@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
 use Wallabag\ImportBundle\Import\WallabagV2Import;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -75,7 +78,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
         $res = $wallabagV2Import->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 22, 'imported' => 2], $wallabagV2Import->getSummary());
+        $this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
     }
 
     public function testImportAndMarkAllAsRead()
@@ -113,7 +116,79 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
 
         $this->assertTrue($res);
 
-        $this->assertEquals(['skipped' => 0, 'imported' => 2], $wallabagV2Import->getSummary());
+        $this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
+    }
+
+    public function testImportWithRabbit()
+    {
+        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $producer
+            ->expects($this->exactly(24))
+            ->method('publish');
+
+        $wallabagV2Import->setProducer($producer);
+
+        $res = $wallabagV2Import->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary());
+    }
+
+    public function testImportWithRedis()
+    {
+        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'wallabag_v2');
+        $producer = new Producer($queue);
+
+        $wallabagV2Import->setProducer($producer);
+
+        $res = $wallabagV2Import->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
     }
 
     public function testImportBadFile()
@@ -152,7 +227,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
         $res = $wallabagV2Import->import();
 
         $this->assertFalse($res);
-        $this->assertEquals(['skipped' => 0, 'imported' => 0], $wallabagV2Import->getSummary());
+        $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 0], $wallabagV2Import->getSummary());
     }
 
     public function testImportWithExceptionFromGraby()
@@ -181,6 +256,6 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
         $res = $wallabagV2Import->import();
 
         $this->assertTrue($res);
-        $this->assertEquals(['skipped' => 24, 'imported' => 0], $wallabagV2Import->getSummary());
+        $this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
     }
 }

tests/Wallabag/ImportBundle/fixtures/readability.json

@@ -11,14 +11,165 @@
             "archive": false
         },
         {
-            "article__excerpt": "TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data. Here’s a demo and the code for the infinite…",
+            "article__title": "Réfugiés: l'UE va créer 100 000 places d'accueil dans les Balkans",
-            "favorite": false,
+            "article__url": "http://www.liberation.fr/planete/2015/10/26/refugies-l-ue-va-creer-100-000-places-d-accueil-dans-les-balkans_1408867",
-            "date_archived": "2016-08-26T12:21:54",
+            "archive": false,
-            "article__url": "https://developers.google.com/web/updates/2016/07/infinite-scroller?imm_mid=0e6839&cmp=em-webops-na-na-newsltr_20160805",
+            "date_added": "2016-09-08T11:55:58+0200",
-            "date_added": "2016-08-06T05:35:26",
+            "favorite": false
-            "date_favorited": null,
+        },
-            "article__title": "Complexities of an infinite scroller | Web Updates",
+        {
-            "archive": true
+            "article__title": "No title found",
+            "article__url": "http://news.nationalgeographic.com/2016/02/160211-albatrosses-mothers-babies-animals-science/&sf20739758=1",
+            "archive": false,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": true
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Échecs",
+            "article__url": "https://fr.wikipedia.org/wiki/Échecs"
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "90% des dossiers médicaux des Coréens du sud vendus à des entreprises privées - ZATAZ",
+            "article__url": "http://www.zataz.com/90-des-dossiers-medicaux-des-coreens-du-sud-vendus-a-des-entreprises-privees/"
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Mass Surveillance As Art",
+            "article__url": "https://www.nationaljournal.com/s/73311/mass-surveillance-art"
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "What David Cameron did to the pig, his party is now doing to the country",
+            "article__url": "http://www.newstatesman.com/2015/09/what-david-cameron-did-pig-his-party-now-doing-country"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "CLICK HERE to support 2016 CES Winner, Revolutionary Auto-Tracking Robot",
+            "article__url": "https://www.indiegogo.com/projects/2016-ces-winner-revolutionary-auto-tracking-robot"
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 1,
+            "article__title": "No title found",
+            "article__url": "http://carnetdevol.shost.ca/wordpress/aide-memoire-sur-les-commandes-associees-a-systemd/"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Présentation d'Arduino - Tuto Arduino - Le blog d'Eskimon",
+            "article__url": "http://eskimon.fr/73-arduino-101-presentation"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Lenovo ThinkPad X1 Carbon Ultrabook Review",
+            "article__url": "http://www.notebookcheck.net/Lenovo-ThinkPad-X1-Carbon-Ultrabook-Review.138033.0.html"
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Visitons le Château de Landsberg !",
+            "article__url": "http://autour-du-mont-sainte-odile.overblog.com/2016/01/visitons-le-chateau-de-landsberg.html"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Contrer les stéréotypes par les livres : “C'est dès l'enfance qu'ils se construisent”",
+            "article__url": "https://www.actualitte.com/article/monde-edition/contrer-les-stereotypes-par-les-livres-c-est-des-l-enfance-qu-ils-se-construisent/64058"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "[ROM][6.0.1][Layers][N5] TipsyOS official builds {UBER TCs}",
+            "article__url": "http://forum.xda-developers.com/google-nexus-5/development/rom-tipsyos-official-builds-uber-tcs-t3325989"
+        },
+        {
+            "archive": 0,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Top 15 Podcasts All Web Developers Should Follow - Envato Tuts+ Code Article",
+            "article__url": "http://code.tutsplus.com/articles/top-15-podcasts-all-web-developers-should-follow--net-14461"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "University of Mississippi",
+            "article__url": "http://olemiss.edu"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "FinnChristiansen.de Jetzt Dank Let’s Encrypt Per HTTPS Erreichbar",
+            "article__url": "https://www.finnchristiansen.de/2015/12/06/finnchristiansen-de-jetzt-dank-lets-encrypt-per-https-erreichbar/"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Le développeur et l'ingénierie logicielle",
+            "article__url": "http://wemucs.com/le-developpeur-et-lingenierie-logicielle/"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "The Role of Methylation in Gene Expression",
+            "article__url": "http://www.nature.com/scitable/topicpage/the-role-of-methylation-in-gene-expression-1070"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "E-Mail-Adresse kostenlos, FreeMail, De-Mail & Nachrichten",
+            "article__url": "http://web.de"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "OpenSSH Server on Arch Linux | DominicM test",
+            "article__url": "http://dominicm.com/openssh-server-arch-linux/"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Site Moved | Site Help",
+            "article__url": "http://g1.com/help/sitemoved.asp"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "#Maroc : le stylo anti-pédophiles EAGLE d’AMESYS est moins bien configuré que les faux-lowers Twitter du roi Mohammed VI",
+            "article__url": "https://reflets.info/maroc-le-stylo-anti-pedophiles-eagle-damesys-est-moins-bien-configure-que-les-faux-lowers-twitter-du-roi-mohammed-vi/"
+        },
+        {
+            "archive": 1,
+            "date_added": "2016-09-08T11:55:58+0200",
+            "favorite": 0,
+            "article__title": "Simple Cloud Infrastructure for Developers",
+            "article__url": "https://www.digitalocean.com/"
         }
    ],
     "recommendations": []