Enable Redis async import

- using javibravo/simpleue
- internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear
- if both option are enable rabbit will be choosen
- services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml`
-

src/Wallabag/CoreBundle/Command/InstallCommand.php

@@ -322,7 +322,12 @@ class InstallCommand extends ContainerAwareCommand
                 'section' => 'import',
             ],
             [
-                'name' => 'rabbitmq',
+                'name' => 'import_with_redis',
+                'value' => '0',
+                'section' => 'import',
+            ],
+            [
+                'name' => 'import_with_rabbitmq',
                 'value' => '0',
                 'section' => 'import',
             ],

src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php

@@ -96,7 +96,12 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
                 'section' => 'import',
             ],
             [
-                'name' => 'rabbitmq',
+                'name' => 'import_with_redis',
+                'value' => '0',
+                'section' => 'import',
+            ],
+            [
+                'name' => 'import_with_rabbitmq',
                 'value' => '0',
                 'section' => 'import',
             ],

src/Wallabag/CoreBundle/Resources/config/services.yml

@@ -125,3 +125,11 @@ services:
         arguments:
             - "@security.token_storage"
             - "@router"
+
+    wallabag_core.redis.client:
+        class: Predis\Client
+        arguments:
+            -
+                host: '%redis_host%'
+                port: '%redis_port%'
+                schema: tcp

src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace Wallabag\ImportBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Config\Definition\Exception\Exception;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Simpleue\Worker\QueueWorker;
+
+class RedisWorkerCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('wallabag:import:redis-worker')
+            ->setDescription('Launch Redis worker')
+            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
+        $output->writeln('Waiting for message ...');
+
+        $serviceName = $input->getArgument('serviceName');
+
+        if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
+            throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
+        }
+
+        $worker = new QueueWorker(
+            $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
+            $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName)
+        );
+
+        $worker->start();
+    }
+}

src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Wallabag\ImportBundle\Consumer\AMPQ;
+namespace Wallabag\ImportBundle\Consumer;
 
 use Doctrine\ORM\EntityManager;
 use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
@@ -12,7 +12,7 @@ use Wallabag\CoreBundle\Entity\Tag;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
 
-class EntryConsumer implements ConsumerInterface
+class AMPQEntryConsumer implements ConsumerInterface
 {
     private $em;
     private $userRepository;
@@ -64,5 +64,7 @@ class EntryConsumer implements ConsumerInterface
 
             return;
         }
+
+        $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
     }
 }

src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php

@@ -0,0 +1,84 @@
+<?php
+
+namespace Wallabag\ImportBundle\Consumer;
+
+use Simpleue\Job\Job;
+use Doctrine\ORM\EntityManager;
+use Wallabag\ImportBundle\Import\AbstractImport;
+use Wallabag\UserBundle\Repository\UserRepository;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Entity\Tag;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+
+class RedisEntryConsumer implements Job
+{
+    private $em;
+    private $userRepository;
+    private $import;
+    private $logger;
+
+    public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
+    {
+        $this->em = $em;
+        $this->userRepository = $userRepository;
+        $this->import = $import;
+        $this->logger = $logger ?: new NullLogger();
+    }
+
+    /**
+     * Handle one message by one message.
+     *
+     * @param string $job Content of the message (directly from Redis)
+     *
+     * @return bool
+     */
+    public function manage($job)
+    {
+        $storedEntry = json_decode($job, true);
+
+        $user = $this->userRepository->find($storedEntry['userId']);
+
+        // no user? Drop message
+        if (null === $user) {
+            $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
+
+            return false;
+        }
+
+        $this->import->setUser($user);
+
+        $entry = $this->import->parseEntry($storedEntry);
+
+        if (null === $entry) {
+            $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
+
+            return false;
+        }
+
+        try {
+            $this->em->flush();
+
+            // clear only affected entities
+            $this->em->clear(Entry::class);
+            $this->em->clear(Tag::class);
+        } catch (\Exception $e) {
+            $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
+
+            return false;
+        }
+
+        $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
+
+        return true;
+    }
+
+    /**
+     * Should tell if the given job will kill the worker.
+     * We don't want to stop it :).
+     */
+    public function isStopJob($job)
+    {
+        return false;
+    }
+}

src/Wallabag/ImportBundle/Controller/PocketController.php

@@ -20,8 +20,10 @@ class PocketController extends Controller
         $pocket = $this->get('wallabag_import.pocket.import');
         $pocket->setUser($this->getUser());
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
-            $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
         }
 
         return $pocket;

src/Wallabag/ImportBundle/Controller/ReadabilityController.php

@@ -20,8 +20,10 @@ class ReadabilityController extends Controller
         $readability = $this->get('wallabag_import.readability.import');
         $readability->setUser($this->getUser());
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
-            $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
         }
 
         if ($form->isValid()) {

src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php

@@ -14,8 +14,10 @@ class WallabagV1Controller extends WallabagController
     {
         $service = $this->get('wallabag_import.wallabag_v1.import');
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
-            $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
         }
 
         return $service;

src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php

@@ -14,8 +14,10 @@ class WallabagV2Controller extends WallabagController
     {
         $service = $this->get('wallabag_import.wallabag_v2.import');
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
-            $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
         }
 
         return $service;

src/Wallabag/ImportBundle/Import/AbstractImport.php

@@ -9,7 +9,7 @@ use Wallabag\CoreBundle\Helper\ContentProxy;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
 use Wallabag\UserBundle\Entity\User;
-use OldSound\RabbitMqBundle\RabbitMq\Producer;
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
 
 abstract class AbstractImport implements ImportInterface
 {
@@ -35,12 +35,12 @@ abstract class AbstractImport implements ImportInterface
     }
 
     /**
-     * Set RabbitMQ Producer to send each entry to a queue.
+     * Set RabbitMQ/Redis Producer to send each entry to a queue.
      * This method should be called when user has enabled RabbitMQ.
      *
-     * @param Producer $producer
+     * @param ProducerInterface $producer
      */
-    public function setRabbitmqProducer(Producer $producer)
+    public function setProducer(ProducerInterface $producer)
     {
         $this->producer = $producer;
     }

src/Wallabag/ImportBundle/Redis/Producer.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Wallabag\ImportBundle\Redis;
+
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
+use Simpleue\Queue\RedisQueue;
+
+/**
+ * This is a proxy class for "Simpleue\Queue\RedisQueue".
+ * It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
+ *
+ * It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
+ * kind of class which implements the same interface.
+ * So we can inject either a RabbitMQ producer or a Redis producer with the same signature
+ */
+class Producer implements ProducerInterface
+{
+    private $queue;
+
+    public function __construct(RedisQueue $queue)
+    {
+        $this->queue = $queue;
+    }
+
+    /**
+     * Publish a message in the Redis queue.
+     *
+     * @param string $msgBody
+     * @param string $routingKey           NOT USED
+     * @param array  $additionalProperties NOT USED
+     */
+    public function publish($msgBody, $routingKey = '', $additionalProperties = array())
+    {
+        $this->queue->sendJob($msgBody);
+    }
+}

src/Wallabag/ImportBundle/Resources/config/rabbit.yml

@@ -0,0 +1,30 @@
+# RabbitMQ stuff
+services:
+    wallabag_import.consumer.ampq.pocket:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+    wallabag_import.consumer.ampq.readability:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+    wallabag_import.consumer.ampq.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+    wallabag_import.consumer.ampq.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"

src/Wallabag/ImportBundle/Resources/config/redis.yml

@@ -0,0 +1,81 @@
+# Redis stuff
+services:
+    # readability
+    wallabag_import.queue.redis.readability:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.readability"
+
+    wallabag_import.producer.redis.readability:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.readability"
+
+    wallabag_import.consumer.redis.readability:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+
+    # pocket
+    wallabag_import.queue.redis.pocket:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.pocket"
+
+    wallabag_import.producer.redis.pocket:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.pocket"
+
+    wallabag_import.consumer.redis.pocket:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+
+    # wallabag v1
+    wallabag_import.queue.redis.wallabag_v1:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v1"
+
+    wallabag_import.producer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v1"
+
+    wallabag_import.consumer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+
+    # wallabag v2
+    wallabag_import.queue.redis.wallabag_v2:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v2"
+
+    wallabag_import.producer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v2"
+
+    wallabag_import.consumer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -1,33 +1,8 @@
-services:
-    wallabag_import.consumer.pocket:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.pocket.import"
-            - "@logger"
-    wallabag_import.consumer.readability:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.readability.import"
-            - "@logger"
-    wallabag_import.consumer.wallabag_v1:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.wallabag_v1.import"
-            - "@logger"
-    wallabag_import.consumer.wallabag_v2:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.wallabag_v2.import"
-            - "@logger"
+imports:
+    - { resource: rabbit.yml }
+    - { resource: redis.yml }
 
+services:
     wallabag_import.chain:
         class: Wallabag\ImportBundle\Import\ImportChain