Added Omnivore Import

src/Wallabag/ImportBundle/Command/ImportCommand.php

@@ -15,6 +15,7 @@ use Wallabag\ImportBundle\Import\ChromeImport;
 use Wallabag\ImportBundle\Import\DeliciousImport;
 use Wallabag\ImportBundle\Import\FirefoxImport;
 use Wallabag\ImportBundle\Import\InstapaperImport;
+use Wallabag\ImportBundle\Import\OmnivoreImport;
 use Wallabag\ImportBundle\Import\PinboardImport;
 use Wallabag\ImportBundle\Import\ReadabilityImport;
 use Wallabag\ImportBundle\Import\WallabagV1Import;
@@ -34,9 +35,10 @@ class ImportCommand extends Command
     private InstapaperImport $instapaperImport;
     private PinboardImport $pinboardImport;
     private DeliciousImport $deliciousImport;
+    private OmnivoreImport $omnivoreImport;
     private WallabagV1Import $wallabagV1Import;
 
-    public function __construct(EntityManagerInterface $entityManager, TokenStorageInterface $tokenStorage, UserRepository $userRepository, WallabagV2Import $wallabagV2Import, FirefoxImport $firefoxImport, ChromeImport $chromeImport, ReadabilityImport $readabilityImport, InstapaperImport $instapaperImport, PinboardImport $pinboardImport, DeliciousImport $deliciousImport, WallabagV1Import $wallabagV1Import)
+    public function __construct(EntityManagerInterface $entityManager, TokenStorageInterface $tokenStorage, UserRepository $userRepository, WallabagV2Import $wallabagV2Import, FirefoxImport $firefoxImport, ChromeImport $chromeImport, ReadabilityImport $readabilityImport, InstapaperImport $instapaperImport, PinboardImport $pinboardImport, DeliciousImport $deliciousImport, OmnivoreImport $omnivoreImport, WallabagV1Import $wallabagV1Import)
     {
         $this->entityManager = $entityManager;
         $this->tokenStorage = $tokenStorage;
@@ -48,6 +50,7 @@ class ImportCommand extends Command
         $this->instapaperImport = $instapaperImport;
         $this->pinboardImport = $pinboardImport;
         $this->deliciousImport = $deliciousImport;
+        $this->omnivoreImport = $omnivoreImport;
         $this->wallabagV1Import = $wallabagV1Import;
 
         parent::__construct();
@@ -120,6 +123,9 @@ class ImportCommand extends Command
             case 'delicious':
                 $import = $this->deliciousImport;
                 break;
+            case 'omnivore':
+                $import = $this->omnivoreImport;
+                break;
             default:
                 $import = $this->wallabagV1Import;
         }

src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php

@@ -27,7 +27,7 @@ class RedisWorkerCommand extends Command
         $this
             ->setName('wallabag:import:redis-worker')
             ->setDescription('Launch Redis worker')
-            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, pinboard, delicious, firefox, chrome or instapaper')
+            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, pinboard, delicious, omnivore, firefox, chrome or instapaper')
             ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stopping', false)
         ;
     }

src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php

@@ -20,8 +20,9 @@ class RabbitMQConsumerTotalProxy
     private Consumer $pinboardConsumer;
     private Consumer $deliciousConsumer;
     private Consumer $elcuratorConsumer;
+    private Consumer $omnivoreConsumer;
 
-    public function __construct(Consumer $pocketConsumer, Consumer $readabilityConsumer, Consumer $wallabagV1Consumer, Consumer $wallabagV2Consumer, Consumer $firefoxConsumer, Consumer $chromeConsumer, Consumer $instapaperConsumer, Consumer $pinboardConsumer, Consumer $deliciousConsumer, Consumer $elcuratorConsumer)
+    public function __construct(Consumer $pocketConsumer, Consumer $readabilityConsumer, Consumer $wallabagV1Consumer, Consumer $wallabagV2Consumer, Consumer $firefoxConsumer, Consumer $chromeConsumer, Consumer $instapaperConsumer, Consumer $pinboardConsumer, Consumer $deliciousConsumer, Consumer $elcuratorConsumer, Consumer $omnivoreConsumer)
     {
         $this->pocketConsumer = $pocketConsumer;
         $this->readabilityConsumer = $readabilityConsumer;
@@ -33,6 +34,7 @@ class RabbitMQConsumerTotalProxy
         $this->pinboardConsumer = $pinboardConsumer;
         $this->deliciousConsumer = $deliciousConsumer;
         $this->elcuratorConsumer = $elcuratorConsumer;
+        $this->omnivoreConsumer = $omnivoreConsumer;
     }
 
     /**
@@ -77,6 +79,9 @@ class RabbitMQConsumerTotalProxy
             case 'elcurator':
                 $consumer = $this->elcuratorConsumer;
                 break;
+            case 'omnivore':
+                $consumer = $this->omnivoreConsumer;
+                break;
             default:
                 return 0;
         }

src/Wallabag/ImportBundle/Controller/ImportController.php

@@ -57,6 +57,7 @@ class ImportController extends AbstractController
                     + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pinboard')
                     + $this->rabbitMQConsumerTotalProxy->getTotalMessage('delicious')
                     + $this->rabbitMQConsumerTotalProxy->getTotalMessage('elcurator')
+                    + $this->rabbitMQConsumerTotalProxy->getTotalMessage('omnivore')
                 ;
             } catch (\Exception $e) {
                 $rabbitNotInstalled = true;
@@ -75,6 +76,7 @@ class ImportController extends AbstractController
                     + $redis->llen('wallabag.import.pinboard')
                     + $redis->llen('wallabag.import.delicious')
                     + $redis->llen('wallabag.import.elcurator')
+                    + $redis->llen('wallabag.import.omnivore')
                 ;
             } catch (\Exception $e) {
                 $redisNotInstalled = true;

src/Wallabag/ImportBundle/Controller/OmnivoreController.php

@@ -0,0 +1,84 @@
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Craue\ConfigBundle\Util\Config;
+use OldSound\RabbitMqBundle\RabbitMq\Producer as RabbitMqProducer;
+use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\Routing\Annotation\Route;
+use Symfony\Contracts\Translation\TranslatorInterface;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+use Wallabag\ImportBundle\Import\OmnivoreImport;
+use Wallabag\ImportBundle\Redis\Producer as RedisProducer;
+
+class OmnivoreController extends AbstractController
+{
+    private RabbitMqProducer $rabbitMqProducer;
+    private RedisProducer $redisProducer;
+
+    public function __construct(RabbitMqProducer $rabbitMqProducer, RedisProducer $redisProducer)
+    {
+        $this->rabbitMqProducer = $rabbitMqProducer;
+        $this->redisProducer = $redisProducer;
+    }
+
+    /**
+     * @Route("/omnivore", name="import_omnivore")
+     */
+    public function indexAction(Request $request, OmnivoreImport $omnivore, Config $craueConfig, TranslatorInterface $translator)
+    {
+        $form = $this->createForm(UploadImportType::class);
+        $form->handleRequest($request);
+
+        $omnivore->setUser($this->getUser());
+
+        if ($craueConfig->get('import_with_rabbitmq')) {
+            $omnivore->setProducer($this->rabbitMqProducer);
+        } elseif ($craueConfig->get('import_with_redis')) {
+            $omnivore->setProducer($this->redisProducer);
+        }
+
+        if ($form->isSubmitted() && $form->isValid()) {
+            $file = $form->get('file')->getData();
+            $markAsRead = $form->get('mark_as_read')->getData();
+            $name = 'omnivore_' . $this->getUser()->getId() . '.json';
+
+            if (null !== $file && \in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes'), true) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+                $res = $omnivore
+                    ->setFilepath($this->getParameter('wallabag_import.resource_dir') . '/' . $name)
+                    ->setMarkAsRead($markAsRead)
+                    ->import();
+
+                $message = 'flashes.import.notice.failed';
+
+                if (true === $res) {
+                    $summary = $omnivore->getSummary();
+                    $message = $translator->trans('flashes.import.notice.summary', [
+                        '%imported%' => $summary['imported'],
+                        '%skipped%' => $summary['skipped'],
+                    ]);
+
+                    if (0 < $summary['queued']) {
+                        $message = $translator->trans('flashes.import.notice.summary_with_queue', [
+                            '%queued%' => $summary['queued'],
+                        ]);
+                    }
+
+                    unlink($this->getParameter('wallabag_import.resource_dir') . '/' . $name);
+                }
+
+                $this->addFlash('notice', $message);
+
+                return $this->redirect($this->generateUrl('homepage'));
+            }
+
+            $this->addFlash('notice', 'flashes.import.notice.failed_on_file');
+        }
+
+        return $this->render('@WallabagImport/Omnivore/index.html.twig', [
+            'form' => $form->createView(),
+            'import' => $omnivore,
+        ]);
+    }
+}

src/Wallabag/ImportBundle/Import/OmnivoreImport.php

@@ -0,0 +1,158 @@
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Wallabag\CoreBundle\Entity\Entry;
+
+class OmnivoreImport extends AbstractImport
+{
+    private $filepath;
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getName()
+    {
+        return 'Omnivore';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getUrl()
+    {
+        return 'import_omnivore';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDescription()
+    {
+        return 'import.omnivore.description';
+    }
+
+    /**
+     * Set file path to the json file.
+     *
+     * @param string $filepath
+     */
+    public function setFilepath($filepath)
+    {
+        $this->filepath = $filepath;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function import()
+    {
+        if (!$this->user) {
+            $this->logger->error('OmnivoreImport: user is not defined');
+
+            return false;
+        }
+
+        if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+            $this->logger->error('OmnivoreImport: unable to read file', ['filepath' => $this->filepath]);
+
+            return false;
+        }
+
+        $data = json_decode(file_get_contents($this->filepath), true);
+
+        if (empty($data)) {
+            $this->logger->error('OmnivoreImport: no entries in imported file');
+
+            return false;
+        }
+
+        if ($this->producer) {
+            $this->parseEntriesForProducer($data);
+
+            return true;
+        }
+
+        $this->parseEntries($data);
+
+        return true;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function validateEntry(array $importedEntry)
+    {
+        if (empty($importedEntry['url'])) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function parseEntry(array $importedEntry)
+    {
+        $existingEntry = $this->em
+            ->getRepository(Entry::class)
+            ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
+
+        if (false !== $existingEntry) {
+            ++$this->skippedEntries;
+
+            return;
+        }
+
+        $data = [
+            'title' => $importedEntry['title'],
+            'url' => $importedEntry['url'],
+            'is_archived' => ('Archived' === $importedEntry['state']) || $this->markAsRead,
+            'is_starred' => false,
+            'created_at' => $importedEntry['savedAt'],
+            'tags' => $importedEntry['labels'],
+            'published_by' => [$importedEntry['author']],
+            'published_at' => $importedEntry['publishedAt'],
+            'preview_picture' => $importedEntry['thumbnail'],
+        ];
+
+        $entry = new Entry($this->user);
+        $entry->setUrl($data['url']);
+        $entry->setTitle($data['title']);
+
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $this->fetchContent($entry, $data['url'], $data);
+
+        if (!empty($data['tags'])) {
+            $this->tagsAssigner->assignTagsToEntry(
+                $entry,
+                $data['tags'],
+                $this->em->getUnitOfWork()->getScheduledEntityInsertions()
+            );
+        }
+
+        $entry->updateArchived($data['is_archived']);
+        $entry->setCreatedAt(\DateTime::createFromFormat('Y-m-d\TH:i:s.u\Z', $data['created_at']));
+        if (null !== $data['published_at']) {
+            $entry->setPublishedAt(\DateTime::createFromFormat('Y-m-d\TH:i:s.u\Z', $data['published_at']));
+        }
+        $entry->setPublishedBy($data['published_by']);
+        $entry->setPreviewPicture($data['preview_picture']);
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
+
+        return $entry;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function setEntryAsRead(array $importedEntry)
+    {
+        return $importedEntry;
+    }
+}

src/Wallabag/ImportBundle/Resources/views/Omnivore/index.html.twig

@@ -0,0 +1,45 @@
+{% extends "@WallabagCore/layout.html.twig" %}
+
+{% block title %}{{ 'import.omnivore.page_title'|trans }}{% endblock %}
+
+{% block content %}
+<div class="row">
+    <div class="col s12">
+        <div class="card-panel settings">
+            {% include '@WallabagImport/Import/_information.html.twig' %}
+
+            <div class="row">
+                <blockquote>{{ import.description|trans }}</blockquote>
+                <p>{{ 'import.omnivore.how_to'|trans }}</p>
+
+                <div class="col s12">
+                    {{ form_start(form, {'method': 'POST'}) }}
+                        {{ form_errors(form) }}
+                        <div class="row">
+                            <div class="file-field input-field col s12">
+                                {{ form_errors(form.file) }}
+                                <div class="btn">
+                                    <span>{{ form.file.vars.label|trans }}</span>
+                                    {{ form_widget(form.file) }}
+                                </div>
+                                <div class="file-path-wrapper">
+                                    <input class="file-path validate" type="text">
+                                </div>
+                            </div>
+                            <div class="input-field col s6 with-checkbox">
+                                <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
+                                {{ form_widget(form.mark_as_read) }}
+                                {{ form_label(form.mark_as_read) }}
+                            </div>
+                        </div>
+
+                        {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }}
+
+                        {{ form_rest(form) }}
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}