Add Delicious import

Since 2021, you can export again your data \o/

Also fix indentation in json fixtures files.

src/Wallabag/CoreBundle/Resources/translations/messages.en.yml

@@ -517,6 +517,10 @@ import:
         page_title: Import > Pinboard
         description: This importer will import all your Pinboard articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").
         how_to: Please select your Pinboard export and click on the below button to upload and import it.
+    delicious:
+        page_title: Import > del.icio.us
+        description: This importer will import all your Delicious bookmarks. Since 2021, you can export again your data from it using the export page (https://del.icio.us/export). Choose the "JSON" format and download it (like "delicious_export.2021.02.06_21.10.json").
+        how_to: Please select your Delicious export and click on the below button to upload and import it.
 developer:
     page_title: API clients management
     welcome_message: Welcome to the wallabag API

src/Wallabag/ImportBundle/Command/ImportCommand.php

@@ -19,7 +19,7 @@ class ImportCommand extends ContainerAwareCommand
             ->setDescription('Import entries from a JSON export')
             ->addArgument('username', InputArgument::REQUIRED, 'User to populate')
             ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file')
-            ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, readability, firefox or chrome', 'v1')
+            ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox or chrome', 'v1')
             ->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false)
             ->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account')
             ->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL')
@@ -77,6 +77,9 @@ class ImportCommand extends ContainerAwareCommand
             case 'pinboard':
                 $import = $this->getContainer()->get('wallabag_import.pinboard.import');
                 break;
+            case 'delicious':
+                $import = $this->getContainer()->get('wallabag_import.delicious.import');
+                break;
             default:
                 $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import');
         }

src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php

@@ -17,7 +17,7 @@ class RedisWorkerCommand extends ContainerAwareCommand
         $this
             ->setName('wallabag:import:redis-worker')
             ->setDescription('Launch Redis worker')
-            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, pinboard, firefox, chrome or instapaper')
+            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, pinboard, delicious, firefox, chrome or instapaper')
             ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false)
         ;
     }

src/Wallabag/ImportBundle/Controller/DeliciousController.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Symfony\Bundle\FrameworkBundle\Controller\Controller;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\Routing\Annotation\Route;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+
+class DeliciousController extends Controller
+{
+    /**
+     * @Route("/delicious", name="import_delicious")
+     */
+    public function indexAction(Request $request)
+    {
+        $form = $this->createForm(UploadImportType::class);
+        $form->handleRequest($request);
+
+        $delicious = $this->get('wallabag_import.delicious.import');
+        $delicious->setUser($this->getUser());
+
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+            $delicious->setProducer($this->get('old_sound_rabbit_mq.import_delicious_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $delicious->setProducer($this->get('wallabag_import.producer.redis.delicious'));
+        }
+
+        if ($form->isSubmitted() && $form->isValid()) {
+            $file = $form->get('file')->getData();
+            $markAsRead = $form->get('mark_as_read')->getData();
+            $name = 'delicious_' . $this->getUser()->getId() . '.json';
+
+            if (null !== $file && \in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes'), true) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+                $res = $delicious
+                    ->setFilepath($this->getParameter('wallabag_import.resource_dir') . '/' . $name)
+                    ->setMarkAsRead($markAsRead)
+                    ->import();
+
+                $message = 'flashes.import.notice.failed';
+
+                if (true === $res) {
+                    $summary = $delicious->getSummary();
+                    $message = $this->get('translator')->trans('flashes.import.notice.summary', [
+                        '%imported%' => $summary['imported'],
+                        '%skipped%' => $summary['skipped'],
+                    ]);
+
+                    if (0 < $summary['queued']) {
+                        $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
+                            '%queued%' => $summary['queued'],
+                        ]);
+                    }
+
+                    unlink($this->getParameter('wallabag_import.resource_dir') . '/' . $name);
+                }
+
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    $message
+                );
+
+                return $this->redirect($this->generateUrl('homepage'));
+            }
+
+            $this->get('session')->getFlashBag()->add(
+                'notice',
+                'flashes.import.notice.failed_on_file'
+            );
+        }
+
+        return $this->render('WallabagImportBundle:Delicious:index.html.twig', [
+            'form' => $form->createView(),
+            'import' => $delicious,
+        ]);
+    }
+}

src/Wallabag/ImportBundle/Controller/ImportController.php

@@ -43,6 +43,7 @@ class ImportController extends Controller
                     + $this->getTotalMessageInRabbitQueue('chrome')
                     + $this->getTotalMessageInRabbitQueue('instapaper')
                     + $this->getTotalMessageInRabbitQueue('pinboard')
+                    + $this->getTotalMessageInRabbitQueue('delicious')
                     + $this->getTotalMessageInRabbitQueue('elcurator')
                 ;
             } catch (\Exception $e) {
@@ -60,6 +61,7 @@ class ImportController extends Controller
                     + $redis->llen('wallabag.import.chrome')
                     + $redis->llen('wallabag.import.instapaper')
                     + $redis->llen('wallabag.import.pinboard')
+                    + $redis->llen('wallabag.import.delicious')
                     + $redis->llen('wallabag.import.elcurator')
                 ;
             } catch (\Exception $e) {

src/Wallabag/ImportBundle/Import/DeliciousImport.php

@@ -0,0 +1,151 @@
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Wallabag\CoreBundle\Entity\Entry;
+
+class DeliciousImport extends AbstractImport
+{
+    private $filepath;
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getName()
+    {
+        return 'Delicious';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getUrl()
+    {
+        return 'import_delicious';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDescription()
+    {
+        return 'import.delicious.description';
+    }
+
+    /**
+     * Set file path to the json file.
+     *
+     * @param string $filepath
+     */
+    public function setFilepath($filepath)
+    {
+        $this->filepath = $filepath;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function import()
+    {
+        if (!$this->user) {
+            $this->logger->error('DeliciousImport: user is not defined');
+
+            return false;
+        }
+
+        if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+            $this->logger->error('DeliciousImport: unable to read file', ['filepath' => $this->filepath]);
+
+            return false;
+        }
+
+        $data = json_decode(file_get_contents($this->filepath), true);
+
+        if (empty($data)) {
+            $this->logger->error('DeliciousImport: no entries in imported file');
+
+            return false;
+        }
+
+        if ($this->producer) {
+            $this->parseEntriesForProducer($data);
+
+            return true;
+        }
+
+        $this->parseEntries($data);
+
+        return true;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function validateEntry(array $importedEntry)
+    {
+        if (empty($importedEntry['url'])) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function parseEntry(array $importedEntry)
+    {
+        $existingEntry = $this->em
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
+
+        if (false !== $existingEntry) {
+            ++$this->skippedEntries;
+
+            return;
+        }
+
+        $data = [
+            'title' => $importedEntry['title'],
+            'url' => $importedEntry['url'],
+            'is_archived' => $this->markAsRead,
+            'is_starred' => false,
+            'created_at' => $importedEntry['created'],
+            'tags' => $importedEntry['tags'],
+        ];
+
+        $entry = new Entry($this->user);
+        $entry->setUrl($data['url']);
+        $entry->setTitle($data['title']);
+
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $this->fetchContent($entry, $data['url'], $data);
+
+        if (!empty($data['tags'])) {
+            $this->tagsAssigner->assignTagsToEntry(
+                $entry,
+                $data['tags'],
+                $this->em->getUnitOfWork()->getScheduledEntityInsertions()
+            );
+        }
+
+        $entry->updateArchived($data['is_archived']);
+        $entry->setStarred($data['is_starred']);
+        $entry->setCreatedAt(\DateTime::createFromFormat('U', $data['created_at']));
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
+
+        return $entry;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function setEntryAsRead(array $importedEntry)
+    {
+        return $importedEntry;
+    }
+}

src/Wallabag/ImportBundle/Resources/config/rabbit.yml

@@ -32,6 +32,14 @@ services:
             - "@wallabag_import.pinboard.import"
             - "@event_dispatcher"
             - "@logger"
+    wallabag_import.consumer.amqp.delicious:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.delicious.import"
+            - "@event_dispatcher"
+            - "@logger"
     wallabag_import.consumer.amqp.wallabag_v1:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
         arguments:

src/Wallabag/ImportBundle/Resources/config/redis.yml

@@ -63,6 +63,27 @@ services:
             - "@event_dispatcher"
             - "@logger"
 
+    # delicious
+    wallabag_import.queue.redis.delicious:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.delicious"
+
+    wallabag_import.producer.redis.delicious:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.delicious"
+
+    wallabag_import.consumer.redis.delicious:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.delicious.import"
+            - "@event_dispatcher"
+            - "@logger"
+
     # pocket
     wallabag_import.queue.redis.pocket:
         class: Simpleue\Queue\RedisQueue

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -96,6 +96,18 @@ services:
         tags:
             -  { name: wallabag_import.import, alias: pinboard }
 
+    wallabag_import.delicious.import:
+        class: Wallabag\ImportBundle\Import\DeliciousImport
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_core.content_proxy"
+            - "@wallabag_core.tags_assigner"
+            - "@event_dispatcher"
+        calls:
+            - [ setLogger, [ "@logger" ]]
+        tags:
+            -  { name: wallabag_import.import, alias: delicious }
+
     wallabag_import.firefox.import:
         class: Wallabag\ImportBundle\Import\FirefoxImport
         arguments:

src/Wallabag/ImportBundle/Resources/views/Delicious/index.html.twig

@@ -0,0 +1,45 @@
+{% extends "WallabagCoreBundle::layout.html.twig" %}
+
+{% block title %}{{ 'import.delicious.page_title'|trans }}{% endblock %}
+
+{% block content %}
+<div class="row">
+    <div class="col s12">
+        <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
+
+            <div class="row">
+                <blockquote>{{ import.description|trans }}</blockquote>
+                <p>{{ 'import.delicious.how_to'|trans }}</p>
+
+                <div class="col s12">
+                    {{ form_start(form, {'method': 'POST'}) }}
+                        {{ form_errors(form) }}
+                        <div class="row">
+                            <div class="file-field input-field col s12">
+                                {{ form_errors(form.file) }}
+                                <div class="btn">
+                                    <span>{{ form.file.vars.label|trans }}</span>
+                                    {{ form_widget(form.file) }}
+                                </div>
+                                <div class="file-path-wrapper">
+                                    <input class="file-path validate" type="text">
+                                </div>
+                            </div>
+                            <div class="input-field col s6 with-checkbox">
+                                <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
+                                {{ form_widget(form.mark_as_read) }}
+                                {{ form_label(form.mark_as_read) }}
+                            </div>
+                        </div>
+
+                        {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
+
+                        {{ form_rest(form) }}
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}