diff --git a/app/config/config.yml b/app/config/config.yml
index cd981163d..1a522ddbe 100644
--- a/app/config/config.yml
+++ b/app/config/config.yml
@@ -283,6 +283,16 @@ old_sound_rabbit_mq:
exchange_options:
name: 'wallabag.import.chrome'
type: topic
+ import_shaarli:
+ connection: default
+ exchange_options:
+ name: 'wallabag.import.shaarli'
+ type: topic
+ import_pocket_html:
+ connection: default
+ exchange_options:
+ name: 'wallabag.import.pocket_html'
+ type: topic
consumers:
import_pocket:
connection: default
@@ -383,6 +393,24 @@ old_sound_rabbit_mq:
name: 'wallabag.import.chrome'
callback: wallabag_import.consumer.amqp.chrome
qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"}
+ import_shaarli:
+ connection: default
+ exchange_options:
+ name: 'wallabag.import.shaarli'
+ type: topic
+ queue_options:
+ name: 'wallabag.import.shaarli'
+ callback: wallabag_import.consumer.amqp.shaarli
+ qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"}
+ import_pocket_html:
+ connection: default
+ exchange_options:
+ name: 'wallabag.import.pocket_html'
+ type: topic
+ queue_options:
+ name: 'wallabag.import.pocket_html'
+ callback: wallabag_import.consumer.amqp.pocket_html
+ qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"}
fos_js_routing:
routes_to_expose:
diff --git a/app/config/services.yml b/app/config/services.yml
index 47f9b3aef..cc7c5329c 100644
--- a/app/config/services.yml
+++ b/app/config/services.yml
@@ -121,6 +121,16 @@ services:
$rabbitMqProducer: '@old_sound_rabbit_mq.import_wallabag_v2_producer'
$redisProducer: '@wallabag_import.producer.redis.wallabag_v2'
+ Wallabag\ImportBundle\Controller\ShaarliController:
+ arguments:
+ $rabbitMqProducer: '@old_sound_rabbit_mq.import_shaarli_producer'
+ $redisProducer: '@wallabag_import.producer.redis.shaarli'
+
+ Wallabag\ImportBundle\Controller\PocketHtmlController:
+ arguments:
+ $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_html_producer'
+ $redisProducer: '@wallabag_import.producer.redis.pocket_html'
+
Wallabag\ImportBundle\:
resource: '../../src/Wallabag/ImportBundle/*'
exclude: '../../src/Wallabag/ImportBundle/{Consumer,Controller,Redis}'
@@ -394,6 +404,14 @@ services:
tags:
- { name: wallabag_import.import, alias: chrome }
+ Wallabag\ImportBundle\Import\ShaarliImport:
+ tags:
+ - { name: wallabag_import.import, alias: shaarli }
+
+ Wallabag\ImportBundle\Import\PocketHtmlImport:
+ tags:
+ - { name: wallabag_import.import, alias: pocket_html }
+
# to factorize the proximity and bypass translation for prev & next
pagerfanta.view.default_wallabag:
class: Pagerfanta\View\OptionableView
diff --git a/app/config/services_rabbit.yml b/app/config/services_rabbit.yml
index b6f9f1d52..e6963733a 100644
--- a/app/config/services_rabbit.yml
+++ b/app/config/services_rabbit.yml
@@ -19,6 +19,8 @@ services:
$deliciousConsumer: '@old_sound_rabbit_mq.import_delicious_consumer'
$elcuratorConsumer: '@old_sound_rabbit_mq.import_elcurator_consumer'
$omnivoreConsumer: '@old_sound_rabbit_mq.import_omnivore_consumer'
+ $shaarliConsumer: '@old_sound_rabbit_mq.import_shaarli_consumer'
+ $pocketHtmlConsumer: '@old_sound_rabbit_mq.import_pocket_html_consumer'
wallabag_import.consumer.amqp.pocket:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -74,3 +76,13 @@ services:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
$import: '@Wallabag\ImportBundle\Import\ChromeImport'
+
+ wallabag_import.consumer.amqp.shaarli:
+ class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+ arguments:
+ $import: '@Wallabag\ImportBundle\Import\ShaarliImport'
+
+ wallabag_import.consumer.amqp.pocket_html:
+ class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+ arguments:
+ $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport'
diff --git a/app/config/services_redis.yml b/app/config/services_redis.yml
index 2f0f1ac48..baeb1ccef 100644
--- a/app/config/services_redis.yml
+++ b/app/config/services_redis.yml
@@ -180,3 +180,35 @@ services:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
$import: '@Wallabag\ImportBundle\Import\ChromeImport'
+
+ # shaarli
+ wallabag_import.queue.redis.shaarli:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ $queueName: "wallabag.import.shaarli"
+
+ wallabag_import.producer.redis.shaarli:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.shaarli"
+
+ wallabag_import.consumer.redis.shaarli:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ $import: '@Wallabag\ImportBundle\Import\ShaarliImport'
+
+ # pocket html
+ wallabag_import.queue.redis.pocket_html:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ $queueName: "wallabag.import.pocket_html"
+
+ wallabag_import.producer.redis.pocket_html:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.pocket_html"
+
+ wallabag_import.consumer.redis.pocket_html:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport'
diff --git a/app/config/wallabag.yml b/app/config/wallabag.yml
index 09d12619b..2c5e53a00 100644
--- a/app/config/wallabag.yml
+++ b/app/config/wallabag.yml
@@ -167,5 +167,5 @@ wallabag_core:
rule: _all ~ "https?://www\.lemonde\.fr/tiny.*"
wallabag_import:
- allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv']
+ allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv', 'text/html']
resource_dir: "%kernel.project_dir%/web/uploads/import"
diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon
index ae5029a61..1e423b482 100644
--- a/phpstan-baseline.neon
+++ b/phpstan-baseline.neon
@@ -59,3 +59,13 @@ parameters:
message: "#^Method FOS\\\\UserBundle\\\\Model\\\\UserManagerInterface\\:\\:updateUser()#"
count: 7
path: src/Wallabag/CoreBundle/Controller/ConfigController.php
+
+ -
+ message: "#^Call to an undefined method Wallabag\\\\ImportBundle\\\\Import\\\\ImportInterface\\:\\:setUser\\(\\)\\.$#"
+ count: 1
+ path: src/Wallabag/ImportBundle/Controller/HtmlController.php
+
+ -
+ message: "#^Call to an undefined method Wallabag\\\\ImportBundle\\\\Import\\\\ImportInterface\\:\\:setFilepath\\(\\)\\.$#"
+ count: 1
+ path: src/Wallabag/ImportBundle/Controller/HtmlController.php
diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php
index edc85c01d..76bd4f451 100644
--- a/src/Wallabag/ImportBundle/Command/ImportCommand.php
+++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php
@@ -13,11 +13,14 @@ use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInt
use Symfony\Component\Security\Core\Authentication\Token\UsernamePasswordToken;
use Wallabag\ImportBundle\Import\ChromeImport;
use Wallabag\ImportBundle\Import\DeliciousImport;
+use Wallabag\ImportBundle\Import\ElcuratorImport;
use Wallabag\ImportBundle\Import\FirefoxImport;
use Wallabag\ImportBundle\Import\InstapaperImport;
use Wallabag\ImportBundle\Import\OmnivoreImport;
use Wallabag\ImportBundle\Import\PinboardImport;
+use Wallabag\ImportBundle\Import\PocketHtmlImport;
use Wallabag\ImportBundle\Import\ReadabilityImport;
+use Wallabag\ImportBundle\Import\ShaarliImport;
use Wallabag\ImportBundle\Import\WallabagV1Import;
use Wallabag\ImportBundle\Import\WallabagV2Import;
use Wallabag\UserBundle\Entity\User;
@@ -37,9 +40,27 @@ class ImportCommand extends Command
private DeliciousImport $deliciousImport;
private OmnivoreImport $omnivoreImport;
private WallabagV1Import $wallabagV1Import;
+ private ElcuratorImport $elcuratorImport;
+ private ShaarliImport $shaarliImport;
+ private PocketHtmlImport $pocketHtmlImport;
- public function __construct(EntityManagerInterface $entityManager, TokenStorageInterface $tokenStorage, UserRepository $userRepository, WallabagV2Import $wallabagV2Import, FirefoxImport $firefoxImport, ChromeImport $chromeImport, ReadabilityImport $readabilityImport, InstapaperImport $instapaperImport, PinboardImport $pinboardImport, DeliciousImport $deliciousImport, OmnivoreImport $omnivoreImport, WallabagV1Import $wallabagV1Import)
- {
+ public function __construct(
+ EntityManagerInterface $entityManager,
+ TokenStorageInterface $tokenStorage,
+ UserRepository $userRepository,
+ WallabagV2Import $wallabagV2Import,
+ FirefoxImport $firefoxImport,
+ ChromeImport $chromeImport,
+ ReadabilityImport $readabilityImport,
+ InstapaperImport $instapaperImport,
+ PinboardImport $pinboardImport,
+ DeliciousImport $deliciousImport,
+ WallabagV1Import $wallabagV1Import,
+ ElcuratorImport $elcuratorImport,
+ OmnivoreImport $omnivoreImport,
+ ShaarliImport $shaarliImport,
+ PocketHtmlImport $pocketHtmlImport
+ ) {
$this->entityManager = $entityManager;
$this->tokenStorage = $tokenStorage;
$this->userRepository = $userRepository;
@@ -52,6 +73,9 @@ class ImportCommand extends Command
$this->deliciousImport = $deliciousImport;
$this->omnivoreImport = $omnivoreImport;
$this->wallabagV1Import = $wallabagV1Import;
+ $this->elcuratorImport = $elcuratorImport;
+ $this->shaarliImport = $shaarliImport;
+ $this->pocketHtmlImport = $pocketHtmlImport;
parent::__construct();
}
@@ -63,7 +87,7 @@ class ImportCommand extends Command
->setDescription('Import entries from a JSON export')
->addArgument('username', InputArgument::REQUIRED, 'User to populate')
->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file')
- ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox or chrome', 'v1')
+ ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli or pocket', 'v1')
->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false)
->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account')
->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL')
@@ -125,6 +149,15 @@ class ImportCommand extends Command
break;
case 'omnivore':
$import = $this->omnivoreImport;
+ break;
+ case 'elcurator':
+ $import = $this->elcuratorImport;
+ break;
+ case 'shaarli':
+ $import = $this->shaarliImport;
+ break;
+ case 'pocket':
+ $import = $this->pocketHtmlImport;
break;
default:
$import = $this->wallabagV1Import;
diff --git a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php
index 096d2b9a6..0d531d884 100644
--- a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php
+++ b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php
@@ -21,9 +21,24 @@ class RabbitMQConsumerTotalProxy
private Consumer $deliciousConsumer;
private Consumer $elcuratorConsumer;
private Consumer $omnivoreConsumer;
+ private Consumer $shaarliConsumer;
+ private Consumer $pocketHtmlConsumer;
- public function __construct(Consumer $pocketConsumer, Consumer $readabilityConsumer, Consumer $wallabagV1Consumer, Consumer $wallabagV2Consumer, Consumer $firefoxConsumer, Consumer $chromeConsumer, Consumer $instapaperConsumer, Consumer $pinboardConsumer, Consumer $deliciousConsumer, Consumer $elcuratorConsumer, Consumer $omnivoreConsumer)
- {
+ public function __construct(
+ Consumer $pocketConsumer,
+ Consumer $readabilityConsumer,
+ Consumer $wallabagV1Consumer,
+ Consumer $wallabagV2Consumer,
+ Consumer $firefoxConsumer,
+ Consumer $chromeConsumer,
+ Consumer $instapaperConsumer,
+ Consumer $pinboardConsumer,
+ Consumer $deliciousConsumer,
+ Consumer $elcuratorConsumer,
+ Consumer $omnivoreConsumer,
+ Consumer $shaarliConsumer,
+ Consumer $pocketHtmlConsumer
+ ) {
$this->pocketConsumer = $pocketConsumer;
$this->readabilityConsumer = $readabilityConsumer;
$this->wallabagV1Consumer = $wallabagV1Consumer;
@@ -35,6 +50,8 @@ class RabbitMQConsumerTotalProxy
$this->deliciousConsumer = $deliciousConsumer;
$this->elcuratorConsumer = $elcuratorConsumer;
$this->omnivoreConsumer = $omnivoreConsumer;
+ $this->shaarliConsumer = $shaarliConsumer;
+ $this->pocketHtmlConsumer = $pocketHtmlConsumer;
}
/**
@@ -82,6 +99,12 @@ class RabbitMQConsumerTotalProxy
case 'omnivore':
$consumer = $this->omnivoreConsumer;
break;
+ case 'shaarli':
+ $consumer = $this->shaarliConsumer;
+ break;
+ case 'pocket_html':
+ $consumer = $this->pocketHtmlConsumer;
+ break;
default:
return 0;
}
diff --git a/src/Wallabag/ImportBundle/Controller/HtmlController.php b/src/Wallabag/ImportBundle/Controller/HtmlController.php
new file mode 100644
index 000000000..e9515561f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/HtmlController.php
@@ -0,0 +1,83 @@
+createForm(UploadImportType::class);
+ $form->handleRequest($request);
+
+ $wallabag = $this->getImportService();
+ $wallabag->setUser($this->getUser());
+
+ if ($form->isSubmitted() && $form->isValid()) {
+ $file = $form->get('file')->getData();
+ $markAsRead = $form->get('mark_as_read')->getData();
+ $name = $this->getUser()->getId() . '.html';
+
+ if (null !== $file && \in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes'), true) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+ $res = $wallabag
+ ->setFilepath($this->getParameter('wallabag_import.resource_dir') . '/' . $name)
+ ->setMarkAsRead($markAsRead)
+ ->import();
+
+ $message = 'flashes.import.notice.failed';
+
+ if (true === $res) {
+ $summary = $wallabag->getSummary();
+ $message = $translator->trans('flashes.import.notice.summary', [
+ '%imported%' => $summary['imported'],
+ '%skipped%' => $summary['skipped'],
+ ]);
+
+ if (0 < $summary['queued']) {
+ $message = $translator->trans('flashes.import.notice.summary_with_queue', [
+ '%queued%' => $summary['queued'],
+ ]);
+ }
+
+ unlink($this->getParameter('wallabag_import.resource_dir') . '/' . $name);
+ }
+
+ $this->addFlash('notice', $message);
+
+ return $this->redirect($this->generateUrl('homepage'));
+ }
+ $this->addFlash('notice', 'flashes.import.notice.failed_on_file');
+ }
+
+ return $this->render($this->getImportTemplate(), [
+ 'form' => $form->createView(),
+ 'import' => $wallabag,
+ ]);
+ }
+
+ /**
+ * Return the service to handle the import.
+ *
+ * @return ImportInterface
+ */
+ abstract protected function getImportService();
+
+ /**
+ * Return the template used for the form.
+ *
+ * @return string
+ */
+ abstract protected function getImportTemplate();
+}
diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php
index 16414d885..6e6e5d7c1 100644
--- a/src/Wallabag/ImportBundle/Controller/ImportController.php
+++ b/src/Wallabag/ImportBundle/Controller/ImportController.php
@@ -58,6 +58,8 @@ class ImportController extends AbstractController
+ $this->rabbitMQConsumerTotalProxy->getTotalMessage('delicious')
+ $this->rabbitMQConsumerTotalProxy->getTotalMessage('elcurator')
+ $this->rabbitMQConsumerTotalProxy->getTotalMessage('omnivore')
+ + $this->rabbitMQConsumerTotalProxy->getTotalMessage('shaarli')
+ + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_html')
;
} catch (\Exception $e) {
$rabbitNotInstalled = true;
@@ -77,6 +79,8 @@ class ImportController extends AbstractController
+ $redis->llen('wallabag.import.delicious')
+ $redis->llen('wallabag.import.elcurator')
+ $redis->llen('wallabag.import.omnivore')
+ + $redis->llen('wallabag.import.shaarli')
+ + $redis->llen('wallabag.import.pocket_html')
;
} catch (\Exception $e) {
$redisNotInstalled = true;
diff --git a/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php b/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php
new file mode 100644
index 000000000..7387bbfdc
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php
@@ -0,0 +1,57 @@
+pocketHtmlImport = $pocketHtmlImport;
+ $this->craueConfig = $craueConfig;
+ $this->rabbitMqProducer = $rabbitMqProducer;
+ $this->redisProducer = $redisProducer;
+ }
+
+ /**
+ * @Route("/pocket_html", name="import_pocket_html")
+ */
+ public function indexAction(Request $request, TranslatorInterface $translator)
+ {
+ return parent::indexAction($request, $translator);
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function getImportService()
+ {
+ if ($this->craueConfig->get('import_with_rabbitmq')) {
+ $this->pocketHtmlImport->setProducer($this->rabbitMqProducer);
+ } elseif ($this->craueConfig->get('import_with_redis')) {
+ $this->pocketHtmlImport->setProducer($this->redisProducer);
+ }
+
+ return $this->pocketHtmlImport;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function getImportTemplate()
+ {
+ return '@WallabagImport/PocketHtml/index.html.twig';
+ }
+}
diff --git a/src/Wallabag/ImportBundle/Controller/ShaarliController.php b/src/Wallabag/ImportBundle/Controller/ShaarliController.php
new file mode 100644
index 000000000..46dfd1473
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/ShaarliController.php
@@ -0,0 +1,57 @@
+shaarliImport = $shaarliImport;
+ $this->craueConfig = $craueConfig;
+ $this->rabbitMqProducer = $rabbitMqProducer;
+ $this->redisProducer = $redisProducer;
+ }
+
+ /**
+ * @Route("/shaarli", name="import_shaarli")
+ */
+ public function indexAction(Request $request, TranslatorInterface $translator)
+ {
+ return parent::indexAction($request, $translator);
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function getImportService()
+ {
+ if ($this->craueConfig->get('import_with_rabbitmq')) {
+ $this->shaarliImport->setProducer($this->rabbitMqProducer);
+ } elseif ($this->craueConfig->get('import_with_redis')) {
+ $this->shaarliImport->setProducer($this->redisProducer);
+ }
+
+ return $this->shaarliImport;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function getImportTemplate()
+ {
+ return '@WallabagImport/Shaarli/index.html.twig';
+ }
+}
diff --git a/src/Wallabag/ImportBundle/Import/HtmlImport.php b/src/Wallabag/ImportBundle/Import/HtmlImport.php
new file mode 100644
index 000000000..a96107e0b
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/HtmlImport.php
@@ -0,0 +1,210 @@
+user) {
+ $this->logger->error('Wallabag HTML Import: user is not defined');
+
+ return false;
+ }
+
+ if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+ $this->logger->error('Wallabag HTML Import: unable to read file', ['filepath' => $this->filepath]);
+
+ return false;
+ }
+
+ $html = new \DOMDocument();
+
+ libxml_use_internal_errors(true);
+ $html->loadHTMLFile($this->filepath);
+ $hrefs = $html->getElementsByTagName('a');
+ libxml_use_internal_errors(false);
+
+ if (0 === $hrefs->length) {
+ $this->logger->error('Wallabag HTML: no entries in imported file');
+
+ return false;
+ }
+
+ $entries = [];
+ foreach ($hrefs as $href) {
+ $entry = [];
+ $entry['url'] = $href->getAttribute('href');
+ $entry['tags'] = $href->getAttribute('tags');
+ $entry['created_at'] = $href->getAttribute('add_date');
+ $entries[] = $entry;
+ }
+
+ if ($this->producer) {
+ $this->parseEntriesForProducer($entries);
+
+ return true;
+ }
+
+ $this->parseEntries($entries);
+
+ return true;
+ }
+
+ /**
+ * Set file path to the html file.
+ *
+ * @param string $filepath
+ */
+ public function setFilepath($filepath)
+ {
+ $this->filepath = $filepath;
+
+ return $this;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public function parseEntry(array $importedEntry)
+ {
+ $url = $importedEntry['url'];
+
+ $existingEntry = $this->em
+ ->getRepository(Entry::class)
+ ->findByUrlAndUserId($url, $this->user->getId());
+
+ if (false !== $existingEntry) {
+ ++$this->skippedEntries;
+
+ return null;
+ }
+
+ $data = $this->prepareEntry($importedEntry);
+
+ $entry = new Entry($this->user);
+ $entry->setUrl($data['url']);
+ $entry->updateArchived($data['is_archived']);
+ $createdAt = new \DateTime();
+ $createdAt->setTimestamp($data['created_at']);
+ $entry->setCreatedAt($createdAt);
+
+ // update entry with content (in case fetching failed, the given entry will be return)
+ $this->fetchContent($entry, $data['url'], $data);
+
+ if (\array_key_exists('tags', $data)) {
+ $this->tagsAssigner->assignTagsToEntry(
+ $entry,
+ $data['tags']
+ );
+ }
+
+ $this->em->persist($entry);
+ ++$this->importedEntries;
+
+ return $entry;
+ }
+
+ /**
+ * Parse and insert all given entries.
+ */
+ protected function parseEntries(array $entries)
+ {
+ $i = 1;
+ $entryToBeFlushed = [];
+
+ foreach ($entries as $importedEntry) {
+ $entry = $this->parseEntry($importedEntry);
+
+ if (null === $entry) {
+ continue;
+ }
+
+ // @see AbstractImport
+ $entryToBeFlushed[] = $entry;
+
+ // flush every 20 entries
+ if (0 === ($i % 20)) {
+ $this->em->flush();
+
+ foreach ($entryToBeFlushed as $entry) {
+ $this->eventDispatcher->dispatch(new EntrySavedEvent($entry), EntrySavedEvent::NAME);
+ }
+
+ $entryToBeFlushed = [];
+ }
+ ++$i;
+ }
+
+ $this->em->flush();
+
+ if (!empty($entryToBeFlushed)) {
+ foreach ($entryToBeFlushed as $entry) {
+ $this->eventDispatcher->dispatch(new EntrySavedEvent($entry), EntrySavedEvent::NAME);
+ }
+ }
+ }
+
+ /**
+ * Parse entries and send them to the queue.
+ * It should just be a simple loop on all item, no call to the database should be done
+ * to speedup queuing.
+ *
+ * Faster parse entries for Producer.
+ * We don't care to make check at this time. They'll be done by the consumer.
+ */
+ protected function parseEntriesForProducer(array $entries)
+ {
+ foreach ($entries as $importedEntry) {
+ if ((array) $importedEntry !== $importedEntry) {
+ continue;
+ }
+
+ // set userId for the producer (it won't know which user is connected)
+ $importedEntry['userId'] = $this->user->getId();
+
+ if ($this->markAsRead) {
+ $importedEntry = $this->setEntryAsRead($importedEntry);
+ }
+
+ ++$this->queuedEntries;
+
+ $this->producer->publish(json_encode($importedEntry));
+ }
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function setEntryAsRead(array $importedEntry)
+ {
+ $importedEntry['is_archived'] = 1;
+
+ return $importedEntry;
+ }
+
+ abstract protected function prepareEntry(array $entry = []);
+}
diff --git a/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php b/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php
new file mode 100644
index 000000000..492a1adfc
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php
@@ -0,0 +1,113 @@
+user) {
+ $this->logger->error('Pocket HTML Import: user is not defined');
+
+ return false;
+ }
+
+ if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+ $this->logger->error('Pocket HTML Import: unable to read file', ['filepath' => $this->filepath]);
+
+ return false;
+ }
+
+ $html = new \DOMDocument();
+
+ libxml_use_internal_errors(true);
+ $html->loadHTMLFile($this->filepath);
+ $hrefs = $html->getElementsByTagName('a');
+ libxml_use_internal_errors(false);
+
+ if (0 === $hrefs->length) {
+ $this->logger->error('Pocket HTML: no entries in imported file');
+
+ return false;
+ }
+
+ $entries = [];
+ foreach ($hrefs as $href) {
+ $entry = [];
+ $entry['url'] = $href->getAttribute('href');
+ $entry['tags'] = $href->getAttribute('tags');
+ $entry['created_at'] = $href->getAttribute('time_added');
+ $entries[] = $entry;
+ }
+
+ if ($this->producer) {
+ $this->parseEntriesForProducer($entries);
+
+ return true;
+ }
+
+ $this->parseEntries($entries);
+
+ return true;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function prepareEntry(array $entry = [])
+ {
+ $data = [
+ 'title' => '',
+ 'html' => false,
+ 'url' => $entry['url'],
+ 'is_archived' => (int) $this->markAsRead,
+ 'is_starred' => false,
+ 'tags' => '',
+ 'created_at' => $entry['created_at'],
+ ];
+
+ if (\array_key_exists('tags', $entry) && '' !== $entry['tags']) {
+ $data['tags'] = $entry['tags'];
+ }
+
+ return $data;
+ }
+}
diff --git a/src/Wallabag/ImportBundle/Import/ShaarliImport.php b/src/Wallabag/ImportBundle/Import/ShaarliImport.php
new file mode 100644
index 000000000..b4c9dc3c3
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/ShaarliImport.php
@@ -0,0 +1,66 @@
+ '',
+ 'html' => false,
+ 'url' => $entry['url'],
+ 'is_archived' => (int) $this->markAsRead,
+ 'is_starred' => false,
+ 'tags' => '',
+ 'created_at' => $entry['created_at'],
+ ];
+
+ if (\array_key_exists('tags', $entry) && '' !== $entry['tags']) {
+ $data['tags'] = $entry['tags'];
+ }
+
+ return $data;
+ }
+}
diff --git a/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig
new file mode 100644
index 000000000..09f2e689f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig
@@ -0,0 +1,45 @@
+{% extends "@WallabagCore/layout.html.twig" %}
+
+{% block title %}{{ 'import.pocket_html.page_title'|trans }}{% endblock %}
+
+{% block content %}
+
+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %}
+
+
+
{{ import.description|trans|raw }}
+
{{ 'import.pocket_html.how_to'|trans }}
+
+
+ {{ form_start(form, {'method': 'POST'}) }}
+ {{ form_errors(form) }}
+
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }}
+ {{ form_label(form.mark_as_read) }}
+
+
+
+ {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }}
+
+ {{ form_rest(form) }}
+
+
+
+
+
+
+{% endblock %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig
new file mode 100644
index 000000000..edb24e468
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig
@@ -0,0 +1,45 @@
+{% extends "@WallabagCore/layout.html.twig" %}
+
+{% block title %}{{ 'import.shaarli.page_title'|trans }}{% endblock %}
+
+{% block content %}
+
+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %}
+
+
+
{{ import.description|trans|raw }}
+
{{ 'import.shaarli.how_to'|trans }}
+
+
+ {{ form_start(form, {'method': 'POST'}) }}
+ {{ form_errors(form) }}
+
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }}
+ {{ form_label(form.mark_as_read) }}
+
+
+
+ {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }}
+
+ {{ form_rest(form) }}
+
+
+
+
+
+
+{% endblock %}
diff --git a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php
index f6dfff4c8..01a7c2d09 100644
--- a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php
+++ b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php
@@ -123,9 +123,9 @@ class FirefoxControllerTest extends WallabagCoreTestCase
);
$this->assertInstanceOf(Entry::class, $content);
- $this->assertNotEmpty($content->getMimetype(), 'Mimetype for http://lexpansion.lexpress.fr is ok');
- $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for http://lexpansion.lexpress.fr is ok');
- $this->assertNotEmpty($content->getLanguage(), 'Language for http://lexpansion.lexpress.fr is ok');
+ $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok');
+ $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok');
+ $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok');
$this->assertCount(3, $content->getTags());
}
diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php
index cf1739841..76b802b6e 100644
--- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php
+++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php
@@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase
$crawler = $client->request('GET', '/import/');
$this->assertSame(200, $client->getResponse()->getStatusCode());
- $this->assertSame(11, $crawler->filter('blockquote')->count());
+ $this->assertSame(13, $crawler->filter('blockquote')->count());
}
}
diff --git a/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php
new file mode 100644
index 000000000..7ae4a247f
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php
@@ -0,0 +1,168 @@
+logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/pocket_html');
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertSame(1, $crawler->filter('input[type=file]')->count());
+ }
+
+ public function testImportPocketHtmlWithRabbitEnabled()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1);
+
+ $crawler = $client->request('GET', '/import/pocket_html');
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertSame(1, $crawler->filter('input[type=file]')->count());
+
+ $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0);
+ }
+
+ public function testImportPocketHtmlBadFile()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/pocket_html');
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $data = [
+ 'upload_import_file[file]' => '',
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ }
+
+ public function testImportPocketHtmlWithRedisEnabled()
+ {
+ $this->checkRedis();
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+ $client->getContainer()->get(Config::class)->set('import_with_redis', 1);
+
+ $crawler = $client->request('GET', '/import/pocket_html');
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertSame(1, $crawler->filter('input[type=file]')->count());
+
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $file = new UploadedFile(__DIR__ . '/../fixtures/ril_export.html', 'Bookmarks');
+
+ $data = [
+ 'upload_import_file[file]' => $file,
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(302, $client->getResponse()->getStatusCode());
+
+ $crawler = $client->followRedirect();
+
+ $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+ $this->assertStringContainsString('flashes.import.notice.summary', $body[0]);
+
+ $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.pocket_html'));
+
+ $client->getContainer()->get(Config::class)->set('import_with_redis', 0);
+ }
+
+ public function testImportWallabagWithPocketHtmlFile()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/pocket_html');
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $file = new UploadedFile(__DIR__ . '/../fixtures/ril_export.html', 'Bookmarks');
+
+ $data = [
+ 'upload_import_file[file]' => $file,
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(302, $client->getResponse()->getStatusCode());
+
+ $crawler = $client->followRedirect();
+
+ $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+ $this->assertStringContainsString('flashes.import.notice.summary', $body[0]);
+
+ $content = $client->getContainer()
+ ->get(EntityManagerInterface::class)
+ ->getRepository(Entry::class)
+ ->findByUrlAndUserId(
+ 'https://www.20minutes.fr/sport/4002755-20220928-tarn-lapins-ravagent-terrain-match-rugby-doit-etre-annule',
+ $this->getLoggedInUserId()
+ );
+
+ $this->assertInstanceOf(Entry::class, $content);
+ $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok');
+ $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok');
+ $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok');
+ $this->assertCount(3, $content->getTags());
+
+ $content = $client->getContainer()
+ ->get(EntityManagerInterface::class)
+ ->getRepository(Entry::class)
+ ->findByUrlAndUserId(
+ 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html',
+ $this->getLoggedInUserId()
+ );
+
+ $this->assertInstanceOf(Entry::class, $content);
+ $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok');
+ $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok');
+ $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok');
+ }
+
+ public function testImportWallabagWithEmptyFile()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/pocket_html');
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $file = new UploadedFile(__DIR__ . '/../fixtures/test.html', 'test.html');
+
+ $data = [
+ 'upload_import_file[file]' => $file,
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(302, $client->getResponse()->getStatusCode());
+
+ $crawler = $client->followRedirect();
+
+ $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+ $this->assertStringContainsString('flashes.import.notice.failed', $body[0]);
+ }
+}
diff --git a/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php
new file mode 100644
index 000000000..8bc9ffb9d
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php
@@ -0,0 +1,168 @@
+logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/shaarli');
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertSame(1, $crawler->filter('input[type=file]')->count());
+ }
+
+ public function testImportShaarliWithRabbitEnabled()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1);
+
+ $crawler = $client->request('GET', '/import/shaarli');
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertSame(1, $crawler->filter('input[type=file]')->count());
+
+ $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0);
+ }
+
+ public function testImportShaarliBadFile()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/shaarli');
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $data = [
+ 'upload_import_file[file]' => '',
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ }
+
+ public function testImportShaarliWithRedisEnabled()
+ {
+ $this->checkRedis();
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+ $client->getContainer()->get(Config::class)->set('import_with_redis', 1);
+
+ $crawler = $client->request('GET', '/import/shaarli');
+
+ $this->assertSame(200, $client->getResponse()->getStatusCode());
+ $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertSame(1, $crawler->filter('input[type=file]')->count());
+
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $file = new UploadedFile(__DIR__ . '/../fixtures/shaarli-bookmarks.html', 'Bookmarks');
+
+ $data = [
+ 'upload_import_file[file]' => $file,
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(302, $client->getResponse()->getStatusCode());
+
+ $crawler = $client->followRedirect();
+
+ $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+ $this->assertStringContainsString('flashes.import.notice.summary', $body[0]);
+
+ $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.shaarli'));
+
+ $client->getContainer()->get(Config::class)->set('import_with_redis', 0);
+ }
+
+ public function testImportWallabagWithShaarliFile()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/shaarli');
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $file = new UploadedFile(__DIR__ . '/../fixtures/shaarli-bookmarks.html', 'Bookmarks');
+
+ $data = [
+ 'upload_import_file[file]' => $file,
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(302, $client->getResponse()->getStatusCode());
+
+ $crawler = $client->followRedirect();
+
+ $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+ $this->assertStringContainsString('flashes.import.notice.summary', $body[0]);
+
+ $content = $client->getContainer()
+ ->get(EntityManagerInterface::class)
+ ->getRepository(Entry::class)
+ ->findByUrlAndUserId(
+ 'https://www.20minutes.fr/sport/4002755-20220928-tarn-lapins-ravagent-terrain-match-rugby-doit-etre-annule',
+ $this->getLoggedInUserId()
+ );
+
+ $this->assertInstanceOf(Entry::class, $content);
+ $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok');
+ $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok');
+ $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok');
+ $this->assertCount(2, $content->getTags());
+
+ $content = $client->getContainer()
+ ->get(EntityManagerInterface::class)
+ ->getRepository(Entry::class)
+ ->findByUrlAndUserId(
+ 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html',
+ $this->getLoggedInUserId()
+ );
+
+ $this->assertInstanceOf(Entry::class, $content);
+ $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok');
+ $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok');
+ $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok');
+ }
+
+ public function testImportWallabagWithEmptyFile()
+ {
+ $this->logInAs('admin');
+ $client = $this->getTestClient();
+
+ $crawler = $client->request('GET', '/import/shaarli');
+ $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+ $file = new UploadedFile(__DIR__ . '/../fixtures/test.html', 'test.html');
+
+ $data = [
+ 'upload_import_file[file]' => $file,
+ ];
+
+ $client->submit($form, $data);
+
+ $this->assertSame(302, $client->getResponse()->getStatusCode());
+
+ $crawler = $client->followRedirect();
+
+ $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
+ $this->assertStringContainsString('flashes.import.notice.failed', $body[0]);
+ }
+}
diff --git a/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php
new file mode 100644
index 000000000..6ff5e13a0
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php
@@ -0,0 +1,254 @@
+getPocketHtmlImport();
+
+ $this->assertSame('Pocket HTML', $pocketHtmlImport->getName());
+ $this->assertNotEmpty($pocketHtmlImport->getUrl());
+ $this->assertSame('import.pocket_html.description', $pocketHtmlImport->getDescription());
+ }
+
+ public function testImport()
+ {
+ $pocketHtmlImport = $this->getPocketHtmlImport(false, 2);
+ $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->exactly(2))
+ ->method('findByUrlAndUserId')
+ ->willReturn(false);
+
+ $this->em
+ ->expects($this->any())
+ ->method('getRepository')
+ ->willReturn($entryRepo);
+
+ $entry = $this->getMockBuilder(Entry::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->exactly(2))
+ ->method('updateEntry')
+ ->willReturn($entry);
+
+ $res = $pocketHtmlImport->import();
+
+ $this->assertTrue($res);
+ $this->assertSame(['skipped' => 0, 'imported' => 2, 'queued' => 0], $pocketHtmlImport->getSummary());
+ }
+
+ public function testImportAndMarkAllAsRead()
+ {
+ $pocketHtmlImport = $this->getPocketHtmlImport(false, 1);
+ $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->exactly(2))
+ ->method('findByUrlAndUserId')
+ ->will($this->onConsecutiveCalls(false, true));
+
+ $this->em
+ ->expects($this->any())
+ ->method('getRepository')
+ ->willReturn($entryRepo);
+
+ $this->contentProxy
+ ->expects($this->exactly(1))
+ ->method('updateEntry')
+ ->willReturn(new Entry($this->user));
+
+ // check that every entry persisted are archived
+ $this->em
+ ->expects($this->any())
+ ->method('persist')
+ ->with($this->callback(function ($persistedEntry) {
+ return (bool) $persistedEntry->isArchived();
+ }));
+
+ $res = $pocketHtmlImport
+ ->setMarkAsRead(true)
+ ->import();
+
+ $this->assertTrue($res);
+
+ $this->assertSame(['skipped' => 1, 'imported' => 1, 'queued' => 0], $pocketHtmlImport->getSummary());
+ }
+
+ public function testImportWithRabbit()
+ {
+ $pocketHtmlImport = $this->getPocketHtmlImport();
+ $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder(Entry::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $producer
+ ->expects($this->exactly(2))
+ ->method('publish');
+
+ $pocketHtmlImport->setProducer($producer);
+
+ $res = $pocketHtmlImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $pocketHtmlImport->getSummary());
+ }
+
+ public function testImportWithRedis()
+ {
+ $pocketHtmlImport = $this->getPocketHtmlImport();
+ $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder(Entry::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter(Client::class, true);
+
+ $queue = new RedisQueue($redisMock, 'pocket_html');
+ $producer = new Producer($queue);
+
+ $pocketHtmlImport->setProducer($producer);
+
+ $res = $pocketHtmlImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $pocketHtmlImport->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('pocket_html'));
+ }
+
+ public function testImportBadFile()
+ {
+ $pocketHtmlImport = $this->getPocketHtmlImport();
+ $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/wallabag-v1.jsonx');
+
+ $res = $pocketHtmlImport->import();
+
+ $this->assertFalse($res);
+
+ $records = $this->logHandler->getRecords();
+ $this->assertStringContainsString('Pocket HTML Import: unable to read file', $records[0]['message']);
+ $this->assertSame('ERROR', $records[0]['level_name']);
+ }
+
+ public function testImportUserNotDefined()
+ {
+ $pocketHtmlImport = $this->getPocketHtmlImport(true);
+ $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html');
+
+ $res = $pocketHtmlImport->import();
+
+ $this->assertFalse($res);
+
+ $records = $this->logHandler->getRecords();
+ $this->assertStringContainsString('Pocket HTML Import: user is not defined', $records[0]['message']);
+ $this->assertSame('ERROR', $records[0]['level_name']);
+ }
+
+ private function getPocketHtmlImport($unsetUser = false, $dispatched = 0)
+ {
+ $this->user = new User();
+
+ $this->em = $this->getMockBuilder(EntityManager::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy = $this->getMockBuilder(ContentProxy::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $dispatcher = $this->getMockBuilder(EventDispatcher::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $dispatcher
+ ->expects($this->exactly($dispatched))
+ ->method('dispatch');
+
+ $this->logHandler = new TestHandler();
+ $logger = new Logger('test', [$this->logHandler]);
+
+ $wallabag = new PocketHtmlImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger);
+
+ if (false === $unsetUser) {
+ $wallabag->setUser($this->user);
+ }
+
+ return $wallabag;
+ }
+}
diff --git a/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php b/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php
new file mode 100644
index 000000000..04f8223dd
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php
@@ -0,0 +1,254 @@
+getShaarliImport();
+
+ $this->assertSame('Shaarli', $shaarliImport->getName());
+ $this->assertNotEmpty($shaarliImport->getUrl());
+ $this->assertSame('import.shaarli.description', $shaarliImport->getDescription());
+ }
+
+ public function testImport()
+ {
+ $shaarliImport = $this->getShaarliImport(false, 2);
+ $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->exactly(2))
+ ->method('findByUrlAndUserId')
+ ->willReturn(false);
+
+ $this->em
+ ->expects($this->any())
+ ->method('getRepository')
+ ->willReturn($entryRepo);
+
+ $entry = $this->getMockBuilder(Entry::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->exactly(2))
+ ->method('updateEntry')
+ ->willReturn($entry);
+
+ $res = $shaarliImport->import();
+
+ $this->assertTrue($res);
+ $this->assertSame(['skipped' => 0, 'imported' => 2, 'queued' => 0], $shaarliImport->getSummary());
+ }
+
+ public function testImportAndMarkAllAsRead()
+ {
+ $shaarliImport = $this->getShaarliImport(false, 1);
+ $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->exactly(2))
+ ->method('findByUrlAndUserId')
+ ->will($this->onConsecutiveCalls(false, true));
+
+ $this->em
+ ->expects($this->any())
+ ->method('getRepository')
+ ->willReturn($entryRepo);
+
+ $this->contentProxy
+ ->expects($this->exactly(1))
+ ->method('updateEntry')
+ ->willReturn(new Entry($this->user));
+
+ // check that every entry persisted are archived
+ $this->em
+ ->expects($this->any())
+ ->method('persist')
+ ->with($this->callback(function ($persistedEntry) {
+ return (bool) $persistedEntry->isArchived();
+ }));
+
+ $res = $shaarliImport
+ ->setMarkAsRead(true)
+ ->import();
+
+ $this->assertTrue($res);
+
+ $this->assertSame(['skipped' => 1, 'imported' => 1, 'queued' => 0], $shaarliImport->getSummary());
+ }
+
+ public function testImportWithRabbit()
+ {
+ $shaarliImport = $this->getShaarliImport();
+ $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder(Entry::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $producer
+ ->expects($this->exactly(2))
+ ->method('publish');
+
+ $shaarliImport->setProducer($producer);
+
+ $res = $shaarliImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $shaarliImport->getSummary());
+ }
+
+ public function testImportWithRedis()
+ {
+ $shaarliImport = $this->getShaarliImport();
+ $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html');
+
+ $entryRepo = $this->getMockBuilder(EntryRepository::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder(Entry::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter(Client::class, true);
+
+ $queue = new RedisQueue($redisMock, 'shaarli');
+ $producer = new Producer($queue);
+
+ $shaarliImport->setProducer($producer);
+
+ $res = $shaarliImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $shaarliImport->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('shaarli'));
+ }
+
+ public function testImportBadFile()
+ {
+ $shaarliImport = $this->getShaarliImport();
+ $shaarliImport->setFilepath(__DIR__ . '/../fixtures/wallabag-v1.jsonx');
+
+ $res = $shaarliImport->import();
+
+ $this->assertFalse($res);
+
+ $records = $this->logHandler->getRecords();
+ $this->assertStringContainsString('Wallabag HTML Import: unable to read file', $records[0]['message']);
+ $this->assertSame('ERROR', $records[0]['level_name']);
+ }
+
+ public function testImportUserNotDefined()
+ {
+ $shaarliImport = $this->getShaarliImport(true);
+ $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html');
+
+ $res = $shaarliImport->import();
+
+ $this->assertFalse($res);
+
+ $records = $this->logHandler->getRecords();
+ $this->assertStringContainsString('Wallabag HTML Import: user is not defined', $records[0]['message']);
+ $this->assertSame('ERROR', $records[0]['level_name']);
+ }
+
+ private function getShaarliImport($unsetUser = false, $dispatched = 0)
+ {
+ $this->user = new User();
+
+ $this->em = $this->getMockBuilder(EntityManager::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy = $this->getMockBuilder(ContentProxy::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $dispatcher = $this->getMockBuilder(EventDispatcher::class)
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $dispatcher
+ ->expects($this->exactly($dispatched))
+ ->method('dispatch');
+
+ $this->logHandler = new TestHandler();
+ $logger = new Logger('test', [$this->logHandler]);
+
+ $wallabag = new ShaarliImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger);
+
+ if (false === $unsetUser) {
+ $wallabag->setUser($this->user);
+ }
+
+ return $wallabag;
+ }
+}
diff --git a/tests/Wallabag/ImportBundle/fixtures/ril_export.html b/tests/Wallabag/ImportBundle/fixtures/ril_export.html
new file mode 100644
index 000000000..310c092dc
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/fixtures/ril_export.html
@@ -0,0 +1,21 @@
+
+
+
+
+
+ Pocket Export
+
+
+ Unread
+
+
+ Read Archive
+
+
+
diff --git a/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html
new file mode 100644
index 000000000..ad401ea74
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html
@@ -0,0 +1,13 @@
+
+
+
+Bookmarks
+Shaarli export of all bookmarks on Mon, 17 Jul 23 14:31:25 +0200
+
+
- The Legacy of Firefox OS. In the two years or so since Mozilla… | by Ben Francis | Medium
+
- In the two years or so since Mozilla announced the end of Firefox OS as a Mozilla-run project, the B2G source code has found its way into a surprising number of commercial products.
+
- Template Filters — Eleventy
+
+
diff --git a/translations/messages.en.yml b/translations/messages.en.yml
index 6bb7e0c4e..e5a195bbb 100644
--- a/translations/messages.en.yml
+++ b/translations/messages.en.yml
@@ -534,6 +534,14 @@ import:
page_title: Import > del.icio.us
description: This importer will import all your Delicious bookmarks. Since 2021, you can export again your data from it using the export page (https://del.icio.us/export). Choose the "JSON" format and download it (like "delicious_export.2021.02.06_21.10.json").
how_to: Please select your Delicious export and click on the button below to upload and import it.
+ shaarli:
+ page_title: Import > Shaarli
+ description: This importer will import all your Shaarli bookmarks. Just go to the Tools section, then into "Export database", choose your bookmarks and export them. You will obtain a HTML file.
+ how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched.
+ pocket_html:
+ page_title: Import > Pocket HTML
+ description: This importer will import all your Pocket bookmarks (via HTML export). Just go to https://getpocket.com/export, then export the HTML file. An HTML file will be downloaded (like "ril_export.html").
+ how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched.
developer:
page_title: API clients management
welcome_message: Welcome to the wallabag API