From 52a8c3b9c760fc8318adb812faf187ce58d74a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 23 May 2025 14:56:05 +0200 Subject: [PATCH] Added CSV Pocket import --- app/config/config.yml | 14 + app/config/services.yml | 9 + app/config/services_rabbit.yml | 6 + app/config/services_redis.yml | 16 ++ src/Command/Import/ImportCommand.php | 5 +- src/Consumer/RabbitMQConsumerTotalProxy.php | 4 + src/Controller/Import/ImportController.php | 2 + src/Controller/Import/PocketCsvController.php | 46 ++++ src/Import/PocketCsvImport.php | 138 ++++++++++ templates/Import/PocketCsv/index.html.twig | 47 ++++ .../Import/ImportControllerTest.php | 2 +- .../Import/PocketCsvControllerTest.php | 155 +++++++++++ tests/Import/PocketCsvImportTest.php | 252 ++++++++++++++++++ tests/fixtures/Import/part_000000.csv | 8 + tests/fixtures/Import/test.csv | 0 translations/messages.cs.yml | 4 + translations/messages.en.yml | 4 + translations/messages.es.yml | 4 + translations/messages.fr.yml | 4 + translations/messages.gl.yml | 4 + translations/messages.hr.yml | 4 + translations/messages.nb.yml | 4 + translations/messages.oc.yml | 4 + translations/messages.pl.yml | 4 + translations/messages.ta.yml | 4 + translations/messages.tr.yml | 4 + translations/messages.zh.yml | 4 + translations/messages.zh_Hant.yml | 4 + 28 files changed, 754 insertions(+), 2 deletions(-) create mode 100644 src/Controller/Import/PocketCsvController.php create mode 100644 src/Import/PocketCsvImport.php create mode 100644 templates/Import/PocketCsv/index.html.twig create mode 100644 tests/Controller/Import/PocketCsvControllerTest.php create mode 100644 tests/Import/PocketCsvImportTest.php create mode 100644 tests/fixtures/Import/part_000000.csv create mode 100644 tests/fixtures/Import/test.csv diff --git a/app/config/config.yml b/app/config/config.yml index 19e139db1..e170430b7 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -305,6 +305,11 @@ old_sound_rabbit_mq: exchange_options: name: 'wallabag.import.pocket_html' type: topic + import_pocket_csv: + connection: default + exchange_options: + name: 'wallabag.import.pocket_csv' + type: topic consumers: import_pocket: connection: default @@ -423,6 +428,15 @@ old_sound_rabbit_mq: name: 'wallabag.import.pocket_html' callback: wallabag.consumer.amqp.pocket_html qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} + import_pocket_csv: + connection: default + exchange_options: + name: 'wallabag.import.pocket_csv' + type: topic + queue_options: + name: 'wallabag.import.pocket_csv' + callback: wallabag.consumer.amqp.pocket_csv + qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} fos_js_routing: routes_to_expose: diff --git a/app/config/services.yml b/app/config/services.yml index 3cca8b30f..620c71087 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -108,6 +108,11 @@ services: $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_html_producer' $redisProducer: '@wallabag.producer.redis.pocket_html' + Wallabag\Controller\Import\PocketCsvController: + arguments: + $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_csv_producer' + $redisProducer: '@wallabag.producer.redis.pocket_csv' + Wallabag\Doctrine\MigrationFactoryDecorator: decorates: doctrine.migrations.migrations_factory @@ -328,6 +333,10 @@ services: tags: - { name: wallabag.import, alias: pocket_html } + Wallabag\Import\PocketCsvImport: + tags: + - { name: wallabag.import, alias: pocket_csv } + # to factorize the proximity and bypass translation for prev & next pagerfanta.view.default_wallabag: class: Pagerfanta\View\OptionableView diff --git a/app/config/services_rabbit.yml b/app/config/services_rabbit.yml index ff00fc8df..8c83b2123 100644 --- a/app/config/services_rabbit.yml +++ b/app/config/services_rabbit.yml @@ -20,6 +20,7 @@ services: $elcuratorConsumer: '@old_sound_rabbit_mq.import_elcurator_consumer' $shaarliConsumer: '@old_sound_rabbit_mq.import_shaarli_consumer' $pocketHtmlConsumer: '@old_sound_rabbit_mq.import_pocket_html_consumer' + $pocketCsvConsumer: '@old_sound_rabbit_mq.import_pocket_csv_consumer' $omnivoreConsumer: '@old_sound_rabbit_mq.import_omnivore_consumer' wallabag.consumer.amqp.pocket: @@ -86,3 +87,8 @@ services: class: Wallabag\Consumer\AMQPEntryConsumer arguments: $import: '@Wallabag\Import\PocketHtmlImport' + + wallabag.consumer.amqp.pocket_csv: + class: Wallabag\Consumer\AMQPEntryConsumer + arguments: + $import: '@Wallabag\Import\PocketCsvImport' diff --git a/app/config/services_redis.yml b/app/config/services_redis.yml index 2065fe4f5..af9b5ad85 100644 --- a/app/config/services_redis.yml +++ b/app/config/services_redis.yml @@ -212,3 +212,19 @@ services: class: Wallabag\Consumer\RedisEntryConsumer arguments: $import: '@Wallabag\Import\PocketHtmlImport' + + # pocket csv + wallabag.queue.redis.pocket_csv: + class: Simpleue\Queue\RedisQueue + arguments: + $queueName: "wallabag.import.pocket_csv" + + wallabag.producer.redis.pocket_csv: + class: Wallabag\Redis\Producer + arguments: + - "@wallabag.queue.redis.pocket_csv" + + wallabag.consumer.redis.pocket_csv: + class: Wallabag\Consumer\RedisEntryConsumer + arguments: + $import: '@Wallabag\Import\PocketCsvImport' diff --git a/src/Command/Import/ImportCommand.php b/src/Command/Import/ImportCommand.php index 33436e059..aa225b03c 100644 --- a/src/Command/Import/ImportCommand.php +++ b/src/Command/Import/ImportCommand.php @@ -21,6 +21,7 @@ use Wallabag\Import\FirefoxImport; use Wallabag\Import\InstapaperImport; use Wallabag\Import\OmnivoreImport; use Wallabag\Import\PinboardImport; +use Wallabag\Import\PocketCsvImport; use Wallabag\Import\PocketHtmlImport; use Wallabag\Import\ReadabilityImport; use Wallabag\Import\ShaarliImport; @@ -48,6 +49,7 @@ class ImportCommand extends Command private readonly ElcuratorImport $elcuratorImport, private readonly ShaarliImport $shaarliImport, private readonly PocketHtmlImport $pocketHtmlImport, + private readonly PocketCsvImport $pocketCsvImport, private readonly OmnivoreImport $omnivoreImport, ) { parent::__construct(); @@ -58,7 +60,7 @@ class ImportCommand extends Command $this ->addArgument('username', InputArgument::REQUIRED, 'User to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli or pocket', 'v1') + ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli, pocket or pocket_csv', 'v1') ->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false) ->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account') ->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL') @@ -109,6 +111,7 @@ class ImportCommand extends Command 'elcurator' => $this->elcuratorImport, 'shaarli' => $this->shaarliImport, 'pocket' => $this->pocketHtmlImport, + 'pocket_csv' => $this->pocketCsvImport, 'omnivore' => $this->omnivoreImport, default => $this->wallabagV1Import, }; diff --git a/src/Consumer/RabbitMQConsumerTotalProxy.php b/src/Consumer/RabbitMQConsumerTotalProxy.php index b32a0653a..45125a3ac 100644 --- a/src/Consumer/RabbitMQConsumerTotalProxy.php +++ b/src/Consumer/RabbitMQConsumerTotalProxy.php @@ -23,6 +23,7 @@ class RabbitMQConsumerTotalProxy private readonly Consumer $elcuratorConsumer, private readonly Consumer $shaarliConsumer, private readonly Consumer $pocketHtmlConsumer, + private readonly Consumer $pocketCsvConsumer, private readonly Consumer $omnivoreConsumer, ) { } @@ -75,6 +76,9 @@ class RabbitMQConsumerTotalProxy case 'pocket_html': $consumer = $this->pocketHtmlConsumer; break; + case 'pocket_csv': + $consumer = $this->pocketCsvConsumer; + break; case 'omnivore': $consumer = $this->omnivoreConsumer; break; diff --git a/src/Controller/Import/ImportController.php b/src/Controller/Import/ImportController.php index 24ae654c0..8ce60437e 100644 --- a/src/Controller/Import/ImportController.php +++ b/src/Controller/Import/ImportController.php @@ -58,6 +58,7 @@ class ImportController extends AbstractController + $this->rabbitMQConsumerTotalProxy->getTotalMessage('elcurator') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('shaarli') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_html') + + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_csv') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('omnivore') ; } catch (\Exception) { @@ -77,6 +78,7 @@ class ImportController extends AbstractController + $this->redisClient->llen('wallabag.import.elcurator') + $this->redisClient->llen('wallabag.import.shaarli') + $this->redisClient->llen('wallabag.import.pocket_html') + + $this->redisClient->llen('wallabag.import.pocket_csv') + $this->redisClient->llen('wallabag.import.omnivore') ; } catch (\Exception) { diff --git a/src/Controller/Import/PocketCsvController.php b/src/Controller/Import/PocketCsvController.php new file mode 100644 index 000000000..c76e2cd6a --- /dev/null +++ b/src/Controller/Import/PocketCsvController.php @@ -0,0 +1,46 @@ +craueConfig->get('import_with_rabbitmq')) { + $this->pocketCsvImport->setProducer($this->rabbitMqProducer); + } elseif ($this->craueConfig->get('import_with_redis')) { + $this->pocketCsvImport->setProducer($this->redisProducer); + } + + return $this->pocketCsvImport; + } + + protected function getImportTemplate() + { + return 'Import/PocketCsv/index.html.twig'; + } +} diff --git a/src/Import/PocketCsvImport.php b/src/Import/PocketCsvImport.php new file mode 100644 index 000000000..2cb3aa6a2 --- /dev/null +++ b/src/Import/PocketCsvImport.php @@ -0,0 +1,138 @@ +filepath = $filepath; + + return $this; + } + + public function validateEntry(array $importedEntry) + { + if (empty($importedEntry['url'])) { + return false; + } + + return true; + } + + public function import() + { + if (!$this->user) { + $this->logger->error('Pocket CSV Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Pocket CSV Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $entries = []; + $handle = fopen($this->filepath, 'r'); + while (false !== ($data = fgetcsv($handle, 10240))) { + if ('title' === $data[0]) { + continue; + } + + $entries[] = [ + 'url' => $data[1], + 'title' => $data[0], + 'is_archived' => 'archive' === $data[4], + 'created_at' => $data[2], + 'tags' => $data[3], + ]; + } + fclose($handle); + + if (empty($entries)) { + $this->logger->error('Pocket CSV Import: no entries in imported file'); + + return false; + } + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository(Entry::class) + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return null; + } + + $entry = new Entry($this->user); + $entry->setUrl($importedEntry['url']); + $entry->setTitle($importedEntry['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $this->fetchContent($entry, $importedEntry['url'], $importedEntry); + + if (!empty($importedEntry['tags'])) { + $tags = str_replace('|', ',', $importedEntry['tags']); + $this->tagsAssigner->assignTagsToEntry( + $entry, + $tags, + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } + + $entry->updateArchived($importedEntry['is_archived']); + $entry->setCreatedAt(\DateTime::createFromFormat('U', $importedEntry['created_at'])); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 'archive'; + + return $importedEntry; + } +} diff --git a/templates/Import/PocketCsv/index.html.twig b/templates/Import/PocketCsv/index.html.twig new file mode 100644 index 000000000..57e311ded --- /dev/null +++ b/templates/Import/PocketCsv/index.html.twig @@ -0,0 +1,47 @@ +{% extends "layout.html.twig" %} + +{% block title %}{{ 'import.pocket_csv.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include 'Import/_information.html.twig' %} + +
+
{{ import.description|trans|raw }}
+

{{ 'import.pocket_csv.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+
+ +
+
+ + {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Controller/Import/ImportControllerTest.php b/tests/Controller/Import/ImportControllerTest.php index 9c156a83e..14adf8318 100644 --- a/tests/Controller/Import/ImportControllerTest.php +++ b/tests/Controller/Import/ImportControllerTest.php @@ -23,6 +23,6 @@ class ImportControllerTest extends WallabagTestCase $crawler = $client->request('GET', '/import/'); $this->assertSame(200, $client->getResponse()->getStatusCode()); - $this->assertSame(13, $crawler->filter('.card-title')->count()); + $this->assertSame(14, $crawler->filter('.card-title')->count()); } } diff --git a/tests/Controller/Import/PocketCsvControllerTest.php b/tests/Controller/Import/PocketCsvControllerTest.php new file mode 100644 index 000000000..d13bcf2da --- /dev/null +++ b/tests/Controller/Import/PocketCsvControllerTest.php @@ -0,0 +1,155 @@ +logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportPocketCsvWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/pocket_csv'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0); + } + + public function testImportPocketCsvBadFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + } + + public function testImportPocketCsvWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getTestClient(); + $client->getContainer()->get(Config::class)->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/pocket_csv'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../../fixtures/Import/part_000000.csv', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.pocket_csv')); + + $client->getContainer()->get(Config::class)->set('import_with_redis', 0); + } + + public function testImportWallabagWithPocketCsvFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../../fixtures/Import/part_000000.csv', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://jp-lambert.me/est-ce-que-jai-besoin-d-un-scrum-master-604f5a471c73', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for jp-lambert.me is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for jp-lambert.me is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for jp-lambert.me is ok'); + $this->assertCount(1, $content->getTags()); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../../fixtures/Import/test.csv', 'test.csv'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Import/PocketCsvImportTest.php b/tests/Import/PocketCsvImportTest.php new file mode 100644 index 000000000..74cc189b1 --- /dev/null +++ b/tests/Import/PocketCsvImportTest.php @@ -0,0 +1,252 @@ +getPocketCsvImport(); + + $this->assertSame('Pocket CSV', $pocketCsvImport->getName()); + $this->assertNotEmpty($pocketCsvImport->getUrl()); + $this->assertSame('import.pocket_csv.description', $pocketCsvImport->getDescription()); + } + + public function testImport() + { + $pocketCsvImport = $this->getPocketCsvImport(false, 7); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/part_000000.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(7)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(7)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $pocketCsvImport->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 7, 'queued' => 0], $pocketCsvImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $pocketCsvImport = $this->getPocketCsvImport(false, 1); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/part_000000.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(7)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(fn ($persistedEntry) => (bool) $persistedEntry->isArchived())); + + $res = $pocketCsvImport + ->setMarkAsRead(true) + ->import(); + + $this->assertTrue($res); + + $this->assertSame(['skipped' => 6, 'imported' => 1, 'queued' => 0], $pocketCsvImport->getSummary()); + } + + public function testImportWithRabbit() + { + $pocketCsvImport = $this->getPocketCsvImport(); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/part_000000.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class) + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(7)) + ->method('publish'); + + $pocketCsvImport->setProducer($producer); + + $res = $pocketCsvImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 7], $pocketCsvImport->getSummary()); + } + + public function testImportWithRedis() + { + $pocketCsvImport = $this->getPocketCsvImport(); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/part_000000.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter(Client::class, true); + + $queue = new RedisQueue($redisMock, 'pocket_csv'); + $producer = new Producer($queue); + + $pocketCsvImport->setProducer($producer); + + $res = $pocketCsvImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 7], $pocketCsvImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('pocket_csv')); + } + + public function testImportBadFile() + { + $pocketCsvImport = $this->getPocketCsvImport(); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/wallabag-v1.jsonx'); + + $res = $pocketCsvImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket CSV Import: unable to read file', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $pocketCsvImport = $this->getPocketCsvImport(true); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/part_000000.csv'); + + $res = $pocketCsvImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket CSV Import: user is not defined', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + private function getPocketCsvImport($unsetUser = false, $dispatched = 0) + { + $this->user = new User(); + + $this->em = $this->getMockBuilder(EntityManager::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder(ContentProxy::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher = $this->getMockBuilder(EventDispatcher::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + + $wallabag = new PocketCsvImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } +} diff --git a/tests/fixtures/Import/part_000000.csv b/tests/fixtures/Import/part_000000.csv new file mode 100644 index 000000000..774caf3e1 --- /dev/null +++ b/tests/fixtures/Import/part_000000.csv @@ -0,0 +1,8 @@ +title,url,time_added,tags,status +You Might Not Need jQuery,http://youmightnotneedjquery.com/,1600322788,,unread +Est-ce que j’ai besoin d’un Scrum Master ? | by Jean-Pierre Lambert | Jean-,https://jp-lambert.me/est-ce-que-jai-besoin-d-un-scrum-master-604f5a471c73,1600172739,,unread +"Avec les accusés d’El Halia, par Gisèle Halimi (Le Monde diplomatique, sept",https://www.monde-diplomatique.fr/2020/09/HALIMI/62165,1599806347,,unread +ArchiveBox question: How do I import links from a RSS feed?,https://www.reddit.com/r/DataHoarder/comments/ioupbk/archivebox_question_how_do_i_import_links_from_a/,1600961496,,archive +« Tu vas pleurer les premières fois » : que se passe-t-il au sein du studio,https://www.numerama.com/politique/646826-tu-vas-pleurer-les-premieres-fois-que-se-passe-t-il-au-sein-du-studio-dubisoft-derriere-trackmania.html#utm_medium=distibuted&utm_source=rss&utm_campaign=646826,1599809025,,unread +Comment Konbini s’est fait piéger par un « père masculiniste »,https://www.nouvelobs.com/rue89/20200911.OBS33165/comment-konbini-s-est-fait-pieger-par-un-pere-masculiniste.html,1599819251,,archive +Des abeilles pour résoudre les « conflits » entre les humains et les élépha,https://reporterre.net/Des-abeilles-pour-resoudre-les-conflits-entre-les-humains-et-les-elephants,1599890673,,unread diff --git a/tests/fixtures/Import/test.csv b/tests/fixtures/Import/test.csv new file mode 100644 index 000000000..e69de29bb diff --git a/translations/messages.cs.yml b/translations/messages.cs.yml index d4fc780f9..7dd5435f3 100644 --- a/translations/messages.cs.yml +++ b/translations/messages.cs.yml @@ -559,6 +559,10 @@ import: how_to: Zvolte soubor zálohy záložek a kliknutím na níže uvedené tlačítko jej naimportujte. Pamatujte na to, že tento proces může trvat dlouho, protože je třeba načíst všechny články. page_title: Import > Pocket HTML description: Tento importér naimportuje všechny vaše záložky z Pocket. Stačí přejít na https://getpocket.com/export) a exportovat soubor HTML, který bude stažen (například „ril_export.html“). + pocket_csv: + how_to: Zvolte soubor zálohy záložek a kliknutím na níže uvedené tlačítko jej naimportujte. Pamatujte na to, že tento proces může trvat dlouho, protože je třeba načíst všechny články. + page_title: Import > Pocket HTML + description: Tento importér naimportuje všechny vaše záložky z Pocket. Stačí přejít na https://getpocket.com/export) a exportovat soubor HTML, který bude stažen (například „ril_export.html“). omnivore: how_to: Rozbalte prosím svůj export z Omnivoru, následně nahrajte jeden po druhém všechny JSON soubory s názevem "metadata_x_to_y.json". page_title: Import > Omnivore diff --git a/translations/messages.en.yml b/translations/messages.en.yml index 2b330f368..fcf566acf 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -553,6 +553,10 @@ import: page_title: Import > Pocket HTML description: This importer will import all your Pocket bookmarks (via HTML export). Just go to https://getpocket.com/export, then export the HTML file. An HTML file will be downloaded (like "ril_export.html"). how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. + pocket_csv: + page_title: Import > Pocket CSV + description: This importer will import all your Pocket bookmarks (via CSV export). Just go to https://getpocket.com/export, then export the file. A ZIP file will be downloaded (like "pocket.zip"). Extract it, you will obtain a CSV file, called "part_000000.csv". + how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. developer: page_title: API clients management welcome_message: Welcome to the wallabag API diff --git a/translations/messages.es.yml b/translations/messages.es.yml index 47b497f12..86e56fe3c 100644 --- a/translations/messages.es.yml +++ b/translations/messages.es.yml @@ -550,6 +550,10 @@ import: description: Esta herramienta importará todos tus marcadores de Pocket (vía HTML). Sólo tienes que ir a https://getpocket.com/export, y luego exportar el archivo HTML. Se descargará un archivo HTML (como "ril_export.html"). page_title: Importar > Pocket HTML how_to: Por favor, elige el archivo de la copia de seguridad del marcador y haz clic en el botón inferior para importarlo. Ten en cuenta que el proceso puede llevar mucho tiempo, ya que hay que recuperar todos los artículos. + pocket_csv: + description: Esta herramienta importará todos tus marcadores de Pocket (vía HTML). Sólo tienes que ir a https://getpocket.com/export, y luego exportar el archivo HTML. Se descargará un archivo HTML (como "ril_export.html"). + page_title: Importar > Pocket HTML + how_to: Por favor, elige el archivo de la copia de seguridad del marcador y haz clic en el botón inferior para importarlo. Ten en cuenta que el proceso puede llevar mucho tiempo, ya que hay que recuperar todos los artículos. omnivore: page_title: Importar > Omnivore description: Este importador importará todos sus artículos de Omnivore. diff --git a/translations/messages.fr.yml b/translations/messages.fr.yml index a0b3b1935..0f9017d23 100644 --- a/translations/messages.fr.yml +++ b/translations/messages.fr.yml @@ -550,6 +550,10 @@ import: page_title: Importer > Pocket HTML description: Cet importateur importera toutes vos signets Pocket (via exportation HTML). Il suffit d'aller à https://getpocket.com/export, puis d'exporter le fichier HTML. Un fichier HTML sera téléchargé (comme « ril_export.html »). how_to: Veuillez choisir le fichier de sauvegarde de signets et cliquez sur le bouton ci-dessous pour l'importer. Pensez au fait que le processus peut prendre longtemps puisque tous les articles doivent être récupérés. + pocket_csv: + page_title: Importer > Pocket CSV + description: Cet importateur importera toutes vos signets Pocket (via exportation CSV). Il suffit d'aller à https://getpocket.com/export, puis d'exporter le fichier. Un fichier ZIP sera téléchargé (comme « pocket.zip »). Décompressez le et vous obtiendrez un fichier CSV appelé "part_000000.csv". + how_to: Veuillez choisir le fichier de sauvegarde de signets et cliquez sur le bouton ci-dessous pour l'importer. Pensez au fait que le processus peut prendre longtemps puisque tous les articles doivent être récupérés. omnivore: description: Cet outil va importer tous vos articles depuis Omnivore. page_title: Importer > Omnivore diff --git a/translations/messages.gl.yml b/translations/messages.gl.yml index ccd429d3a..8f02a5744 100644 --- a/translations/messages.gl.yml +++ b/translations/messages.gl.yml @@ -477,6 +477,10 @@ import: page_title: Importar > Pocket HTML description: Este importador traerá todos os teus marcadores en Pocket (vía exportación HTML). Vai a https://getpocket.com/export, e exporta o ficheiro HTML. Descargarás un ficheiro HTML (tipo "ril_export.html"). how_to: Elixe o ficheiro da copia de apoio cos marcadores e preme no botón para importalo. Ten en conta que o proceso podería demorarse porque hai que importar todos os artigos. + pocket_csv: + page_title: Importar > Pocket HTML + description: Este importador traerá todos os teus marcadores en Pocket (vía exportación HTML). Vai a https://getpocket.com/export, e exporta o ficheiro HTML. Descargarás un ficheiro HTML (tipo "ril_export.html"). + how_to: Elixe o ficheiro da copia de apoio cos marcadores e preme no botón para importalo. Ten en conta que o proceso podería demorarse porque hai que importar todos os artigos. shaarli: page_title: Importar > Shaarli description: Este importador traerá todos os teus marcadores de Shaarli. Vai á sección Ferramentas, e despois en "Exportar base de datos", elixe os teus marcadores e expórtaos. Terás un ficheiro HTML. diff --git a/translations/messages.hr.yml b/translations/messages.hr.yml index e20bd4b28..559eaad2c 100644 --- a/translations/messages.hr.yml +++ b/translations/messages.hr.yml @@ -113,6 +113,10 @@ import: page_title: Uvoz > Pocket HTML how_to: Odaberi datoteku sigurnosne kopije zabilježaka i pritisni donji gumb za uvoz. Postupak može trajati dugo jer se moraju dohvatiti svi članci. description: Ovaj uvoznik uvozi sve tvoje Pocket zabilješke (putem HTML izvoza). Idi na https://getpocket.com/export, zatim izvezi HTML datoteku. Preuzet će se HTML datoteka (poput „ril_export.html”). + pocket_csv: + page_title: Uvoz > Pocket HTML + how_to: Odaberi datoteku sigurnosne kopije zabilježaka i pritisni donji gumb za uvoz. Postupak može trajati dugo jer se moraju dohvatiti svi članci. + description: Ovaj uvoznik uvozi sve tvoje Pocket zabilješke (putem HTML izvoza). Idi na https://getpocket.com/export, zatim izvezi HTML datoteku. Preuzet će se HTML datoteka (poput „ril_export.html”). omnivore: page_title: Uvoz > Omnivore description: Ovaj uvoznik uvozi sve tvoje Omnivor članke. diff --git a/translations/messages.nb.yml b/translations/messages.nb.yml index d8168b6c3..9139865f3 100644 --- a/translations/messages.nb.yml +++ b/translations/messages.nb.yml @@ -473,6 +473,10 @@ import: how_to: Velg sikkerhetskopifilen for bokmerke og klikk på knappen nedenfor for å importere den. Merk at prosessen kan ta lang tid siden alle artiklene må hentes. page_title: Importer > Pocket HTML description: Denne modulen vil importere alle Pocket-bokmerkene dine (via HTML-eksport). Gå til https://getpocket.com/export, og eksporter HTML-filen derfra. En HTML-fil vil bli lastet ned (som «ril_export.html»). + pocket_csv: + how_to: Velg sikkerhetskopifilen for bokmerke og klikk på knappen nedenfor for å importere den. Merk at prosessen kan ta lang tid siden alle artiklene må hentes. + page_title: Importer > Pocket HTML + description: Denne modulen vil importere alle Pocket-bokmerkene dine (via HTML-eksport). Gå til https://getpocket.com/export, og eksporter HTML-filen derfra. En HTML-fil vil bli lastet ned (som «ril_export.html»). tag: new: placeholder: Du kan legge til flere stikkord, delt med komma. diff --git a/translations/messages.oc.yml b/translations/messages.oc.yml index 9172dba88..22aadbe89 100644 --- a/translations/messages.oc.yml +++ b/translations/messages.oc.yml @@ -552,6 +552,10 @@ import: page_title: Importar > Pocket HTML description: Aquesta aisina d’importacion importarà totes vòstres marcadors Pocket (via un expòrt HTML). Anatz simplament a https://getpocket.com/export puèi exportar lo fichièr HTML. Se telecargarà un fichièr HTML (coma « ril_export.html »). how_to: Mercés de seleccionar lo fichièr de salvagarda de marcadors e de clicar lo boton çai-jos per l’importar. Remarcatz qu’aquò pòt trigar, lo temps que totes los articles sián recuperats. + pocket_csv: + page_title: Importar > Pocket HTML + description: Aquesta aisina d’importacion importarà totes vòstres marcadors Pocket (via un expòrt HTML). Anatz simplament a https://getpocket.com/export puèi exportar lo fichièr HTML. Se telecargarà un fichièr HTML (coma « ril_export.html »). + how_to: Mercés de seleccionar lo fichièr de salvagarda de marcadors e de clicar lo boton çai-jos per l’importar. Remarcatz qu’aquò pòt trigar, lo temps que totes los articles sián recuperats. omnivore: page_title: Importar > Omnivore developer: diff --git a/translations/messages.pl.yml b/translations/messages.pl.yml index 44e7e390b..9d8b536db 100644 --- a/translations/messages.pl.yml +++ b/translations/messages.pl.yml @@ -553,6 +553,10 @@ import: page_title: Importuj > Pocket HTML description: Ten importer zaimportuje wszystkie zakładki Pocket (za pośrednictwem eksportu HTML). Po prostu przejdź do https://getpocket.com/export, a następnie wyeksportuj plik HTML. Plik HTML zostanie pobrany (np. „ril_export.html”). how_to: Wybierz swój plik z zakładkami i naciśnij poniższy przycisk, aby je zaimportować. Może to zająć dłuższą chwilę, zanim wszystkie artykuły zostaną przeniesione. + pocket_csv: + page_title: Importuj > Pocket HTML + description: Ten importer zaimportuje wszystkie zakładki Pocket (za pośrednictwem eksportu HTML). Po prostu przejdź do https://getpocket.com/export, a następnie wyeksportuj plik HTML. Plik HTML zostanie pobrany (np. „ril_export.html”). + how_to: Wybierz swój plik z zakładkami i naciśnij poniższy przycisk, aby je zaimportować. Może to zająć dłuższą chwilę, zanim wszystkie artykuły zostaną przeniesione. omnivore: description: Ten importer zaimportuje wszystkie Twoje artykuły z Omnivore. page_title: Importuj > Omnivore diff --git a/translations/messages.ta.yml b/translations/messages.ta.yml index dfface4c8..8dd361203 100644 --- a/translations/messages.ta.yml +++ b/translations/messages.ta.yml @@ -260,6 +260,10 @@ import: page_title: இறக்குமதி> பாக்கெட் உஉகுமொ description: இந்த இறக்குமதியாளர் உங்கள் அனைத்து பாக்கெட் புக்மார்க்குகளையும் (HTML ஏற்றுமதி வழியாக) இறக்குமதி செய்வார். Https://getpocket.com/export க்குச் சென்று, பின்னர் உஉகுமொ கோப்பை ஏற்றுமதி செய்யுங்கள். ஒரு உஉகுமொ கோப்பு பதிவிறக்கம் செய்யப்படும் ("RIL_EXPORT.HTML" போன்றவை). how_to: புக்மார்க்கு காப்புப்பிரதி கோப்பைத் தேர்ந்தெடுத்து அதை இறக்குமதி செய்ய கீழே உள்ள பொத்தானைக் சொடுக்கு செய்க. அனைத்து கட்டுரைகளையும் பெற வேண்டியிருப்பதால் செயல்முறை நீண்ட நேரம் ஆகலாம் என்பதை நினைவில் கொள்க. + pocket_csv: + page_title: இறக்குமதி> பாக்கெட் உஉகுமொ + description: இந்த இறக்குமதியாளர் உங்கள் அனைத்து பாக்கெட் புக்மார்க்குகளையும் (HTML ஏற்றுமதி வழியாக) இறக்குமதி செய்வார். Https://getpocket.com/export க்குச் சென்று, பின்னர் உஉகுமொ கோப்பை ஏற்றுமதி செய்யுங்கள். ஒரு உஉகுமொ கோப்பு பதிவிறக்கம் செய்யப்படும் ("RIL_EXPORT.HTML" போன்றவை). + how_to: புக்மார்க்கு காப்புப்பிரதி கோப்பைத் தேர்ந்தெடுத்து அதை இறக்குமதி செய்ய கீழே உள்ள பொத்தானைக் சொடுக்கு செய்க. அனைத்து கட்டுரைகளையும் பெற வேண்டியிருப்பதால் செயல்முறை நீண்ட நேரம் ஆகலாம் என்பதை நினைவில் கொள்க. developer: howto: description: diff --git a/translations/messages.tr.yml b/translations/messages.tr.yml index c83bce803..e12179173 100644 --- a/translations/messages.tr.yml +++ b/translations/messages.tr.yml @@ -549,6 +549,10 @@ import: page_title: İçe Aktar > Pocket HTML description: Bu içe aktarıcı tüm Pocket yer imlerinizi (HTML dışa aktarmayla) içe aktaracaktır. https://getpocket.com adresine gidin, sonra HTML dosyasını dışa aktarın. Bir HTML dosyası ("ril_export.html" gibi) indirilecektir. how_to: Lütfen yer imi yedek dosyasını seçin ve onu içe aktarmak aşağıdaki düğmeye tıklayın. Tüm makaleler sitelerinden çekileceği için sürecin uzun sürebileceğini unutmayın. + pocket_csv: + page_title: İçe Aktar > Pocket HTML + description: Bu içe aktarıcı tüm Pocket yer imlerinizi (HTML dışa aktarmayla) içe aktaracaktır. https://getpocket.com adresine gidin, sonra HTML dosyasını dışa aktarın. Bir HTML dosyası ("ril_export.html" gibi) indirilecektir. + how_to: Lütfen yer imi yedek dosyasını seçin ve onu içe aktarmak aşağıdaki düğmeye tıklayın. Tüm makaleler sitelerinden çekileceği için sürecin uzun sürebileceğini unutmayın. user: form: username_label: Kullanıcı adı diff --git a/translations/messages.zh.yml b/translations/messages.zh.yml index b38d86f92..ef08978a3 100644 --- a/translations/messages.zh.yml +++ b/translations/messages.zh.yml @@ -553,6 +553,10 @@ import: page_title: 导入 > Pocket HTML how_to: 请选择书签备份文件并点击下面的按钮导入。请注意，由于必须获取所有文章，因此该过程可能需要很长时间。 description: 此导入器将导入您所有的 Pocket 书签（通过 HTML 导出）。只需转到 https://getpocket.com/export，然后导出 HTML 文件。将下载一个 HTML 文件（如“ril_export.html”）。 + pocket_csv: + page_title: 导入 > Pocket HTML + how_to: 请选择书签备份文件并点击下面的按钮导入。请注意，由于必须获取所有文章，因此该过程可能需要很长时间。 + description: 此导入器将导入您所有的 Pocket 书签（通过 HTML 导出）。只需转到 https://getpocket.com/export，然后导出 HTML 文件。将下载一个 HTML 文件（如“ril_export.html”）。 developer: page_title: 'API 客户端管理' welcome_message: '欢迎来到 wallabag API' diff --git a/translations/messages.zh_Hant.yml b/translations/messages.zh_Hant.yml index 966b17fd9..4d6a1faae 100644 --- a/translations/messages.zh_Hant.yml +++ b/translations/messages.zh_Hant.yml @@ -483,6 +483,10 @@ import: page_title: 匯入 > Pocket HTML description: 這匯入器將匯入你在 Pocket 中的全部書籤（經 HTML 匯出）。你只須前往 https://getpocket.com/export 匯出 HTML檔案，將提供一個可下載的 HTML 檔案（名為「ril_export.html」之類）。 how_to: 請選取書籤備份檔案然後點擊以下按鈕匯入。須注意由於需要擷取全部的文章，整個過程可能會花費一段時間。 + pocket_csv: + page_title: 匯入 > Pocket HTML + description: 這匯入器將匯入你在 Pocket 中的全部書籤（經 HTML 匯出）。你只須前往 https://getpocket.com/export 匯出 HTML檔案，將提供一個可下載的 HTML 檔案（名為「ril_export.html」之類）。 + how_to: 請選取書籤備份檔案然後點擊以下按鈕匯入。須注意由於需要擷取全部的文章，整個過程可能會花費一段時間。 firefox: page_title: 匯入 > Firefox description: 這匯入器將匯入你在 Firefox 中的全部書籤。在書籤收藏庫（Ctrl+Shift+O）的「匯入及備份」中選擇「備份」後，你將取得一個 JSON 檔案。