diff --git a/app/config/config.yml b/app/config/config.yml index 7ee0cfb83..a4584a1ba 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -246,6 +246,16 @@ old_sound_rabbit_mq: exchange_options: name: 'wallabag.import.wallabag_v2' type: topic + import_firefox: + connection: default + exchange_options: + name: 'wallabag.import.firefox' + type: topic + import_chrome: + connection: default + exchange_options: + name: 'wallabag.import.chrome' + type: topic consumers: import_pocket: connection: default @@ -279,3 +289,19 @@ old_sound_rabbit_mq: queue_options: name: 'wallabag.import.wallabag_v2' callback: wallabag_import.consumer.amqp.wallabag_v2 + import_firefox: + connection: default + exchange_options: + name: 'wallabag.import.firefox' + type: topic + queue_options: + name: 'wallabag.import.firefox' + callback: wallabag_import.consumer.amqp.firefox + import_chrome: + connection: default + exchange_options: + name: 'wallabag.import.chrome' + type: topic + queue_options: + name: 'wallabag.import.chrome' + callback: wallabag_import.consumer.amqp.chrome diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index c6fcb355f..9eeb210ba 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -349,6 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + #chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index c0e82b59b..a9ec25198 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -349,6 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Entwickler' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 6f262209a..c0d8656d9 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -349,6 +349,14 @@ import: how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + firefox: + page_title: 'Import > Firefox' + description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + chrome: + page_title: 'Import > Chrome' + description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index 7b9810697..1d6993db0 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -349,6 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Promotor' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 99fcc3781..68272f995 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -349,6 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index dd82e7f59..b28068b6b 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -349,6 +349,14 @@ import: how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer." worker: enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :" + firefox: + page_title: 'Import > Firefox' + description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox.

Pour Firefox, ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json.

" + how_to: "Choisissez le fichier de sauvegarde de vos marques-page et cliquez sur le bouton pour l'importer. Soyez avertis que le processus peut prendre un temps assez long car tous les articles doivent être récupérés en ligne." + chrome: + page_title: 'Import > Chrome' + description: "Cet outil va vous permettre d'importer tous vos marques-pages de Google Chrome/Chromium. Pour Google Chrome, la situation du fichier dépend de votre système d'exploitation : Une fois que vous y êtes, copiez le fichier Bookmarks à un endroit où vous le retrouverez.
Notez que si vous utilisez Chromium à la place de Chrome, vous devez corriger les chemins en conséquence.

" + how_to: "Choisissez le fichier de sauvegarde de vos marques-page et cliquez sur le bouton pour l'importer. Soyez avertis que le processus peut prendre un temps assez long car tous les articles doivent être récupérés en ligne." developer: page_title: 'Développeur' @@ -432,7 +440,7 @@ flashes: notice: failed: "L'import a échoué, veuillez ré-essayer" failed_on_file: "Erreur lors du traitement de l'import. Vérifier votre fichier." - summary: "Rapport d'import: %imported% importés, %skipped% déjà présent." + summary: "Rapport d'import: %imported% importés, %skipped% déjà présents." summary_with_queue: "Rapport d'import: %queued% en cours de traitement." error: redis_enabled_not_installed: Redis est activé pour les imports asynchrones mais impossible de s'y connecter. Vérifier la configuration de Redis. diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index d3ce30c9e..bd98c8c0a 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -348,6 +348,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Sviluppatori' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index d040daea6..6da9ff18a 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -349,6 +349,14 @@ import: how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Desvolopador' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index a51ed1f27..daa34bc0e 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -349,6 +349,14 @@ import: how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' worker: enabled: "Import jest wykonywany asynchronicznie. Od momentu rozpoczęcia importu, zewnętrzna usługa może zajmować się na raz tylko jednym zadaniem. Bieżącą usługą jest:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Deweloper' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index de21f0b35..067f78786 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -349,6 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index d4b7a7a22..62c695103 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -349,6 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 20ecc6e18..1df38295f 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -17,7 +17,7 @@ class ImportCommand extends ContainerAwareCommand ->setDescription('Import entries from a JSON export from a wallabag v1 instance') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1 or v2', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -40,10 +40,20 @@ class ImportCommand extends ContainerAwareCommand throw new Exception(sprintf('User with id "%s" not found', $input->getArgument('userId'))); } - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); - - if ('v2' === $input->getOption('importer')) { - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + switch ($input->getOption('importer')) { + case 'v2': + $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + break; + case 'firefox': + $wallabag = $this->getContainer()->get('wallabag_import.firefox.import'); + break; + case 'chrome': + $wallabag = $this->getContainer()->get('wallabag_import.chrome.import'); + break; + case 'v1': + default: + $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); + break; } $wallabag->setMarkAsRead($input->getOption('markAsRead')); diff --git a/src/Wallabag/ImportBundle/Controller/BrowserController.php b/src/Wallabag/ImportBundle/Controller/BrowserController.php new file mode 100644 index 000000000..144a4880d --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/BrowserController.php @@ -0,0 +1,90 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = $this->getUser()->getId().'.json'; + + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $wallabag + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $wallabag->getSummary(); + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render($this->getImportTemplate(), [ + 'form' => $form->createView(), + 'import' => $wallabag, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Controller/ChromeController.php b/src/Wallabag/ImportBundle/Controller/ChromeController.php new file mode 100644 index 000000000..454f33474 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/ChromeController.php @@ -0,0 +1,41 @@ +get('wallabag_import.chrome.import'); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_chrome_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.chrome')); + } + + return $service; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Chrome:index.html.twig'; + } + + /** + * @Route("/chrome", name="import_chrome") + */ + public function indexAction(Request $request) + { + return parent::indexAction($request); + } +} diff --git a/src/Wallabag/ImportBundle/Controller/FirefoxController.php b/src/Wallabag/ImportBundle/Controller/FirefoxController.php new file mode 100644 index 000000000..c329b9c4a --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/FirefoxController.php @@ -0,0 +1,41 @@ +get('wallabag_import.firefox.import'); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_firefox_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.firefox')); + } + + return $service; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Firefox:index.html.twig'; + } + + /** + * @Route("/firefox", name="import_firefox") + */ + public function indexAction(Request $request) + { + return parent::indexAction($request); + } +} diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php new file mode 100644 index 000000000..e15443c40 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -0,0 +1,205 @@ +user) { + $this->logger->error('Wallabag Browser Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Wallabag Browser Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + return false; + } + + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + + $this->parseEntries($data); + + return true; + } + + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) + { + $i = 1; + + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + } + ++$i; + } + + $this->em->flush(); + } + + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) + { + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { + $this->parseEntries($importedEntry); + + return; + } + + if (array_key_exists('children', $importedEntry)) { + $this->parseEntries($importedEntry['children']); + + return; + } + + if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) { + return; + } + + $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url']; + + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $data = $this->prepareEntry($importedEntry); + + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); + + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] + ); + } + + $entry->setArchived($data['is_archived']); + + if (!empty($data['created_at'])) { + $dt = new \DateTime(); + $entry->setCreatedAt($dt->setTimestamp($data['created_at'])); + } + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php new file mode 100644 index 000000000..d7620bcb7 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -0,0 +1,53 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => substr($entry['date_added'], 0, 10), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php new file mode 100644 index 000000000..e010f5a46 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -0,0 +1,53 @@ + $entry['title'], + 'html' => '', + 'url' => $entry['uri'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => substr($entry['dateAdded'], 0, 10), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index aa049749e..6ada63025 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -28,3 +28,17 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" - "@logger" + wallabag_import.consumer.amqp.firefox: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.firefox.import" + - "@logger" + wallabag_import.consumer.amqp.chrome: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.chrome.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 7d3248e5e..c9c2cf26e 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -79,3 +79,43 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" - "@logger" + + # firefox + wallabag_import.queue.redis.firefox: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.firefox" + + wallabag_import.producer.redis.firefox: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.firefox" + + wallabag_import.consumer.redis.firefox: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.firefox.import" + - "@logger" + + # chrome + wallabag_import.queue.redis.chrome: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.chrome" + + wallabag_import.producer.redis.chrome: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.chrome" + + wallabag_import.consumer.redis.chrome: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.chrome.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index f03404ae5..990f336df 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -56,3 +56,22 @@ services: - [ setLogger, [ "@logger" ]] tags: - { name: wallabag_import.import, alias: readability } + + wallabag_import.firefox.import: + class: Wallabag\ImportBundle\Import\FirefoxImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: firefox } + wallabag_import.chrome.import: + class: Wallabag\ImportBundle\Import\ChromeImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: chrome } diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig new file mode 100644 index 000000000..ead828c6e --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.chrome.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans|raw }}
+

{{ 'import.chrome.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig new file mode 100644 index 000000000..f975da3fe --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.firefox.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans|raw }}
+

{{ 'import.firefox.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index aebbfa208..6ea5e0f42 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig @@ -11,7 +11,7 @@ {% for import in imports %}
  • {{ import.name }}
    -
    {{ import.description|trans }}
    +
    {{ import.description|trans|raw }}

    {{ 'import.action.import_contents'|trans }}

  • {% endfor %} diff --git a/tests/Wallabag/ApiBundle/Controller/WallabagRestControllerTest.php b/tests/Wallabag/ApiBundle/Controller/WallabagRestControllerTest.php index ee5b2ab7c..101c20eb6 100644 --- a/tests/Wallabag/ApiBundle/Controller/WallabagRestControllerTest.php +++ b/tests/Wallabag/ApiBundle/Controller/WallabagRestControllerTest.php @@ -80,7 +80,7 @@ class WallabagRestControllerTest extends WallabagApiTestCase public function testGetStarredEntries() { - $this->client->request('GET', '/api/entries', ['star' => 1, 'sort' => 'updated']); + $this->client->request('GET', '/api/entries', ['starred' => 1, 'sort' => 'updated']); $this->assertEquals(200, $this->client->getResponse()->getStatusCode()); diff --git a/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php b/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php index eb7fce79f..7be1eb18f 100644 --- a/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php +++ b/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php @@ -6,7 +6,6 @@ use Symfony\Bundle\FrameworkBundle\Console\Application; use Symfony\Component\Console\Tester\CommandTester; use Wallabag\ImportBundle\Command\ImportCommand; use Tests\Wallabag\CoreBundle\WallabagCoreTestCase; -use M6Web\Component\RedisMock\RedisMockFactory; class ImportCommandTest extends WallabagCoreTestCase { diff --git a/tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php new file mode 100644 index 000000000..23c80bec8 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php @@ -0,0 +1,152 @@ +logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportChromeWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/chrome'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportChromeBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + + public function testImportChromeWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/chrome'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/chrome-bookmarks', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.chrome')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + } + + public function testImportWallabagWithChromeFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/chrome-bookmarks', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'http://www.usinenouvelle.com/article/la-multiplication-des-chefs-de-projet-est-une-catastrophe-manageriale-majeure-affirme-le-sociologue-francois-dupuy.N307730', + $this->getLoggedInUserId() + ); + + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertNotEmpty($content->getLanguage()); + $this->assertEquals(0, count($content->getTags())); + + $createdAt = $content->getCreatedAt(); + $this->assertEquals('2011', $createdAt->format('Y')); + $this->assertEquals('07', $createdAt->format('m')); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/test.txt', 'test.txt'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php new file mode 100644 index 000000000..98f13d728 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php @@ -0,0 +1,165 @@ +logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/firefox'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportFirefoxWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/firefox'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportFirefoxBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/firefox'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + + public function testImportFirefoxWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/firefox'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/firefox-bookmarks.json', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.firefox')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + } + + public function testImportWallabagWithFirefoxFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/firefox'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/firefox-bookmarks.json', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'http://lexpansion.lexpress.fr/high-tech/orange-offre-un-meilleur-reseau-mobile-que-bouygues-et-sfr-free-derriere_1811554.html', + $this->getLoggedInUserId() + ); + + $this->assertNotEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertNotEmpty($content->getLanguage()); + $this->assertEquals(2, count($content->getTags())); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'http://stackoverflow.com/questions/15017163/parser-for-exported-bookmarks-html-file-of-google-chrome-and-mozilla-in-java', + $this->getLoggedInUserId() + ); + + $this->assertNotEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertEmpty($content->getLanguage()); + + $createdAt = $content->getCreatedAt(); + $this->assertEquals('2013', $createdAt->format('Y')); + $this->assertEquals('12', $createdAt->format('m')); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/firefox'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/test.txt', 'test.txt'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php index d869cdf9c..b6783a560 100644 --- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php @@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase $crawler = $client->request('GET', '/import/'); $this->assertEquals(200, $client->getResponse()->getStatusCode()); - $this->assertEquals(4, $crawler->filter('blockquote')->count()); + $this->assertEquals(6, $crawler->filter('blockquote')->count()); } } diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index 87ecb9d33..916dd2978 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -57,7 +57,6 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $this->checkRedis(); $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('import_with_redis', 1); $crawler = $client->request('GET', '/import/readability'); diff --git a/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php new file mode 100644 index 000000000..1e52615c0 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php @@ -0,0 +1,233 @@ +user = new User(); + + $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') + ->disableOriginalConstructor() + ->getMock(); + + $wallabag = new ChromeImport($this->em, $this->contentProxy); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + $wallabag->setLogger($logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } + + public function testInit() + { + $chromeImport = $this->getChromeImport(); + + $this->assertEquals('Chrome', $chromeImport->getName()); + $this->assertNotEmpty($chromeImport->getUrl()); + $this->assertEquals('import.chrome.description', $chromeImport->getDescription()); + } + + public function testImport() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(1)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $chromeImport->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 1, 'queued' => 0], $chromeImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(1)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return $persistedEntry->isArchived(); + })); + + $res = $chromeImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + + $this->assertEquals(['skipped' => 0, 'imported' => 1, 'queued' => 0], $chromeImport->getSummary()); + } + + public function testImportWithRabbit() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(1)) + ->method('publish'); + + $chromeImport->setProducer($producer); + + $res = $chromeImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $chromeImport->getSummary()); + } + + public function testImportWithRedis() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'chrome'); + $producer = new Producer($queue); + + $chromeImport->setProducer($producer); + + $res = $chromeImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $chromeImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('chrome')); + } + + public function testImportBadFile() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx'); + + $res = $chromeImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('Wallabag Browser Import: unable to read file', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $chromeImport = $this->getChromeImport(true); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $res = $chromeImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('Wallabag Browser Import: user is not defined', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } +} diff --git a/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php new file mode 100644 index 000000000..007dda6a2 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php @@ -0,0 +1,233 @@ +user = new User(); + + $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') + ->disableOriginalConstructor() + ->getMock(); + + $wallabag = new FirefoxImport($this->em, $this->contentProxy); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + $wallabag->setLogger($logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } + + public function testInit() + { + $firefoxImport = $this->getFirefoxImport(); + + $this->assertEquals('Firefox', $firefoxImport->getName()); + $this->assertNotEmpty($firefoxImport->getUrl()); + $this->assertEquals('import.firefox.description', $firefoxImport->getDescription()); + } + + public function testImport() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(2)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $firefoxImport->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $firefoxImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return $persistedEntry->isArchived(); + })); + + $res = $firefoxImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + + $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $firefoxImport->getSummary()); + } + + public function testImportWithRabbit() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(1)) + ->method('publish'); + + $firefoxImport->setProducer($producer); + + $res = $firefoxImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $firefoxImport->getSummary()); + } + + public function testImportWithRedis() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'firefox'); + $producer = new Producer($queue); + + $firefoxImport->setProducer($producer); + + $res = $firefoxImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $firefoxImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('firefox')); + } + + public function testImportBadFile() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx'); + + $res = $firefoxImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('Wallabag Browser Import: unable to read file', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $firefoxImport = $this->getFirefoxImport(true); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $res = $firefoxImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('Wallabag Browser Import: user is not defined', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } +} diff --git a/tests/Wallabag/ImportBundle/fixtures/chrome-bookmarks b/tests/Wallabag/ImportBundle/fixtures/chrome-bookmarks new file mode 100644 index 000000000..0478eb41c --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/chrome-bookmarks @@ -0,0 +1,36 @@ +{ + "checksum": "f3aa0e9c0edad632a246f7e98ec64918", + "roots": { + "bookmark_bar": { + "children": [ { + "date_added": "13118850929335823", + "id": "6", + "name": "\"La multiplication des chefs de projet est une catastrophe managériale majeure\", affirme le sociologue François Dupuy - Ressources humaines", + "type": "url", + "url": "http://www.usinenouvelle.com/article/la-multiplication-des-chefs-de-projet-est-une-catastrophe-manageriale-majeure-affirme-le-sociologue-francois-dupuy.N307730" + } ], + "date_added": "13118829474385693", + "date_modified": "13118850929335823", + "id": "1", + "name": "Barre de favoris", + "type": "folder" + }, + "other": { + "children": [ ], + "date_added": "13118829474385701", + "date_modified": "0", + "id": "2", + "name": "Autres favoris", + "type": "folder" + }, + "synced": { + "children": [ ], + "date_added": "13118829474385702", + "date_modified": "0", + "id": "3", + "name": "Favoris sur mobile", + "type": "folder" + } + }, + "version": 1 +} diff --git a/tests/Wallabag/ImportBundle/fixtures/firefox-bookmarks.json b/tests/Wallabag/ImportBundle/fixtures/firefox-bookmarks.json new file mode 100644 index 000000000..406b56976 --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/firefox-bookmarks.json @@ -0,0 +1,63 @@ +{ + "guid": "root________", + "title": "", + "index": 0, + "dateAdded": 1388166091504000, + "lastModified": 1472897622350000, + "id": 1, + "type": "text/x-moz-place-container", + "root": "placesRoot", + "children": [ + { + "guid": "toolbar_____", + "title": "Barre personnelle", + "index": 1, + "dateAdded": 1388166091504000, + "lastModified": 1472897622263000, + "id": 3, + "annos": [ + { + "name": "bookmarkProperties/description", + "flags": 0, + "expires": 4, + "value": "Ajoutez des marque-pages dans ce dossier pour les voir apparaître sur votre barre personnelle" + } + ], + "type": "text/x-moz-place-container", + "root": "toolbarFolder", + "children": [ + { + "guid": "tard77lzbC5H", + "title": "Orange offre un meilleur réseau mobile que Bouygues et SFR, Free derrière - L'Express L'Expansion", + "index": 1, + "dateAdded": 1388166091644000, + "lastModified": 1388166091644000, + "tags":"test,tag", + "id": 4, + "type": "text/x-moz-place", + "uri": "http://lexpansion.lexpress.fr/high-tech/orange-offre-un-meilleur-reseau-mobile-que-bouygues-et-sfr-free-derriere_1811554.html" + }, + { + "guid": "E385l9vZ_LVn", + "title": "Parser for Exported Bookmarks HTML file of Google Chrome and Mozilla in Java", + "index": 1, + "dateAdded": 1388166091544000, + "lastModified": 1388166091545000, + "id": 5, + "type": "text/x-moz-place", + "uri": "http://stackoverflow.com/questions/15017163/parser-for-exported-bookmarks-html-file-of-google-chrome-and-mozilla-in-java" + } + ] + }, + { + "guid": "unfiled_____", + "title": "Autres marque-pages", + "index": 3, + "dateAdded": 1388166091504000, + "lastModified": 1388166091542000, + "id": 6, + "type": "text/x-moz-place-container", + "root": "unfiledBookmarksFolder" + } + ] +}