oss/wallabag
oss/wallabag
1
0
Fork 0
mirror of https://github.com/wallabag/wallabag.git synced 2025-07-22 17:18:37 +00:00
Code Wiki Activity

Merge pull request #1941 from wallabag/v2-asynchronous-jobs

Browse source 
Use asynchronous jobs for imports
This commit is contained in:
Jeremy Benoist 2016-09-19 07:15:40 +02:00 • committed by GitHub
commit da18a4682f
85 changed files with 2905 additions and 466 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/Wallabag/CoreBundle/Command/InstallCommand.php
View file

@ -317,8 +317,13 @@ class InstallCommand extends ContainerAwareCommand
'section' => 'export',
],
[
'name' => 'pocket_consumer_key',
'value' => null,
'name' => 'import_with_redis',
'value' => '0',
'section' => 'import',
],
[
'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],
[

50
src/Wallabag/CoreBundle/Controller/EntryController.php
View file

@ -17,26 +17,35 @@ use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache;
class EntryController extends Controller
{
/**
* @param Entry $entry
* Fetch content and update entry.
* In case it fails, entry will return to avod loosing the data.
*
* @param Entry $entry
* @param string $prefixMessage Should be the translation key: entry_saved or entry_reloaded
*
* @return Entry
*/
private function updateEntry(Entry $entry)
private function updateEntry(Entry $entry, $prefixMessage = 'entry_saved')
{
// put default title in case of fetching content failed
$entry->setTitle('No title found');
$message = 'flashes.entry.notice.'.$prefixMessage;
try {
$entry = $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl());
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
} catch (\Exception $e) {
$this->get('logger')->error('Error while saving an entry', [
'exception' => $e,
'entry' => $entry,
]);
return false;
$message = 'flashes.entry.notice.'.$prefixMessage.'_failed';
}
return true;
$this->get('session')->getFlashBag()->add('notice', $message);
return $entry;
}
/**
@ -66,12 +75,11 @@ class EntryController extends Controller
return $this->redirect($this->generateUrl('view', ['id' => $existingEntry->getId()]));
}
$message = 'flashes.entry.notice.entry_saved';
if (false === $this->updateEntry($entry)) {
$message = 'flashes.entry.notice.entry_saved_failed';
}
$this->updateEntry($entry);
$this->get('session')->getFlashBag()->add('notice', $message);
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
return $this->redirect($this->generateUrl('homepage'));
}
@ -95,6 +103,10 @@ class EntryController extends Controller
if (false === $this->checkIfEntryAlreadyExists($entry)) {
$this->updateEntry($entry);
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
}
return $this->redirect($this->generateUrl('homepage'));
@ -316,15 +328,11 @@ class EntryController extends Controller
{
$this->checkUserAction($entry);
$message = 'flashes.entry.notice.entry_reloaded';
if (false === $this->updateEntry($entry)) {
$message = 'flashes.entry.notice.entry_reload_failed';
}
$this->updateEntry($entry, 'entry_reloaded');
$this->get('session')->getFlashBag()->add(
'notice',
$message
);
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()]));
}

3
src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php
View file

@ -20,6 +20,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
$adminConfig->setItemsPerPage(30);
$adminConfig->setReadingSpeed(1);
$adminConfig->setLanguage('en');
$adminConfig->setPocketConsumerKey('xxxxx');
$manager->persist($adminConfig);
@ -30,6 +31,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
$bobConfig->setItemsPerPage(10);
$bobConfig->setReadingSpeed(1);
$bobConfig->setLanguage('fr');
$bobConfig->setPocketConsumerKey(null);
$manager->persist($bobConfig);
@ -40,6 +42,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
$emptyConfig->setItemsPerPage(10);
$emptyConfig->setReadingSpeed(1);
$emptyConfig->setLanguage('en');
$emptyConfig->setPocketConsumerKey(null);
$manager->persist($emptyConfig);

9
src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php
View file

@ -91,8 +91,13 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
'section' => 'export',
],
[
'name' => 'pocket_consumer_key',
'value' => null,
'name' => 'import_with_redis',
'value' => '0',
'section' => 'import',
],
[
'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],
[

31
src/Wallabag/CoreBundle/Entity/Config.php
View file

@ -80,6 +80,13 @@ class Config
*/
private $readingSpeed;
/**
* @var string
*
* @ORM\Column(name="pocket_consumer_key", type="string", nullable=true)
*/
private $pocketConsumerKey;
/**
* @ORM\OneToOne(targetEntity="Wallabag\UserBundle\Entity\User", inversedBy="config")
*/
@ -278,6 +285,30 @@ class Config
return $this->readingSpeed;
}
/**
* Set pocketConsumerKey.
*
* @param string $pocketConsumerKey
*
* @return Config
*/
public function setPocketConsumerKey($pocketConsumerKey)
{
$this->pocketConsumerKey = $pocketConsumerKey;
return $this;
}
/**
* Get pocketConsumerKey.
*
* @return string
*/
public function getPocketConsumerKey()
{
return $this->pocketConsumerKey;
}
/**
* @param TaggingRule $rule
*

23
src/Wallabag/CoreBundle/Entity/Entry.php
View file

@ -97,7 +97,7 @@ class Entry
private $content;
/**
* @var date
* @var \DateTime
*
* @ORM\Column(name="created_at", type="datetime")
*
@ -106,7 +106,7 @@ class Entry
private $createdAt;
/**
* @var date
* @var \DateTime
*
* @ORM\Column(name="updated_at", type="datetime")
*
@ -410,7 +410,22 @@ class Entry
}
/**
* @return string
* Set created_at.
* Only used when importing data from an other service.
*
* @param \DateTime $createdAt
*
* @return Entry
*/
public function setCreatedAt(\DateTime $createdAt)
{
$this->createdAt = $createdAt;
return $this;
}
/**
* @return \DateTime
*/
public function getCreatedAt()
{
@ -418,7 +433,7 @@ class Entry
}
/**
* @return string
* @return \DateTime
*/
public function getUpdatedAt()
{

3
src/Wallabag/CoreBundle/Form/Type/ConfigType.php
View file

@ -52,6 +52,9 @@ class ConfigType extends AbstractType
'choices' => array_flip($this->languages),
'label' => 'config.form_settings.language_label',
])
->add('pocket_consumer_key', null, [
'label' => 'config.form_settings.pocket_consumer_key_label',
])
->add('save', SubmitType::class, [
'label' => 'config.form.save',
])

8
src/Wallabag/CoreBundle/Resources/config/services.yml
View file

@ -125,3 +125,11 @@ services:
arguments:
- "@security.token_storage"
- "@router"
wallabag_core.redis.client:
class: Predis\Client
arguments:
-
host: '%redis_host%'
port: '%redis_port%'
schema: tcp

8
src/Wallabag/CoreBundle/Resources/translations/messages.da.yml
View file

@ -68,6 +68,7 @@ config:
# 200_word: 'I read ~200 words per minute'
# 300_word: 'I read ~300 words per minute'
# 400_word: 'I read ~400 words per minute'
pocket_consumer_key_label: Brugers nøgle til Pocket for at importere materialer
form_rss:
description: 'RSS-feeds fra wallabag gør det muligt at læse de artikler, der gemmes i wallabag, med din RSS-læser. Det kræver, at du genererer et token først.'
token_label: 'RSS-Token'
@ -346,6 +347,8 @@ import:
# page_title: 'Import > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
# entry_already_saved: 'Entry already saved on %date%'
# entry_saved: 'Entry saved'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
# entry_updated: 'Entry updated'
# entry_reloaded: 'Entry reloaded'
# entry_reload_failed: 'Failed to reload entry'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Artikel arkiveret'
entry_unarchived: 'Artikel ikke længere arkiveret'
entry_starred: 'Artikel markeret som favorit'
@ -428,6 +431,7 @@ flashes:
# failed: 'Import failed, please try again.'
# failed_on_file: 'Error while processing import. Please verify your import file.'
# summary: 'Import summary: %imported% imported, %skipped% already saved.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.de.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Ich lese ~200 Wörter pro Minute'
300_word: 'Ich lese ~300 Wörter pro Minute'
400_word: 'Ich lese ~400 Wörter pro Minute'
pocket_consumer_key_label: Consumer-Key für Pocket, um Inhalte zu importieren
form_rss:
description: 'Die RSS-Feeds von wallabag erlauben es dir, deine gespeicherten Artikel mit deinem bevorzugten RSS-Reader zu lesen. Vorher musst du jedoch einen Token erstellen.'
token_label: 'RSS-token'
@ -346,6 +347,8 @@ import:
page_title: 'Aus Readability importieren'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Entwickler'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Eintrag bereits am %date% gespeichert'
entry_saved: 'Eintrag gespeichert'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Eintrag aktualisiert'
entry_reloaded: 'Eintrag neugeladen'
entry_reload_failed: 'Neuladen des Eintrags fehlgeschlagen'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Artikel archiviert'
entry_unarchived: 'Artikel dearchiviert'
entry_starred: 'Artikel favorisiert'
@ -428,6 +431,7 @@ flashes:
failed: 'Import fehlgeschlagen, bitte erneut probieren.'
failed_on_file: 'Fehler während des Imports. Bitte überprüfe deine Import-Datei.'
summary: 'Import-Zusammenfassung: %imported% importiert, %skipped% bereits gespeichert.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Neuer Client erstellt.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'I read ~200 words per minute'
300_word: 'I read ~300 words per minute'
400_word: 'I read ~400 words per minute'
pocket_consumer_key_label: Consumer key for Pocket to import contents
form_rss:
description: 'RSS feeds provided by wallabag allow you to read your saved articles with your favourite RSS reader. You need to generate a token first.'
token_label: 'RSS token'
@ -346,6 +347,8 @@ import:
page_title: 'Import > Readability'
description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Developer'
@ -413,10 +416,10 @@ flashes:
notice:
entry_already_saved: 'Entry already saved on %date%'
entry_saved: 'Entry saved'
entry_saved_failed: 'Failed to save entry'
entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Entry updated'
entry_reloaded: 'Entry reloaded'
entry_reload_failed: 'Failed to reload entry'
entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Entry archived'
entry_unarchived: 'Entry unarchived'
entry_starred: 'Entry starred'
@ -430,6 +433,7 @@ flashes:
failed: 'Import failed, please try again.'
failed_on_file: 'Error while processing import. Please verify your import file.'
summary: 'Import summary: %imported% imported, %skipped% already saved.'
summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'New client %name% created.'

14
src/Wallabag/CoreBundle/Resources/translations/messages.es.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Leo ~200 palabras por minuto'
300_word: 'Leo ~300 palabras por minuto'
400_word: 'Leo ~400 palabras por minuto'
# pocket_consumer_key_label: Consumer key for Pocket to import contents
form_rss:
description: 'Los feeds RSS de wallabag permiten leer los artículos guardados con su lector RSS favorito. Necesita generar un token primero'
token_label: 'RSS token'
@ -346,6 +347,8 @@ import:
page_title: 'Importar > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Promotor'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Entrada ya guardada por %fecha%'
entry_saved: 'Entrada guardada'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Entrada actualizada'
entry_reloaded: 'Entrada recargada'
entry_reload_failed: 'Entrada recargada reprobada'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Artículo archivado'
entry_unarchived: 'Artículo desarchivado'
entry_starred: 'Artículo guardado en los favoritos'
@ -425,9 +428,10 @@ flashes:
tag_added: 'Etiqueta añadida'
import:
notice:
failed: 'Importación reprobada, por favor inténtelo de nuevo.'
failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.'
summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.'
failed: 'Importación reprobada, por favor inténtelo de nuevo.'
failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.'
summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Nuevo cliente creado.'

12
src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'من تقریباً ۲۰۰ واژه را در دقیقه می‌خوانم'
300_word: 'من تقریباً ۳۰۰ واژه را در دقیقه می‌خوانم'
400_word: 'من تقریباً ۴۰۰ واژه را در دقیقه می‌خوانم'
pocket_consumer_key_label: کلید کاربری Pocket برای درون‌ریزی مطالب
form_rss:
description: 'با خوراک آر-اس-اس که wallabag در اختیارتان می‌گذارد، می‌توانید مقاله‌های ذخیره‌شده را در نرم‌افزار آر-اس-اس دلخواه خود بخوانید. برای این کار نخست باید یک کد بسازید.'
token_label: 'کد آر-اس-اس'
@ -344,8 +345,10 @@ import:
description: 'این برنامه همهٔ داده‌های شما را در نسخهٔ ۲ wallabag درون‌ریزی می‌کند. به بخش «همهٔ مقاله‌ها» بروید و در بخش «برون‌ریزی» روی "JSON" کلیک کنید. با این کار شما پرونده‌ای به شکل "All articles.json" دریافت خواهید کرد.'
readability:
page_title: 'درون‌ریزی > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'این مقاله در تاریخ %date% ذخیره شده بود'
entry_saved: 'مقاله ذخیره شد'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'مقاله به‌روز شد'
entry_reloaded: 'مقاله به‌روز شد'
entry_reload_failed: 'به‌روزرسانی مقاله شکست خورد'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'مقاله بایگانی شد'
entry_unarchived: 'مقاله از بایگانی درآمد'
entry_starred: 'مقاله برگزیده شد'
@ -428,6 +431,7 @@ flashes:
failed: 'درون‌ریزی شکست خورد. لطفاً دوباره تلاش کنید.'
failed_on_file: 'خطا هنگام پردازش پروندهٔ ورودی. آیا پروندهٔ درون‌ریزی شده سالم است؟'
summary: 'گزارش درون‌ریزی: %imported% وارد شد, %skipped% از قبل ذخیره شده بود.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Je lis environ 200 mots par minute'
300_word: 'Je lis environ 300 mots par minute'
400_word: 'Je lis environ 400 mots par minute'
pocket_consumer_key_label: Clé d'authentification Pocket pour importer les données
form_rss:
description: "Les flux RSS fournis par wallabag vous permettent de lire vos articles sauvegardés dans votre lecteur de flux préféré. Pour pouvoir les utiliser, vous devez d'abord créer un jeton."
token_label: 'Jeton RSS'
@ -346,6 +347,8 @@ import:
page_title: 'Importer > Readability'
description: 'Cet outil va importer toutes vos données de Readability. Sur la page des outils (https://www.readability.com/tools/), cliquez sur "Export your data" dans la section "Data Export". Vous allez recevoir un email avec un lien pour télécharger le json.'
how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer."
worker:
enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :"
developer:
page_title: 'Développeur'
@ -413,10 +416,10 @@ flashes:
notice:
entry_already_saved: 'Article déjà sauvergardé le %date%'
entry_saved: 'Article enregistré'
entry_saved_failed: "L'enregistrement a échoué"
entry_saved_failed: 'Article enregistré mais impossible de récupérer le contenu'
entry_updated: 'Article mis à jour'
entry_reloaded: 'Article rechargé'
entry_reload_failed: "Le rechargement de l'article a échoué"
entry_reload_failed: "Article mis à jour mais impossible de récupérer le contenu"
entry_archived: 'Article marqué comme lu'
entry_unarchived: 'Article marqué comme non lu'
entry_starred: 'Article ajouté dans les favoris'
@ -430,6 +433,7 @@ flashes:
failed: "L'import a échoué, veuillez ré-essayer"
failed_on_file: "Erreur lors du traitement de l'import. Vérifier votre fichier."
summary: "Rapport d'import: %imported% importés, %skipped% déjà présent."
summary_with_queue: "Rapport d'import: %queued% en cours de traitement."
developer:
notice:
client_created: 'Nouveau client %name% créé'

12
src/Wallabag/CoreBundle/Resources/translations/messages.it.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Leggo ~200 parole al minuto'
300_word: 'Leggo ~300 parole al minuto'
400_word: 'Leggo ~400 parole al minuto'
pocket_consumer_key_label: Consumer key per Pocket per importare i contenuti
form_rss:
description: 'I feed RSS generati da wallabag ti permettono di leggere i tuoi contenuti salvati con il tuo lettore di RSS preferito. Prima, devi generare un token.'
token_label: 'RSS token'
@ -343,8 +344,10 @@ import:
description: 'Questo importatore copierà tutti i tuoi dati da un wallabag v2. Vai in "Tutti i contenuti", e, nella sidebar di esportazione, clicca su "JSON". Otterrai un file "Tutti i contenuti.json".'
readability:
page_title: 'Importa da > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Sviluppatori'
@ -410,10 +413,10 @@ flashes:
notice:
entry_already_saved: 'Contenuto già salvato in data %date%'
entry_saved: 'Contenuto salvato'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Contenuto aggiornato'
entry_reloaded: 'Contenuto ricaricato'
entry_reload_failed: 'Errore nel ricaricamento del contenuto'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Contenuto archiviato'
entry_unarchived: 'Contenuto dis-archiviato'
entry_starred: 'Contenuto segnato come preferito'
@ -427,6 +430,7 @@ flashes:
failed: 'Importazione fallita, riprova.'
failed_on_file: 'Errore durante la processazione dei dati da importare. Verifica il tuo file di import.'
summary: 'Sommario di importazione: %imported% importati, %skipped% già salvati.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Nuovo client creato.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml
View file

@ -68,6 +68,7 @@ config:
200_word: "Legissi a l'entorn de 200 mots per minuta"
300_word: "Legissi a l'entorn de 300 mots per minuta"
400_word: "Legissi a l'entorn de 400 mots per minuta"
pocket_consumer_key_label: Clau d'autentificacion Pocket per importar las donadas
form_rss:
description: "Los fluxes RSS fornits per wallabag vos permeton de legir vòstres articles salvagardats dins vòstre lector de fluxes preferit. Per los poder emplegar, vos cal, d'en primièr crear un geton."
token_label: 'Geton RSS'
@ -346,6 +347,8 @@ import:
page_title: 'Importer > Readability'
description: "Aquesta aisina importarà totas vòstres articles de Readability. Sus la pagina de l'aisina (https://www.readability.com/tools/), clicatz sus \"Export your data\" dins la seccion \"Data Export\". Recebretz un corrièl per telecargar un json (qu'acaba pas amb un .json de fach)."
how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar."
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Desvolopador'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Article ja salvargardat lo %date%'
entry_saved: 'Article enregistrat'
entry_saved_failed: "Fracàs de l'enregistrament de l'entrada"
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Article mes a jorn'
entry_reloaded: 'Article recargat'
entry_reload_failed: "Fracàs de l'actualizacion de l'article"
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Article marcat coma legit'
entry_unarchived: 'Article marcat coma pas legit'
entry_starred: 'Article apondut dins los favorits'
@ -428,6 +431,7 @@ flashes:
failed: "L'importacion a fracassat, mercés de tornar ensajar"
failed_on_file: "Errorr pendent du tractament de l'import. Mercés de verificar vòstre fichièr."
summary: "Rapòrt d'import: %imported% importats, %skipped% ja presents."
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Novèl client creat'

8
src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Czytam ~200 słów na minutę'
300_word: 'Czytam ~300 słów na minutę'
400_word: 'Czytam ~400 słów na minutę'
pocket_consumer_key_label: Klucz klienta Pocket do importu zawartości
form_rss:
description: 'Kanały RSS prowadzone przez wallabag pozwalają Ci na czytanie twoich zapisanych artykułów w twoium ulubionym czytniku RSS. Musisz najpierw wynegenerować tokena.‌'
token_label: 'Token RSS'
@ -346,6 +347,8 @@ import:
page_title: 'Import > Readability'
description: 'Ten importer, zaimportuje wszystkie twoje artykuły z Readability. Na stronie narzędzi (https://www.readability.com/tools/), kliknij na "Export your data" w sekcji "Data Export". Otrzymach email z plikiem JSON (plik nie będzie zawierał rozszerzenia .json).'
how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Deweloper'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Wpis już został dodany %date%'
entry_saved: 'Wpis zapisany'
entry_saved_failed: 'Zapis artykułu się nie powiódł'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Wpis zaktualizowany'
entry_reloaded: 'Wpis ponownie załadowany'
entry_reload_failed: 'Błąd ponownego załadowania'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Wpis dodany do archiwum'
entry_unarchived: 'Wpis usunięty z archiwum'
entry_starred: 'Wpis oznaczony gwiazdką'
@ -428,6 +431,7 @@ flashes:
failed: 'Nieudany import, prosimy spróbować ponownie.'
failed_on_file: 'Błąd podczas ptrzetwarzania pliku. Sprawdż swój importowany plik.'
summary: 'Podsumowanie importu: %imported% zaimportowane, %skipped% już zapisane.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Nowy klient utworzony.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml
View file

@ -68,6 +68,7 @@ config:
# 200_word: 'I read ~200 words per minute'
# 300_word: 'I read ~300 words per minute'
# 400_word: 'I read ~400 words per minute'
pocket_consumer_key_label: Cheie consumator pentru importarea contentului din Pocket
form_rss:
description: 'Feed-urile RSS oferite de wallabag îți permit să-ți citești articolele salvate în reader-ul tău preferat RSS.'
token_label: 'RSS-Token'
@ -346,6 +347,8 @@ import:
# page_title: 'Import > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
# entry_already_saved: 'Entry already saved on %date%'
# entry_saved: 'Entry saved'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
# entry_updated: 'Entry updated'
# entry_reloaded: 'Entry reloaded'
# entry_reload_failed: 'Failed to reload entry'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Articol arhivat'
entry_unarchived: 'Articol dezarhivat'
entry_starred: 'Articol adăugat la favorite'
@ -428,6 +431,7 @@ flashes:
# failed: 'Import failed, please try again.'
# failed_on_file: 'Error while processing import. Please verify your import file.'
# summary: 'Import summary: %imported% imported, %skipped% already saved.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

12
src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml
View file

@ -68,6 +68,7 @@ config:
# 200_word: 'I read ~200 words per minute'
# 300_word: 'I read ~300 words per minute'
# 400_word: 'I read ~400 words per minute'
# pocket_consumer_key_label: Consumer key for Pocket to import contents
form_rss:
description: 'wallabag RSS akışı kaydetmiş olduğunuz makalelerini favori RSS okuyucunuzda görüntülemenizi sağlar. Bunu yapabilmek için öncelikle belirteç (token) oluşturmalısınız.'
token_label: 'RSS belirteci (token)'
@ -344,8 +345,10 @@ import:
# description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.'
readability:
page_title: 'İçe Aktar > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Entry already saved on %date%'
entry_saved: 'Makale kaydedildi'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
# entry_updated: 'Entry updated'
entry_reloaded: 'Makale içeriği yenilendi'
# entry_reload_failed: 'Failed to reload entry'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Makale arşivlendi'
entry_unarchived: 'Makale arşivden çıkartıldı'
entry_starred: 'Makale favorilere eklendi'
@ -428,6 +431,7 @@ flashes:
# failed: 'Import failed, please try again.'
# failed_on_file: 'Error while processing import. Please verify your import file.'
# summary: 'Import summary: %imported% imported, %skipped% already saved.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

12
src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig
View file

@ -44,6 +44,18 @@
</div>
</fieldset>
<fieldset class="w500p inline">
<div class="row">
{{ form_label(form.config.pocket_consumer_key) }}
{{ form_errors(form.config.pocket_consumer_key) }}
{{ form_widget(form.config.pocket_consumer_key) }}
<p>
&raquo;
<a href="https://getpocket.com/developer/docs/authentication">https://getpocket.com/developer/docs/authentication</a>
</p>
</div>
</fieldset>
{{ form_rest(form.config) }}
</form>

12
src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig
View file

@ -62,6 +62,18 @@
</div>
</div>
<div class="row">
<div class="input-field col s12">
{{ form_label(form.config.pocket_consumer_key) }}
{{ form_errors(form.config.pocket_consumer_key) }}
{{ form_widget(form.config.pocket_consumer_key) }}
<p>
&raquo;
<a href="https://getpocket.com/developer/docs/authentication">https://getpocket.com/developer/docs/authentication</a>
</p>
</div>
</div>
{{ form_widget(form.config.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }}
{{ form_rest(form.config) }}
</form>

2
src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig
View file

@ -19,6 +19,8 @@
Materialize.toast('{{ flashMessage|trans }}', 4000);
</script>
{% endfor %}
{{ render(controller("WallabagImportBundle:Import:checkQueue")) }}
{% endblock %}
{% block menu %}

6
src/Wallabag/ImportBundle/Command/ImportCommand.php
View file

@ -26,6 +26,10 @@ class ImportCommand extends ContainerAwareCommand
{
$output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---');
if (!file_exists($input->getArgument('filepath'))) {
throw new Exception(sprintf('File "%s" not found', $input->getArgument('filepath')));
}
$em = $this->getContainer()->get('doctrine')->getManager();
// Turning off doctrine default logs queries for saving memory
$em->getConnection()->getConfiguration()->setSQLLogger(null);
@ -43,9 +47,9 @@ class ImportCommand extends ContainerAwareCommand
}
$wallabag->setMarkAsRead($input->getOption('markAsRead'));
$wallabag->setUser($user);
$res = $wallabag
->setUser($user)
->setFilepath($input->getArgument('filepath'))
->import();

44
src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php Normal file
View file

@ -0,0 +1,44 @@
<?php
namespace Wallabag\ImportBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Config\Definition\Exception\Exception;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Simpleue\Worker\QueueWorker;
class RedisWorkerCommand extends ContainerAwareCommand
{
protected function configure()
{
$this
->setName('wallabag:import:redis-worker')
->setDescription('Launch Redis worker')
->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false)
;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
$output->writeln('Waiting for message ...');
$serviceName = $input->getArgument('serviceName');
if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
}
$worker = new QueueWorker(
$this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
$this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName),
$input->getOption('maxIterations')
);
$worker->start();
}
}

17
src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php Normal file
View file

@ -0,0 +1,17 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
use PhpAmqpLib\Message\AMQPMessage;
class AMQPEntryConsumer extends AbstractConsumer implements ConsumerInterface
{
/**
* {@inheritdoc}
*/
public function execute(AMQPMessage $msg)
{
return $this->handleMessage($msg->body);
}
}

74
src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php Normal file
View file

@ -0,0 +1,74 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use Doctrine\ORM\EntityManager;
use Wallabag\ImportBundle\Import\AbstractImport;
use Wallabag\UserBundle\Repository\UserRepository;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
abstract class AbstractConsumer
{
protected $em;
protected $userRepository;
protected $import;
protected $logger;
public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
{
$this->em = $em;
$this->userRepository = $userRepository;
$this->import = $import;
$this->logger = $logger ?: new NullLogger();
}
/**
* Handle a message and save it.
*
* @param string $body Message from the queue (in json)
*
* @return bool
*/
protected function handleMessage($body)
{
$storedEntry = json_decode($body, true);
$user = $this->userRepository->find($storedEntry['userId']);
// no user? Drop message
if (null === $user) {
$this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
return false;
}
$this->import->setUser($user);
$entry = $this->import->parseEntry($storedEntry);
if (null === $entry) {
$this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
return false;
}
try {
$this->em->flush();
// clear only affected entities
$this->em->clear(Entry::class);
$this->em->clear(Tag::class);
} catch (\Exception $e) {
$this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
return false;
}
$this->logger->info('Content with url imported! ('.$entry->getUrl().')');
return true;
}
}

29
src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php Normal file
View file

@ -0,0 +1,29 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use Simpleue\Job\Job;
class RedisEntryConsumer extends AbstractConsumer implements Job
{
/**
* Handle one message by one message.
*
* @param string $job Content of the message (directly from Redis)
*
* @return bool
*/
public function manage($job)
{
return $this->handleMessage($job);
}
/**
* Should tell if the given job will kill the worker.
* We don't want to stop it :).
*/
public function isStopJob($job)
{
return false;
}
}

62
src/Wallabag/ImportBundle/Controller/ImportController.php
View file

@ -16,4 +16,66 @@ class ImportController extends Controller
'imports' => $this->get('wallabag_import.chain')->getAll(),
]);
}
/**
* Display how many messages are queue (both in Redis and RabbitMQ).
* Only for admins.
*/
public function checkQueueAction()
{
$nbRedisMessages = null;
$nbRabbitMessages = null;
if (!$this->get('security.authorization_checker')->isGranted('ROLE_SUPER_ADMIN')) {
return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
'nbRedisMessages' => $nbRedisMessages,
'nbRabbitMessages' => $nbRabbitMessages,
]);
}
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket')
+ $this->getTotalMessageInRabbitQueue('readability')
+ $this->getTotalMessageInRabbitQueue('wallabag_v1')
+ $this->getTotalMessageInRabbitQueue('wallabag_v2')
;
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$redis = $this->get('wallabag_core.redis.client');
$nbRedisMessages = $redis->llen('wallabag.import.pocket')
+ $redis->llen('wallabag.import.readability')
+ $redis->llen('wallabag.import.wallabag_v1')
+ $redis->llen('wallabag.import.wallabag_v2')
;
}
return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
'nbRedisMessages' => $nbRedisMessages,
'nbRabbitMessages' => $nbRabbitMessages,
]);
}
/**
* Count message in RabbitMQ queue.
* It get one message without acking it (so it'll stay in the queue)
* which will include the total of *other* messages in the queue.
* Adding one to that messages will result in the full total message.
*
* @param string $importService The import service related: pocket, readability, wallabag_v1 or wallabag_v2
*
* @return int
*/
private function getTotalMessageInRabbitQueue($importService)
{
$message = $this
->get('old_sound_rabbit_mq.import_'.$importService.'_consumer')
->getChannel()
->basic_get('wallabag.import.'.$importService);
if (null === $message) {
return 0;
}
return $message->delivery_info['message_count'] + 1;
}
}

35
src/Wallabag/ImportBundle/Controller/PocketController.php
View file

@ -10,12 +10,31 @@ use Symfony\Component\Form\Extension\Core\Type\CheckboxType;
class PocketController extends Controller
{
/**
* Return Pocket Import Service with or without RabbitMQ enabled.
*
* @return \Wallabag\ImportBundle\Import\PocketImport
*/
private function getPocketImportService()
{
$pocket = $this->get('wallabag_import.pocket.import');
$pocket->setUser($this->getUser());
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
}
return $pocket;
}
/**
* @Route("/pocket", name="import_pocket")
*/
public function indexAction()
{
$pocket = $this->get('wallabag_import.pocket.import');
$pocket = $this->getPocketImportService();
$form = $this->createFormBuilder($pocket)
->add('mark_as_read', CheckboxType::class, [
'label' => 'import.form.mark_as_read_label',
@ -24,8 +43,8 @@ class PocketController extends Controller
->getForm();
return $this->render('WallabagImportBundle:Pocket:index.html.twig', [
'import' => $this->get('wallabag_import.pocket.import'),
'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true,
'import' => $this->getPocketImportService(),
'has_consumer_key' => '' === trim($this->getUser()->getConfig()->getPocketConsumerKey()) ? false : true,
'form' => $form->createView(),
]);
}
@ -35,7 +54,7 @@ class PocketController extends Controller
*/
public function authAction(Request $request)
{
$requestToken = $this->get('wallabag_import.pocket.import')
$requestToken = $this->getPocketImportService()
->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL));
if (false === $requestToken) {
@ -62,7 +81,7 @@ class PocketController extends Controller
public function callbackAction()
{
$message = 'flashes.import.notice.failed';
$pocket = $this->get('wallabag_import.pocket.import');
$pocket = $this->getPocketImportService();
$markAsRead = $this->get('session')->get('mark_as_read');
$this->get('session')->remove('mark_as_read');
@ -83,6 +102,12 @@ class PocketController extends Controller
'%imported%' => $summary['imported'],
'%skipped%' => $summary['skipped'],
]);
if (0 < $summary['queued']) {
$message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
'%queued%' => $summary['queued'],
]);
}
}
$this->get('session')->getFlashBag()->add(

16
src/Wallabag/ImportBundle/Controller/ReadabilityController.php
View file

@ -18,15 +18,21 @@ class ReadabilityController extends Controller
$form->handleRequest($request);
$readability = $this->get('wallabag_import.readability.import');
$readability->setUser($this->getUser());
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
}
if ($form->isValid()) {
$file = $form->get('file')->getData();
$markAsRead = $form->get('mark_as_read')->getData();
$name = 'readability_'.$this->getUser()->getId().'.json';
if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
$res = $readability
->setUser($this->getUser())
->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
->setMarkAsRead($markAsRead)
->import();
@ -40,6 +46,12 @@ class ReadabilityController extends Controller
'%skipped%' => $summary['skipped'],
]);
if (0 < $summary['queued']) {
$message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
'%queued%' => $summary['queued'],
]);
}
unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
}

10
src/Wallabag/ImportBundle/Controller/WallabagController.php
View file

@ -38,15 +38,15 @@ abstract class WallabagController extends Controller
$form->handleRequest($request);
$wallabag = $this->getImportService();
$wallabag->setUser($this->getUser());
if ($form->isValid()) {
$file = $form->get('file')->getData();
$markAsRead = $form->get('mark_as_read')->getData();
$name = $this->getUser()->getId().'.json';
if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
$res = $wallabag
->setUser($this->getUser())
->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
->setMarkAsRead($markAsRead)
->import();
@ -60,6 +60,12 @@ abstract class WallabagController extends Controller
'%skipped%' => $summary['skipped'],
]);
if (0 < $summary['queued']) {
$message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
'%queued%' => $summary['queued'],
]);
}
unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
}

10
src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
View file

@ -12,7 +12,15 @@ class WallabagV1Controller extends WallabagController
*/
protected function getImportService()
{
return $this->get('wallabag_import.wallabag_v1.import');
$service = $this->get('wallabag_import.wallabag_v1.import');
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
}
return $service;
}
/**

10
src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
View file

@ -12,7 +12,15 @@ class WallabagV2Controller extends WallabagController
*/
protected function getImportService()
{
return $this->get('wallabag_import.wallabag_v2.import');
$service = $this->get('wallabag_import.wallabag_v2.import');
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
}
return $service;
}
/**

1
src/Wallabag/ImportBundle/Form/Type/UploadImportType.php
View file

@ -15,6 +15,7 @@ class UploadImportType extends AbstractType
$builder
->add('file', FileType::class, [
'label' => 'import.form.file_label',
'required' => true,
])
->add('mark_as_read', CheckboxType::class, [
'label' => 'import.form.mark_as_read_label',

144
src/Wallabag/ImportBundle/Import/AbstractImport.php
View file

@ -7,12 +7,21 @@ use Psr\Log\NullLogger;
use Doctrine\ORM\EntityManager;
use Wallabag\CoreBundle\Helper\ContentProxy;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Wallabag\UserBundle\Entity\User;
use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
abstract class AbstractImport implements ImportInterface
{
protected $em;
protected $logger;
protected $contentProxy;
protected $producer;
protected $user;
protected $markAsRead;
protected $skippedEntries = 0;
protected $importedEntries = 0;
protected $queuedEntries = 0;
public function __construct(EntityManager $em, ContentProxy $contentProxy)
{
@ -26,22 +35,151 @@ abstract class AbstractImport implements ImportInterface
$this->logger = $logger;
}
/**
* Set RabbitMQ/Redis Producer to send each entry to a queue.
* This method should be called when user has enabled RabbitMQ.
*
* @param ProducerInterface $producer
*/
public function setProducer(ProducerInterface $producer)
{
$this->producer = $producer;
}
/**
* Set current user.
* Could the current *connected* user or one retrieve by the consumer.
*
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
*/
public function setMarkAsRead($markAsRead)
{
$this->markAsRead = $markAsRead;
return $this;
}
/**
* Get whether articles must be all marked as read.
*/
public function getMarkAsRead()
{
return $this->markAsRead;
}
/**
* Fetch content from the ContentProxy (using graby).
* If it fails return false instead of the updated entry.
* If it fails return the given entry to be saved in all case (to avoid user to loose the content).
*
* @param Entry $entry Entry to update
* @param string $url Url to grab content for
* @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
*
* @return Entry|false
* @return Entry
*/
protected function fetchContent(Entry $entry, $url, array $content = [])
{
try {
return $this->contentProxy->updateEntry($entry, $url, $content);
} catch (\Exception $e) {
return false;
return $entry;
}
}
/**
* Parse and insert all given entries.
*
* @param $entries
*/
protected function parseEntries($entries)
{
$i = 1;
foreach ($entries as $importedEntry) {
$entry = $this->parseEntry($importedEntry);
if (null === $entry) {
continue;
}
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
// clear only affected entities
$this->em->clear(Entry::class);
$this->em->clear(Tag::class);
}
++$i;
}
$this->em->flush();
}
/**
* Parse entries and send them to the queue.
* It should just be a simple loop on all item, no call to the database should be done
* to speedup queuing.
*
* Faster parse entries for Producer.
* We don't care to make check at this time. They'll be done by the consumer.
*
* @param array $entries
*/
protected function parseEntriesForProducer(array $entries)
{
foreach ($entries as $importedEntry) {
// set userId for the producer (it won't know which user is connected)
$importedEntry['userId'] = $this->user->getId();
if ($this->markAsRead) {
$importedEntry = $this->setEntryAsRead($importedEntry);
}
++$this->queuedEntries;
$this->producer->publish(json_encode($importedEntry));
}
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
'queued' => $this->queuedEntries,
];
}
/**
* Parse one entry.
*
* @param array $importedEntry
*
* @return Entry
*/
abstract public function parseEntry(array $importedEntry);
/**
* Set current imported entry to archived / read.
* Implementation is different accross all imports.
*
* @param array $importedEntry
*
* @return array
*/
abstract protected function setEntryAsRead(array $importedEntry);
}

212
src/Wallabag/ImportBundle/Import/PocketImport.php
View file

@ -6,30 +6,33 @@ use Psr\Log\NullLogger;
use Doctrine\ORM\EntityManager;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Helper\ContentProxy;
use Craue\ConfigBundle\Util\Config;
class PocketImport extends AbstractImport
{
private $user;
private $client;
private $consumerKey;
private $skippedEntries = 0;
private $importedEntries = 0;
private $markAsRead;
protected $accessToken;
private $accessToken;
public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
const NB_ELEMENTS = 5000;
public function __construct(EntityManager $em, ContentProxy $contentProxy)
{
$this->user = $tokenStorage->getToken()->getUser();
$this->em = $em;
$this->contentProxy = $contentProxy;
$this->consumerKey = $craueConfig->get('pocket_consumer_key');
$this->logger = new NullLogger();
}
/**
* Only used for test purpose.
*
* @return string
*/
public function getAccessToken()
{
return $this->accessToken;
}
/**
* {@inheritdoc}
*/
@ -66,7 +69,7 @@ class PocketImport extends AbstractImport
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
[
'body' => json_encode([
'consumer_key' => $this->consumerKey,
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'redirect_uri' => $redirectUri,
]),
]
@ -96,7 +99,7 @@ class PocketImport extends AbstractImport
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
[
'body' => json_encode([
'consumer_key' => $this->consumerKey,
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'code' => $code,
]),
]
@ -115,39 +118,23 @@ class PocketImport extends AbstractImport
return true;
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
*/
public function setMarkAsRead($markAsRead)
{
$this->markAsRead = $markAsRead;
return $this;
}
/**
* Get whether articles must be all marked as read.
*/
public function getMarkAsRead()
{
return $this->markAsRead;
}
/**
* {@inheritdoc}
*/
public function import()
public function import($offset = 0)
{
static $run = 0;
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
[
'body' => json_encode([
'consumer_key' => $this->consumerKey,
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'access_token' => $this->accessToken,
'detailType' => 'complete',
'state' => 'all',
'sort' => 'oldest',
'sort' => 'newest',
'count' => self::NB_ELEMENTS,
'offset' => $offset,
]),
]
);
@ -162,22 +149,26 @@ class PocketImport extends AbstractImport
$entries = $response->json();
$this->parseEntries($entries['list']);
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
} else {
$this->parseEntries($entries['list']);
}
// if we retrieve exactly the amount of items requested it means we can get more
// re-call import and offset item by the amount previous received:
// - first call get 5k offset 0
// - second call get 5k offset 5k
// - and so on
if (count($entries['list']) === self::NB_ELEMENTS) {
++$run;
return $this->import(self::NB_ELEMENTS * $run);
}
return true;
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
];
}
/**
* Set the Guzzle client.
*
@ -189,77 +180,74 @@ class PocketImport extends AbstractImport
}
/**
* @see https://getpocket.com/developer/docs/v3/retrieve
* {@inheritdoc}
*
* @param $entries
* @see https://getpocket.com/developer/docs/v3/retrieve
*/
private function parseEntries($entries)
public function parseEntry(array $importedEntry)
{
$i = 1;
$url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
foreach ($entries as $pocketEntry) {
$url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($url, $this->user->getId());
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($url, $this->user->getId());
if (false !== $existingEntry) {
++$this->skippedEntries;
if (false !== $existingEntry) {
++$this->skippedEntries;
continue;
}
$entry = new Entry($this->user);
$entry = $this->fetchContent($entry, $url);
// jump to next entry in case of problem while getting content
if (false === $entry) {
++$this->skippedEntries;
continue;
}
// 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
if ($pocketEntry['status'] == 1 || $this->markAsRead) {
$entry->setArchived(true);
}
// 0 or 1 - 1 If the item is starred
if ($pocketEntry['favorite'] == 1) {
$entry->setStarred(true);
}
$title = 'Untitled';
if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
$title = $pocketEntry['resolved_title'];
} elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
$title = $pocketEntry['given_title'];
}
$entry->setTitle($title);
// 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
$entry->setPreviewPicture($pocketEntry['images'][1]['src']);
}
if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
$this->contentProxy->assignTagsToEntry(
$entry,
array_keys($pocketEntry['tags'])
);
}
$this->em->persist($entry);
++$this->importedEntries;
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
}
++$i;
return;
}
$this->em->flush();
$this->em->clear();
$entry = new Entry($this->user);
$entry->setUrl($url);
// update entry with content (in case fetching failed, the given entry will be return)
$entry = $this->fetchContent($entry, $url);
// 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
$entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
// 0 or 1 - 1 If the item is starred
$entry->setStarred($importedEntry['favorite'] == 1);
$title = 'Untitled';
if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
$title = $importedEntry['resolved_title'];
} elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
$title = $importedEntry['given_title'];
}
$entry->setTitle($title);
// 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
$entry->setPreviewPicture($importedEntry['images'][1]['src']);
}
if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
$this->contentProxy->assignTagsToEntry(
$entry,
array_keys($importedEntry['tags'])
);
}
if (!empty($importedEntry['time_added'])) {
$entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
}
$this->em->persist($entry);
++$this->importedEntries;
return $entry;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['status'] = '1';
return $importedEntry;
}
}

145
src/Wallabag/ImportBundle/Import/ReadabilityImport.php
View file

@ -3,28 +3,10 @@
namespace Wallabag\ImportBundle\Import;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\UserBundle\Entity\User;
class ReadabilityImport extends AbstractImport
{
private $user;
private $skippedEntries = 0;
private $importedEntries = 0;
private $filepath;
private $markAsRead;
/**
* We define the user in a custom call because on the import command there is no logged in user.
* So we can't retrieve user from the `security.token_storage` service.
*
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
return $this;
}
/**
* {@inheritdoc}
@ -62,37 +44,6 @@ class ReadabilityImport extends AbstractImport
return $this;
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
*/
public function setMarkAsRead($markAsRead)
{
$this->markAsRead = $markAsRead;
return $this;
}
/**
* Get whether articles must be all marked as read.
*/
public function getMarkAsRead()
{
return $this->markAsRead;
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
];
}
/**
* {@inheritdoc}
*/
@ -116,64 +67,66 @@ class ReadabilityImport extends AbstractImport
return false;
}
if ($this->producer) {
$this->parseEntriesForProducer($data['bookmarks']);
return true;
}
$this->parseEntries($data['bookmarks']);
return true;
}
/**
* Parse and insert all given entries.
*
* @param $entries
* {@inheritdoc}
*/
protected function parseEntries($entries)
public function parseEntry(array $importedEntry)
{
$i = 1;
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
foreach ($entries as $importedEntry) {
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
if (false !== $existingEntry) {
++$this->skippedEntries;
if (false !== $existingEntry) {
++$this->skippedEntries;
continue;
}
$data = [
'title' => $importedEntry['article__title'],
'url' => $importedEntry['article__url'],
'content_type' => '',
'language' => '',
'is_archived' => $importedEntry['archive'] || $this->markAsRead,
'is_starred' => $importedEntry['favorite'],
];
$entry = $this->fetchContent(
new Entry($this->user),
$data['url'],
$data
);
// jump to next entry in case of problem while getting content
if (false === $entry) {
++$this->skippedEntries;
continue;
}
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
$this->em->persist($entry);
++$this->importedEntries;
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
}
++$i;
return;
}
$this->em->flush();
$this->em->clear();
$data = [
'title' => $importedEntry['article__title'],
'url' => $importedEntry['article__url'],
'content_type' => '',
'language' => '',
'is_archived' => $importedEntry['archive'] || $this->markAsRead,
'is_starred' => $importedEntry['favorite'],
'created_at' => $importedEntry['date_added'],
];
$entry = new Entry($this->user);
$entry->setUrl($data['url']);
$entry->setTitle($data['title']);
// update entry with content (in case fetching failed, the given entry will be return)
$entry = $this->fetchContent($entry, $data['url'], $data);
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
$entry->setCreatedAt(new \DateTime($data['created_at']));
$this->em->persist($entry);
++$this->importedEntries;
return $entry;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['archive'] = 1;
return $importedEntry;
}
}

143
src/Wallabag/ImportBundle/Import/WallabagImport.php
View file

@ -3,15 +3,10 @@
namespace Wallabag\ImportBundle\Import;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\UserBundle\Entity\User;
abstract class WallabagImport extends AbstractImport
{
protected $user;
protected $skippedEntries = 0;
protected $importedEntries = 0;
protected $filepath;
protected $markAsRead;
// untitled in all languages from v1
protected $untitled = [
'Untitled',
@ -28,19 +23,6 @@ abstract class WallabagImport extends AbstractImport
'',
];
/**
* We define the user in a custom call because on the import command there is no logged in user.
* So we can't retrieve user from the `security.token_storage` service.
*
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
return $this;
}
/**
* {@inheritdoc}
*/
@ -79,22 +61,17 @@ abstract class WallabagImport extends AbstractImport
return false;
}
if ($this->producer) {
$this->parseEntriesForProducer($data);
return true;
}
$this->parseEntries($data);
return true;
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
];
}
/**
* Set file path to the json file.
*
@ -108,85 +85,59 @@ abstract class WallabagImport extends AbstractImport
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
* {@inheritdoc}
*/
public function setMarkAsRead($markAsRead)
public function parseEntry(array $importedEntry)
{
$this->markAsRead = $markAsRead;
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
return $this;
}
if (false !== $existingEntry) {
++$this->skippedEntries;
/**
* Parse and insert all given entries.
*
* @param $entries
*/
protected function parseEntries($entries)
{
$i = 1;
foreach ($entries as $importedEntry) {
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
if (false !== $existingEntry) {
++$this->skippedEntries;
continue;
}
$data = $this->prepareEntry($importedEntry, $this->markAsRead);
$entry = $this->fetchContent(
new Entry($this->user),
$importedEntry['url'],
$data
);
// jump to next entry in case of problem while getting content
if (false === $entry) {
++$this->skippedEntries;
continue;
}
if (array_key_exists('tags', $data)) {
$this->contentProxy->assignTagsToEntry(
$entry,
$data['tags']
);
}
if (isset($importedEntry['preview_picture'])) {
$entry->setPreviewPicture($importedEntry['preview_picture']);
}
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
$this->em->persist($entry);
++$this->importedEntries;
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
}
++$i;
return;
}
$this->em->flush();
$this->em->clear();
$data = $this->prepareEntry($importedEntry);
$entry = new Entry($this->user);
$entry->setUrl($data['url']);
$entry->setTitle($data['title']);
// update entry with content (in case fetching failed, the given entry will be return)
$entry = $this->fetchContent($entry, $data['url'], $data);
if (array_key_exists('tags', $data)) {
$this->contentProxy->assignTagsToEntry(
$entry,
$data['tags']
);
}
if (isset($importedEntry['preview_picture'])) {
$entry->setPreviewPicture($importedEntry['preview_picture']);
}
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
if (!empty($data['created_at'])) {
$entry->setCreatedAt(new \DateTime($data['created_at']));
}
$this->em->persist($entry);
++$this->importedEntries;
return $entry;
}
/**
* This should return a cleaned array for a given entry to be given to `updateEntry`.
*
* @param array $entry Data from the imported file
* @param bool $markAsRead Should we mark as read content?
* @param array $entry Data from the imported file
*
* @return array
*/
abstract protected function prepareEntry($entry = [], $markAsRead = false);
abstract protected function prepareEntry($entry = []);
}

15
src/Wallabag/ImportBundle/Import/WallabagV1Import.php
View file

@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport
/**
* {@inheritdoc}
*/
protected function prepareEntry($entry = [], $markAsRead = false)
protected function prepareEntry($entry = [])
{
$data = [
'title' => $entry['title'],
@ -39,9 +39,10 @@ class WallabagV1Import extends WallabagImport
'url' => $entry['url'],
'content_type' => '',
'language' => '',
'is_archived' => $entry['is_read'] || $markAsRead,
'is_archived' => $entry['is_read'] || $this->markAsRead,
'is_starred' => $entry['is_fav'],
'tags' => '',
'created_at' => '',
];
// force content to be refreshed in case on bad fetch in the v1 installation
@ -56,4 +57,14 @@ class WallabagV1Import extends WallabagImport
return $data;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['is_read'] = 1;
return $importedEntry;
}
}

14
src/Wallabag/ImportBundle/Import/WallabagV2Import.php
View file

@ -31,12 +31,22 @@ class WallabagV2Import extends WallabagImport
/**
* {@inheritdoc}
*/
protected function prepareEntry($entry = [], $markAsRead = false)
protected function prepareEntry($entry = [])
{
return [
'html' => $entry['content'],
'content_type' => $entry['mimetype'],
'is_archived' => ($entry['is_archived'] || $markAsRead),
'is_archived' => ($entry['is_archived'] || $this->markAsRead),
] + $entry;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['is_archived'] = 1;
return $importedEntry;
}
}

36
src/Wallabag/ImportBundle/Redis/Producer.php Normal file
View file

@ -0,0 +1,36 @@
<?php
namespace Wallabag\ImportBundle\Redis;
use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
use Simpleue\Queue\RedisQueue;
/**
* This is a proxy class for "Simpleue\Queue\RedisQueue".
* It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
*
* It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
* kind of class which implements the same interface.
* So we can inject either a RabbitMQ producer or a Redis producer with the same signature
*/
class Producer implements ProducerInterface
{
private $queue;
public function __construct(RedisQueue $queue)
{
$this->queue = $queue;
}
/**
* Publish a message in the Redis queue.
*
* @param string $msgBody
* @param string $routingKey NOT USED
* @param array $additionalProperties NOT USED
*/
public function publish($msgBody, $routingKey = '', $additionalProperties = array())
{
$this->queue->sendJob($msgBody);
}
}

30
src/Wallabag/ImportBundle/Resources/config/rabbit.yml Normal file
View file

@ -0,0 +1,30 @@
# RabbitMQ stuff
services:
wallabag_import.consumer.amqp.pocket:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
wallabag_import.consumer.amqp.readability:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
wallabag_import.consumer.amqp.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
wallabag_import.consumer.amqp.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"

81
src/Wallabag/ImportBundle/Resources/config/redis.yml Normal file
View file

@ -0,0 +1,81 @@
# Redis stuff
services:
# readability
wallabag_import.queue.redis.readability:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.readability"
wallabag_import.producer.redis.readability:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.readability"
wallabag_import.consumer.redis.readability:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
# pocket
wallabag_import.queue.redis.pocket:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.pocket"
wallabag_import.producer.redis.pocket:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.pocket"
wallabag_import.consumer.redis.pocket:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
# wallabag v1
wallabag_import.queue.redis.wallabag_v1:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.wallabag_v1"
wallabag_import.producer.redis.wallabag_v1:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.wallabag_v1"
wallabag_import.consumer.redis.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
# wallabag v2
wallabag_import.queue.redis.wallabag_v2:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.wallabag_v2"
wallabag_import.producer.redis.wallabag_v2:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.wallabag_v2"
wallabag_import.consumer.redis.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"

5
src/Wallabag/ImportBundle/Resources/config/services.yml
View file

@ -1,3 +1,7 @@
imports:
- { resource: rabbit.yml }
- { resource: redis.yml }
services:
wallabag_import.chain:
class: Wallabag\ImportBundle\Import\ImportChain
@ -14,7 +18,6 @@ services:
wallabag_import.pocket.import:
class: Wallabag\ImportBundle\Import\PocketImport
arguments:
- "@security.token_storage"
- "@doctrine.orm.entity_manager"
- "@wallabag_core.content_proxy"
- "@craue_config"

8
src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig Normal file
View file

@ -0,0 +1,8 @@
{% set redis = craue_setting('import_with_redis') %}
{% set rabbit = craue_setting('import_with_rabbitmq') %}
{% if redis or rabbit %}
<div class="card-panel yellow darken-1 black-text">
{{ 'import.worker.enabled'|trans }} <strong>{% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}</strong>
</div>
{% endif %}

11
src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig Normal file
View file

@ -0,0 +1,11 @@
{% if nbRedisMessages > 0 %}
<script>
Materialize.toast('Messages in queue: {{ nbRedisMessages }}', 4000);
</script>
{% endif %}
{% if nbRabbitMessages > 0 %}
<script>
Materialize.toast('Messages in queue: {{ nbRabbitMessages }}', 4000);
</script>
{% endif %}

12
src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
View file

@ -6,15 +6,13 @@
<div class="row">
<div class="col s12">
<div class="card-panel settings">
{% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
{% if not has_consumer_key %}
<div class="card-panel red darken-1">
<div class="card-panel red white-text">
{{ 'import.pocket.config_missing.description'|trans }}
{% if is_granted('ROLE_SUPER_ADMIN') %}
{{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('craue_config_settings_modify') ~ '#set-import">', '%keyurle%':'</a>'})|raw }}
{% else %}
{{ 'import.pocket.config_missing.user_message'|trans }}
{% endif %}
{{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('config') ~ '">', '%keyurle%':'</a>'})|raw }}
</div>
{% endif %}
@ -29,7 +27,7 @@
{{ form_label(form.mark_as_read) }}
</div>
</div>
<button class="btn waves-effect waves-light" type="submit" name="action">
<button class="btn waves-effect waves-light" type="submit" name="action" {% if not has_consumer_key %}disabled="disabled"{% endif %}>
{{ 'import.pocket.connect_to_pocket'|trans }}
</button>
</form>

2
src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
View file

@ -6,6 +6,8 @@
<div class="row">
<div class="col s12">
<div class="card-panel settings">
{% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
<div class="row">
<blockquote>{{ import.description|trans }}</blockquote>
<p>{{ 'import.readability.how_to'|trans }}</p>

2
src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
View file

@ -6,6 +6,8 @@
<div class="row">
<div class="col s12">
<div class="card-panel settings">
{% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
<div class="row">
<blockquote>{{ import.description|trans }}</blockquote>
<p>{{ 'import.wallabag_v1.how_to'|trans }}</p>

6
src/Wallabag/UserBundle/Resources/config/services.yml
View file

@ -14,3 +14,9 @@ services:
- "@router"
tags:
- { name: kernel.event_subscriber }
wallabag_user.user_repository:
class: Wallabag\UserBundle\Repository\UserRepository
factory: [ "@doctrine.orm.default_entity_manager", getRepository ]
arguments:
- WallabagUserBundle:User