Move internal packages to an internal folder

For reference: https://go.dev/doc/go1.4#internalpackages

internal/cli/ask_credentials.go

@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"golang.org/x/term"
+)
+
+func askCredentials() (string, string) {
+	fd := int(os.Stdin.Fd())
+
+	if !term.IsTerminal(fd) {
+		fmt.Fprintf(os.Stderr, "This is not a terminal, exiting.\n")
+		os.Exit(1)
+	}
+
+	fmt.Print("Enter Username: ")
+
+	reader := bufio.NewReader(os.Stdin)
+	username, _ := reader.ReadString('\n')
+
+	fmt.Print("Enter Password: ")
+
+	state, _ := term.GetState(fd)
+	defer term.Restore(fd, state)
+	bytePassword, _ := term.ReadPassword(fd)
+
+	fmt.Printf("\n")
+	return strings.TrimSpace(username), strings.TrimSpace(string(bytePassword))
+}

internal/cli/cleanup_tasks.go

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"time"
+
+	"miniflux.app/v2/internal/config"
+	"miniflux.app/v2/internal/logger"
+	"miniflux.app/v2/internal/metric"
+	"miniflux.app/v2/internal/model"
+	"miniflux.app/v2/internal/storage"
+)
+
+func runCleanupTasks(store *storage.Storage) {
+	nbSessions := store.CleanOldSessions(config.Opts.CleanupRemoveSessionsDays())
+	nbUserSessions := store.CleanOldUserSessions(config.Opts.CleanupRemoveSessionsDays())
+	logger.Info("[Sessions] Removed %d application sessions and %d user sessions", nbSessions, nbUserSessions)
+
+	startTime := time.Now()
+	if rowsAffected, err := store.ArchiveEntries(model.EntryStatusRead, config.Opts.CleanupArchiveReadDays(), config.Opts.CleanupArchiveBatchSize()); err != nil {
+		logger.Error("[ArchiveReadEntries] %v", err)
+	} else {
+		logger.Info("[ArchiveReadEntries] %d entries changed", rowsAffected)
+
+		if config.Opts.HasMetricsCollector() {
+			metric.ArchiveEntriesDuration.WithLabelValues(model.EntryStatusRead).Observe(time.Since(startTime).Seconds())
+		}
+	}
+
+	startTime = time.Now()
+	if rowsAffected, err := store.ArchiveEntries(model.EntryStatusUnread, config.Opts.CleanupArchiveUnreadDays(), config.Opts.CleanupArchiveBatchSize()); err != nil {
+		logger.Error("[ArchiveUnreadEntries] %v", err)
+	} else {
+		logger.Info("[ArchiveUnreadEntries] %d entries changed", rowsAffected)
+
+		if config.Opts.HasMetricsCollector() {
+			metric.ArchiveEntriesDuration.WithLabelValues(model.EntryStatusUnread).Observe(time.Since(startTime).Seconds())
+		}
+	}
+}

internal/cli/cli.go

@@ -0,0 +1,207 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"flag"
+	"fmt"
+
+	"miniflux.app/v2/internal/config"
+	"miniflux.app/v2/internal/database"
+	"miniflux.app/v2/internal/locale"
+	"miniflux.app/v2/internal/logger"
+	"miniflux.app/v2/internal/storage"
+	"miniflux.app/v2/internal/ui/static"
+	"miniflux.app/v2/internal/version"
+)
+
+const (
+	flagInfoHelp            = "Show build information"
+	flagVersionHelp         = "Show application version"
+	flagMigrateHelp         = "Run SQL migrations"
+	flagFlushSessionsHelp   = "Flush all sessions (disconnect users)"
+	flagCreateAdminHelp     = "Create admin user"
+	flagResetPasswordHelp   = "Reset user password"
+	flagResetFeedErrorsHelp = "Clear all feed errors for all users"
+	flagDebugModeHelp       = "Show debug logs"
+	flagConfigFileHelp      = "Load configuration file"
+	flagConfigDumpHelp      = "Print parsed configuration values"
+	flagHealthCheckHelp     = `Perform a health check on the given endpoint (the value "auto" try to guess the health check endpoint).`
+	flagRefreshFeedsHelp    = "Refresh a batch of feeds and exit"
+	flagRunCleanupTasksHelp = "Run cleanup tasks (delete old sessions and archives old entries)"
+)
+
+// Parse parses command line arguments.
+func Parse() {
+	var (
+		err                 error
+		flagInfo            bool
+		flagVersion         bool
+		flagMigrate         bool
+		flagFlushSessions   bool
+		flagCreateAdmin     bool
+		flagResetPassword   bool
+		flagResetFeedErrors bool
+		flagDebugMode       bool
+		flagConfigFile      string
+		flagConfigDump      bool
+		flagHealthCheck     string
+		flagRefreshFeeds    bool
+		flagRunCleanupTasks bool
+	)
+
+	flag.BoolVar(&flagInfo, "info", false, flagInfoHelp)
+	flag.BoolVar(&flagInfo, "i", false, flagInfoHelp)
+	flag.BoolVar(&flagVersion, "version", false, flagVersionHelp)
+	flag.BoolVar(&flagVersion, "v", false, flagVersionHelp)
+	flag.BoolVar(&flagMigrate, "migrate", false, flagMigrateHelp)
+	flag.BoolVar(&flagFlushSessions, "flush-sessions", false, flagFlushSessionsHelp)
+	flag.BoolVar(&flagCreateAdmin, "create-admin", false, flagCreateAdminHelp)
+	flag.BoolVar(&flagResetPassword, "reset-password", false, flagResetPasswordHelp)
+	flag.BoolVar(&flagResetFeedErrors, "reset-feed-errors", false, flagResetFeedErrorsHelp)
+	flag.BoolVar(&flagDebugMode, "debug", false, flagDebugModeHelp)
+	flag.StringVar(&flagConfigFile, "config-file", "", flagConfigFileHelp)
+	flag.StringVar(&flagConfigFile, "c", "", flagConfigFileHelp)
+	flag.BoolVar(&flagConfigDump, "config-dump", false, flagConfigDumpHelp)
+	flag.StringVar(&flagHealthCheck, "healthcheck", "", flagHealthCheckHelp)
+	flag.BoolVar(&flagRefreshFeeds, "refresh-feeds", false, flagRefreshFeedsHelp)
+	flag.BoolVar(&flagRunCleanupTasks, "run-cleanup-tasks", false, flagRunCleanupTasksHelp)
+	flag.Parse()
+
+	cfg := config.NewParser()
+
+	if flagConfigFile != "" {
+		config.Opts, err = cfg.ParseFile(flagConfigFile)
+		if err != nil {
+			logger.Fatal("%v", err)
+		}
+	}
+
+	config.Opts, err = cfg.ParseEnvironmentVariables()
+	if err != nil {
+		logger.Fatal("%v", err)
+	}
+
+	if flagConfigDump {
+		fmt.Print(config.Opts)
+		return
+	}
+
+	if config.Opts.LogDateTime() {
+		logger.EnableDateTime()
+	}
+
+	if flagDebugMode || config.Opts.HasDebugMode() {
+		logger.EnableDebug()
+	}
+
+	if flagHealthCheck != "" {
+		doHealthCheck(flagHealthCheck)
+		return
+	}
+
+	if flagInfo {
+		info()
+		return
+	}
+
+	if flagVersion {
+		fmt.Println(version.Version)
+		return
+	}
+
+	if config.Opts.IsDefaultDatabaseURL() {
+		logger.Info("The default value for DATABASE_URL is used")
+	}
+
+	logger.Debug("Loading translations...")
+	if err := locale.LoadCatalogMessages(); err != nil {
+		logger.Fatal("Unable to load translations: %v", err)
+	}
+
+	logger.Debug("Loading static assets...")
+	if err := static.CalculateBinaryFileChecksums(); err != nil {
+		logger.Fatal("Unable to calculate binary files checksum: %v", err)
+	}
+
+	if err := static.GenerateStylesheetsBundles(); err != nil {
+		logger.Fatal("Unable to generate Stylesheet bundles: %v", err)
+	}
+
+	if err := static.GenerateJavascriptBundles(); err != nil {
+		logger.Fatal("Unable to generate Javascript bundles: %v", err)
+	}
+
+	db, err := database.NewConnectionPool(
+		config.Opts.DatabaseURL(),
+		config.Opts.DatabaseMinConns(),
+		config.Opts.DatabaseMaxConns(),
+		config.Opts.DatabaseConnectionLifetime(),
+	)
+	if err != nil {
+		logger.Fatal("Unable to initialize database connection pool: %v", err)
+	}
+	defer db.Close()
+
+	store := storage.NewStorage(db)
+
+	if err := store.Ping(); err != nil {
+		logger.Fatal("Unable to connect to the database: %v", err)
+	}
+
+	if flagMigrate {
+		if err := database.Migrate(db); err != nil {
+			logger.Fatal(`%v`, err)
+		}
+		return
+	}
+
+	if flagResetFeedErrors {
+		store.ResetFeedErrors()
+		return
+	}
+
+	if flagFlushSessions {
+		flushSessions(store)
+		return
+	}
+
+	if flagCreateAdmin {
+		createAdmin(store)
+		return
+	}
+
+	if flagResetPassword {
+		resetPassword(store)
+		return
+	}
+
+	// Run migrations and start the daemon.
+	if config.Opts.RunMigrations() {
+		if err := database.Migrate(db); err != nil {
+			logger.Fatal(`%v`, err)
+		}
+	}
+
+	if err := database.IsSchemaUpToDate(db); err != nil {
+		logger.Fatal(`You must run the SQL migrations, %v`, err)
+	}
+
+	// Create admin user and start the daemon.
+	if config.Opts.CreateAdmin() {
+		createAdmin(store)
+	}
+
+	if flagRefreshFeeds {
+		refreshFeeds(store)
+		return
+	}
+
+	if flagRunCleanupTasks {
+		runCleanupTasks(store)
+		return
+	}
+
+	startDaemon(store)
+}

internal/cli/create_admin.go

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"fmt"
+	"os"
+
+	"miniflux.app/v2/internal/config"
+	"miniflux.app/v2/internal/logger"
+	"miniflux.app/v2/internal/model"
+	"miniflux.app/v2/internal/storage"
+	"miniflux.app/v2/internal/validator"
+)
+
+func createAdmin(store *storage.Storage) {
+	userCreationRequest := &model.UserCreationRequest{
+		Username: config.Opts.AdminUsername(),
+		Password: config.Opts.AdminPassword(),
+		IsAdmin:  true,
+	}
+
+	if userCreationRequest.Username == "" || userCreationRequest.Password == "" {
+		userCreationRequest.Username, userCreationRequest.Password = askCredentials()
+	}
+
+	if store.UserExists(userCreationRequest.Username) {
+		logger.Info(`User %q already exists, skipping creation`, userCreationRequest.Username)
+		return
+	}
+
+	if validationErr := validator.ValidateUserCreationWithPassword(store, userCreationRequest); validationErr != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", validationErr)
+		os.Exit(1)
+	}
+
+	if _, err := store.CreateUser(userCreationRequest); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

internal/cli/daemon.go

@@ -0,0 +1,87 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"miniflux.app/v2/internal/config"
+	httpd "miniflux.app/v2/internal/http/server"
+	"miniflux.app/v2/internal/logger"
+	"miniflux.app/v2/internal/metric"
+	"miniflux.app/v2/internal/storage"
+	"miniflux.app/v2/internal/systemd"
+	"miniflux.app/v2/internal/worker"
+)
+
+func startDaemon(store *storage.Storage) {
+	logger.Info("Starting daemon...")
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt)
+	signal.Notify(stop, syscall.SIGTERM)
+
+	pool := worker.NewPool(store, config.Opts.WorkerPoolSize())
+
+	if config.Opts.HasSchedulerService() && !config.Opts.HasMaintenanceMode() {
+		runScheduler(store, pool)
+	}
+
+	var httpServer *http.Server
+	if config.Opts.HasHTTPService() {
+		httpServer = httpd.StartWebServer(store, pool)
+	}
+
+	if config.Opts.HasMetricsCollector() {
+		collector := metric.NewCollector(store, config.Opts.MetricsRefreshInterval())
+		go collector.GatherStorageMetrics()
+	}
+
+	if systemd.HasNotifySocket() {
+		logger.Info("Sending readiness notification to Systemd")
+
+		if err := systemd.SdNotify(systemd.SdNotifyReady); err != nil {
+			logger.Error("Unable to send readiness notification to systemd: %v", err)
+		}
+
+		if config.Opts.HasWatchdog() && systemd.HasSystemdWatchdog() {
+			logger.Info("Activating Systemd watchdog")
+
+			go func() {
+				interval, err := systemd.WatchdogInterval()
+				if err != nil {
+					logger.Error("Unable to parse watchdog interval from systemd: %v", err)
+					return
+				}
+
+				for {
+					err := store.Ping()
+					if err != nil {
+						logger.Error(`Systemd Watchdog: %v`, err)
+					} else {
+						systemd.SdNotify(systemd.SdNotifyWatchdog)
+					}
+
+					time.Sleep(interval / 3)
+				}
+			}()
+		}
+	}
+
+	<-stop
+	logger.Info("Shutting down the process...")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if httpServer != nil {
+		httpServer.Shutdown(ctx)
+	}
+
+	logger.Info("Process gracefully stopped")
+}

internal/cli/flush_sessions.go

@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"fmt"
+	"os"
+
+	"miniflux.app/v2/internal/storage"
+)
+
+func flushSessions(store *storage.Storage) {
+	fmt.Println("Flushing all sessions (disconnect users)")
+	if err := store.FlushAllSessions(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

internal/cli/health_check.go

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"net/http"
+	"time"
+
+	"miniflux.app/v2/internal/config"
+	"miniflux.app/v2/internal/logger"
+)
+
+func doHealthCheck(healthCheckEndpoint string) {
+	if healthCheckEndpoint == "auto" {
+		healthCheckEndpoint = "http://" + config.Opts.ListenAddr() + config.Opts.BasePath() + "/healthcheck"
+	}
+
+	logger.Debug(`Executing health check on %s`, healthCheckEndpoint)
+
+	client := &http.Client{Timeout: 3 * time.Second}
+	resp, err := client.Get(healthCheckEndpoint)
+	if err != nil {
+		logger.Fatal(`Health check failure: %v`, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		logger.Fatal(`Health check failed with status code %d`, resp.StatusCode)
+	}
+
+	logger.Debug(`Health check is OK`)
+}

internal/cli/info.go

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"fmt"
+	"runtime"
+
+	"miniflux.app/v2/internal/version"
+)
+
+func info() {
+	fmt.Println("Version:", version.Version)
+	fmt.Println("Commit:", version.Commit)
+	fmt.Println("Build Date:", version.BuildDate)
+	fmt.Println("Go Version:", runtime.Version())
+	fmt.Println("Compiler:", runtime.Compiler)
+	fmt.Println("Arch:", runtime.GOARCH)
+	fmt.Println("OS:", runtime.GOOS)
+}

internal/cli/refresh_feeds.go

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"sync"
+	"time"
+
+	"miniflux.app/v2/internal/config"
+	"miniflux.app/v2/internal/logger"
+	"miniflux.app/v2/internal/model"
+	feedHandler "miniflux.app/v2/internal/reader/handler"
+	"miniflux.app/v2/internal/storage"
+)
+
+func refreshFeeds(store *storage.Storage) {
+	var wg sync.WaitGroup
+
+	startTime := time.Now()
+	jobs, err := store.NewBatch(config.Opts.BatchSize())
+	if err != nil {
+		logger.Error("[Cronjob] %v", err)
+	}
+
+	nbJobs := len(jobs)
+	logger.Info("[Cronjob]] Created %d jobs from a batch size of %d", nbJobs, config.Opts.BatchSize())
+	var jobQueue = make(chan model.Job, nbJobs)
+
+	logger.Info("[Cronjob] Starting a pool of %d workers", config.Opts.WorkerPoolSize())
+	for i := 0; i < config.Opts.WorkerPoolSize(); i++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			for job := range jobQueue {
+				logger.Info("[Cronjob] Refreshing feed #%d for user #%d in worker #%d", job.FeedID, job.UserID, workerID)
+				if err := feedHandler.RefreshFeed(store, job.UserID, job.FeedID, false); err != nil {
+					logger.Error("[Cronjob] Refreshing the feed #%d returned this error: %v", job.FeedID, err)
+				}
+			}
+		}(i)
+	}
+
+	for _, job := range jobs {
+		jobQueue <- job
+	}
+	close(jobQueue)
+
+	wg.Wait()
+	logger.Info("[Cronjob] Refreshed %d feed(s) in %s", nbJobs, time.Since(startTime))
+}

internal/cli/reset_password.go

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"fmt"
+	"os"
+
+	"miniflux.app/v2/internal/model"
+	"miniflux.app/v2/internal/storage"
+	"miniflux.app/v2/internal/validator"
+)
+
+func resetPassword(store *storage.Storage) {
+	username, password := askCredentials()
+	user, err := store.UserByUsername(username)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	if user == nil {
+		fmt.Fprintf(os.Stderr, "User not found!\n")
+		os.Exit(1)
+	}
+
+	userModificationRequest := &model.UserModificationRequest{
+		Password: &password,
+	}
+	if validationErr := validator.ValidateUserModification(store, user.ID, userModificationRequest); validationErr != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", validationErr)
+		os.Exit(1)
+	}
+
+	user.Password = password
+	if err := store.UpdateUser(user); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("Password changed!")
+}

internal/cli/scheduler.go

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cli // import "miniflux.app/v2/internal/cli"
+
+import (
+	"time"
+
+	"miniflux.app/v2/internal/config"
+	"miniflux.app/v2/internal/logger"
+	"miniflux.app/v2/internal/storage"
+	"miniflux.app/v2/internal/worker"
+)
+
+func runScheduler(store *storage.Storage, pool *worker.Pool) {
+	logger.Info(`Starting background scheduler...`)
+
+	go feedScheduler(
+		store,
+		pool,
+		config.Opts.PollingFrequency(),
+		config.Opts.BatchSize(),
+	)
+
+	go cleanupScheduler(
+		store,
+		config.Opts.CleanupFrequencyHours(),
+	)
+}
+
+func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize int) {
+	for range time.Tick(time.Duration(frequency) * time.Minute) {
+		jobs, err := store.NewBatch(batchSize)
+		logger.Info("[Scheduler:Feed] Pushing %d jobs to the queue", len(jobs))
+		if err != nil {
+			logger.Error("[Scheduler:Feed] %v", err)
+		} else {
+			pool.Push(jobs)
+		}
+	}
+}
+
+func cleanupScheduler(store *storage.Storage, frequency int) {
+	for range time.Tick(time.Duration(frequency) * time.Hour) {
+		runCleanupTasks(store)
+	}
+}