mirror of
https://github.com/miniflux/v2.git
synced 2025-08-11 17:51:01 +00:00
34499b887b
Each batch of feeds sent to the worker pool is now guaranteed to contain unique feed URLs. When `POLLING_LIMIT_PER_HOST` is set, an additional limit is applied to the number of concurrent requests per hostname, helping to prevent overloading a single server. Note: Additional requests may still be made during feed refresh. For example, to fetch feed icons or when the web scraper is enabled for a particular feed.
59 lines
1.6 KiB
Go
59 lines
1.6 KiB
Go
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package cli // import "miniflux.app/v2/internal/cli"
|
|
|
|
import (
|
|
"log/slog"
|
|
"time"
|
|
|
|
"miniflux.app/v2/internal/config"
|
|
"miniflux.app/v2/internal/storage"
|
|
"miniflux.app/v2/internal/worker"
|
|
)
|
|
|
|
func runScheduler(store *storage.Storage, pool *worker.Pool) {
|
|
slog.Debug(`Starting background scheduler...`)
|
|
|
|
go feedScheduler(
|
|
store,
|
|
pool,
|
|
config.Opts.PollingFrequency(),
|
|
config.Opts.BatchSize(),
|
|
config.Opts.PollingParsingErrorLimit(),
|
|
config.Opts.PollingLimitPerHost(),
|
|
)
|
|
|
|
go cleanupScheduler(
|
|
store,
|
|
config.Opts.CleanupFrequencyHours(),
|
|
)
|
|
}
|
|
|
|
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize, errorLimit, limitPerHost int) {
|
|
for range time.Tick(time.Duration(frequency) * time.Minute) {
|
|
// Generate a batch of feeds for any user that has feeds to refresh.
|
|
batchBuilder := store.NewBatchBuilder()
|
|
batchBuilder.WithBatchSize(batchSize)
|
|
batchBuilder.WithErrorLimit(errorLimit)
|
|
batchBuilder.WithoutDisabledFeeds()
|
|
batchBuilder.WithNextCheckExpired()
|
|
batchBuilder.WithLimitPerHost(limitPerHost)
|
|
|
|
if jobs, err := batchBuilder.FetchJobs(); err != nil {
|
|
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
|
} else if len(jobs) > 0 {
|
|
slog.Info("Created a batch of feeds",
|
|
slog.Int("nb_jobs", len(jobs)),
|
|
)
|
|
slog.Debug("Feed URLs in this batch", slog.Any("feed_urls", jobs.FeedURLs()))
|
|
pool.Push(jobs)
|
|
}
|
|
}
|
|
}
|
|
|
|
func cleanupScheduler(store *storage.Storage, frequency int) {
|
|
for range time.Tick(time.Duration(frequency) * time.Hour) {
|
|
runCleanupTasks(store)
|
|
}
|
|
}
|