oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-08-11 17:51:01 +00:00
Code Wiki Activity

feat: add POLLING_LIMIT_PER_HOST to limit concurrent requests per host

Browse source 
Each batch of feeds sent to the worker pool is now guaranteed to contain unique feed URLs.

When `POLLING_LIMIT_PER_HOST` is set, an additional limit is applied to the number of concurrent requests per hostname, helping to prevent overloading a single server.

Note: Additional requests may still be made during feed refresh. For example, to fetch feed icons or when the web scraper is enabled for a particular feed.
This commit is contained in:
Frédéric Guillot 2025-08-08 12:19:01 -07:00
parent a4f672b589
commit 34499b887b
13 changed files with 146 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

5
internal/cli/scheduler.go
View file

@ -21,6 +21,7 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) {
config.Opts.PollingFrequency(),
config.Opts.BatchSize(),
config.Opts.PollingParsingErrorLimit(),
config.Opts.PollingLimitPerHost(),
)
go cleanupScheduler(
@ -29,7 +30,7 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) {
)
}
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize, errorLimit int) {
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize, errorLimit, limitPerHost int) {
for range time.Tick(time.Duration(frequency) * time.Minute) {
// Generate a batch of feeds for any user that has feeds to refresh.
batchBuilder := store.NewBatchBuilder()
@ -37,6 +38,7 @@ func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSi
batchBuilder.WithErrorLimit(errorLimit)
batchBuilder.WithoutDisabledFeeds()
batchBuilder.WithNextCheckExpired()
batchBuilder.WithLimitPerHost(limitPerHost)
if jobs, err := batchBuilder.FetchJobs(); err != nil {
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
@ -44,6 +46,7 @@ func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSi
slog.Info("Created a batch of feeds",
slog.Int("nb_jobs", len(jobs)),
)
slog.Debug("Feed URLs in this batch", slog.Any("feed_urls", jobs.FeedURLs()))
pool.Push(jobs)
}
}