oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-08-11 17:51:01 +00:00
Code Wiki Activity

feat: add POLLING_LIMIT_PER_HOST to limit concurrent requests per host

Browse source 
Each batch of feeds sent to the worker pool is now guaranteed to contain unique feed URLs.

When `POLLING_LIMIT_PER_HOST` is set, an additional limit is applied to the number of concurrent requests per hostname, helping to prevent overloading a single server.

Note: Additional requests may still be made during feed refresh. For example, to fetch feed icons or when the web scraper is enabled for a particular feed.
This commit is contained in:
Frédéric Guillot 2025-08-08 12:19:01 -07:00
parent a4f672b589
commit 34499b887b
13 changed files with 146 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

3
internal/cli/refresh_feeds.go
View file

@ -25,6 +25,7 @@ func refreshFeeds(store *storage.Storage) {
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
batchBuilder.WithoutDisabledFeeds()
batchBuilder.WithNextCheckExpired()
batchBuilder.WithLimitPerHost(config.Opts.PollingLimitPerHost())
jobs, err := batchBuilder.FetchJobs()
if err != nil {
@ -39,6 +40,8 @@ func refreshFeeds(store *storage.Storage) {
slog.Int("batch_size", config.Opts.BatchSize()),
)
slog.Debug("Feed URLs in this batch", slog.Any("feed_urls", jobs.FeedURLs()))
var jobQueue = make(chan model.Job, nbJobs)
slog.Info("Starting a pool of workers",