oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-09-15 18:57:04 +00:00
Code Wiki Activity

feat: add POLLING_LIMIT_PER_HOST to limit concurrent requests per host

Browse source 
Each batch of feeds sent to the worker pool is now guaranteed to contain unique feed URLs.

When `POLLING_LIMIT_PER_HOST` is set, an additional limit is applied to the number of concurrent requests per hostname, helping to prevent overloading a single server.

Note: Additional requests may still be made during feed refresh. For example, to fetch feed icons or when the web scraper is enabled for a particular feed.
This commit is contained in:
Frédéric Guillot 2025-08-08 12:19:01 -07:00
parent a4f672b589
commit 34499b887b
13 changed files with 146 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

33
internal/config/config_test.go
View file

@ -2104,3 +2104,36 @@ func TestInvalidHTTPClientProxy(t *testing.T) {
t.Fatalf(`Expected error for invalid HTTP_CLIENT_PROXY value, but got none`)
}
}
func TestDefaultPollingLimitPerHost(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 0
result := opts.PollingLimitPerHost()
if result != expected {
t.Fatalf(`Unexpected default PollingLimitPerHost value, got %v instead of %v`, result, expected)
}
}
func TestCustomPollingLimitPerHost(t *testing.T) {
os.Clearenv()
os.Setenv("POLLING_LIMIT_PER_HOST", "10")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 10
result := opts.PollingLimitPerHost()
if result != expected {
t.Fatalf(`Unexpected custom PollingLimitPerHost value, got %v instead of %v`, result, expected)
}
}

34
internal/config/options.go
View file

@ -130,16 +130,17 @@ type options struct {
cleanupArchiveUnreadDays int
cleanupArchiveBatchSize int
cleanupRemoveSessionsDays int
pollingFrequency int
forceRefreshInterval int
batchSize int
pollingScheduler string
schedulerEntryFrequencyMinInterval int
schedulerEntryFrequencyMaxInterval int
schedulerEntryFrequencyFactor int
schedulerRoundRobinMinInterval int
schedulerRoundRobinMaxInterval int
pollingFrequency int
pollingLimitPerHost int
pollingParsingErrorLimit int
pollingScheduler string
workerPoolSize int
createAdmin bool
adminUsername string
@ -390,11 +391,6 @@ func (o *options) WorkerPoolSize() int {
return o.workerPoolSize
}
// PollingFrequency returns the interval to refresh feeds in the background.
func (o *options) PollingFrequency() int {
return o.pollingFrequency
}
// ForceRefreshInterval returns the force refresh interval
func (o *options) ForceRefreshInterval() int {
return o.forceRefreshInterval
@ -405,6 +401,22 @@ func (o *options) BatchSize() int {
return o.batchSize
}
// PollingFrequency returns the interval to refresh feeds in the background.
func (o *options) PollingFrequency() int {
return o.pollingFrequency
}
// PollingLimitPerHost returns the limit of concurrent requests per host.
// Set to zero to disable.
func (o *options) PollingLimitPerHost() int {
return o.pollingLimitPerHost
}
// PollingParsingErrorLimit returns the limit of errors when to stop polling.
func (o *options) PollingParsingErrorLimit() int {
return o.pollingParsingErrorLimit
}
// PollingScheduler returns the scheduler used for polling feeds.
func (o *options) PollingScheduler() string {
return o.pollingScheduler
@ -433,11 +445,6 @@ func (o *options) SchedulerRoundRobinMaxInterval() int {
return o.schedulerRoundRobinMaxInterval
}
// PollingParsingErrorLimit returns the limit of errors when to stop polling.
func (o *options) PollingParsingErrorLimit() int {
return o.pollingParsingErrorLimit
}
// IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users.
func (o *options) IsOAuth2UserCreationAllowed() bool {
return o.oauth2UserCreationAllowed
@ -762,8 +769,9 @@ func (o *options) SortedOptions(redactSecret bool) []*option {
"OAUTH2_REDIRECT_URL": o.oauth2RedirectURL,
"OAUTH2_USER_CREATION": o.oauth2UserCreationAllowed,
"DISABLE_LOCAL_AUTH": o.disableLocalAuth,
"POLLING_FREQUENCY": o.pollingFrequency,
"FORCE_REFRESH_INTERVAL": o.forceRefreshInterval,
"POLLING_FREQUENCY": o.pollingFrequency,
"POLLING_LIMIT_PER_HOST": o.pollingLimitPerHost,
"POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit,
"POLLING_SCHEDULER": o.pollingScheduler,
"MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout,

10
internal/config/parser.go
View file

@ -137,12 +137,16 @@ func (p *parser) parseLines(lines []string) (err error) {
p.opts.cleanupRemoveSessionsDays = parseInt(value, defaultCleanupRemoveSessionsDays)
case "WORKER_POOL_SIZE":
p.opts.workerPoolSize = parseInt(value, defaultWorkerPoolSize)
case "POLLING_FREQUENCY":
p.opts.pollingFrequency = parseInt(value, defaultPollingFrequency)
case "FORCE_REFRESH_INTERVAL":
p.opts.forceRefreshInterval = parseInt(value, defaultForceRefreshInterval)
case "BATCH_SIZE":
p.opts.batchSize = parseInt(value, defaultBatchSize)
case "POLLING_FREQUENCY":
p.opts.pollingFrequency = parseInt(value, defaultPollingFrequency)
case "POLLING_LIMIT_PER_HOST":
p.opts.pollingLimitPerHost = parseInt(value, 0)
case "POLLING_PARSING_ERROR_LIMIT":
p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
case "POLLING_SCHEDULER":
p.opts.pollingScheduler = strings.ToLower(parseString(value, defaultPollingScheduler))
case "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL":
@ -155,8 +159,6 @@ func (p *parser) parseLines(lines []string) (err error) {
p.opts.schedulerRoundRobinMinInterval = parseInt(value, defaultSchedulerRoundRobinMinInterval)
case "SCHEDULER_ROUND_ROBIN_MAX_INTERVAL":
p.opts.schedulerRoundRobinMaxInterval = parseInt(value, defaultSchedulerRoundRobinMaxInterval)
case "POLLING_PARSING_ERROR_LIMIT":
p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
case "MEDIA_PROXY_HTTP_CLIENT_TIMEOUT":
p.opts.mediaProxyHTTPClientTimeout = parseInt(value, defaultMediaProxyHTTPClientTimeout)
case "MEDIA_PROXY_MODE":