oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-06-27 16:36:00 +00:00
Code Wiki Activity

feat(config): add SCHEDULER_ROUND_ROBIN_MAX_INTERVAL option

Browse source 
Add option to cap maximum refresh interval when RSS TTL, Retry-After, Cache-Control, or Expires headers specify excessively high values.
This commit is contained in:
Frédéric Guillot 2025-04-11 15:32:19 -07:00
parent 0ef21e85c2
commit c87c93d85f
7 changed files with 89 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

35
internal/config/config_test.go
View file

@ -1028,6 +1028,41 @@ func TestSchedulerRoundRobin(t *testing.T) {
}
}
func TestDefaultSchedulerRoundRobinMaxIntervalValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultSchedulerRoundRobinMaxInterval
result := opts.SchedulerRoundRobinMaxInterval()
if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ROUND_ROBIN_MAX_INTERVAL value, got %v instead of %v`, result, expected)
}
}
func TestSchedulerRoundRobinMaxInterval(t *testing.T) {
os.Clearenv()
os.Setenv("SCHEDULER_ROUND_ROBIN_MAX_INTERVAL", "150")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 150
result := opts.SchedulerRoundRobinMaxInterval()
if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ROUND_ROBIN_MAX_INTERVAL value, got %v instead of %v`, result, expected)
}
}
func TestPollingParsingErrorLimit(t *testing.T) {
os.Clearenv()
os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100")

8
internal/config/options.go
View file

@ -37,6 +37,7 @@ const (
defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
defaultSchedulerEntryFrequencyFactor = 1
defaultSchedulerRoundRobinMinInterval = 60
defaultSchedulerRoundRobinMaxInterval = 1440
defaultPollingParsingErrorLimit = 3
defaultRunMigrations = false
defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
@ -137,6 +138,7 @@ type Options struct {
schedulerEntryFrequencyMaxInterval int
schedulerEntryFrequencyFactor int
schedulerRoundRobinMinInterval int
schedulerRoundRobinMaxInterval int
pollingParsingErrorLimit int
workerPoolSize int
createAdmin bool
@ -220,6 +222,7 @@ func NewOptions() *Options {
schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
schedulerEntryFrequencyFactor: defaultSchedulerEntryFrequencyFactor,
schedulerRoundRobinMinInterval: defaultSchedulerRoundRobinMinInterval,
schedulerRoundRobinMaxInterval: defaultSchedulerRoundRobinMaxInterval,
pollingParsingErrorLimit: defaultPollingParsingErrorLimit,
workerPoolSize: defaultWorkerPoolSize,
createAdmin: defaultCreateAdmin,
@ -433,6 +436,10 @@ func (o *Options) SchedulerRoundRobinMinInterval() int {
return o.schedulerRoundRobinMinInterval
}
func (o *Options) SchedulerRoundRobinMaxInterval() int {
return o.schedulerRoundRobinMaxInterval
}
// PollingParsingErrorLimit returns the limit of errors when to stop polling.
func (o *Options) PollingParsingErrorLimit() int {
return o.pollingParsingErrorLimit
@ -778,6 +785,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": o.schedulerEntryFrequencyMinInterval,
"SCHEDULER_ENTRY_FREQUENCY_FACTOR": o.schedulerEntryFrequencyFactor,
"SCHEDULER_ROUND_ROBIN_MIN_INTERVAL": o.schedulerRoundRobinMinInterval,
"SCHEDULER_ROUND_ROBIN_MAX_INTERVAL": o.schedulerRoundRobinMaxInterval,
"SCHEDULER_SERVICE": o.schedulerService,
"SERVER_TIMING_HEADER": o.serverTimingHeader,
"WATCHDOG": o.watchdog,

2
internal/config/parser.go
View file

@ -160,6 +160,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.schedulerEntryFrequencyFactor = parseInt(value, defaultSchedulerEntryFrequencyFactor)
case "SCHEDULER_ROUND_ROBIN_MIN_INTERVAL":
p.opts.schedulerRoundRobinMinInterval = parseInt(value, defaultSchedulerRoundRobinMinInterval)
case "SCHEDULER_ROUND_ROBIN_MAX_INTERVAL":
p.opts.schedulerRoundRobinMaxInterval = parseInt(value, defaultSchedulerRoundRobinMaxInterval)
case "POLLING_PARSING_ERROR_LIMIT":
p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
case "PROXY_IMAGES":

15
internal/model/feed.go
View file

@ -117,9 +117,7 @@ func (f *Feed) CheckedNow() {
}
// ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration.
func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) {
f.TTL = refreshDelayInMinutes
func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) int {
// Default to the global config Polling Frequency.
intervalMinutes := config.Opts.SchedulerRoundRobinMinInterval()
@ -133,12 +131,21 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) {
}
}
// If the feed has a TTL or a Retry-After defined, we use it to make sure we don't check it too often.
// Use the RSS TTL field, Retry-After, Cache-Control or Expires HTTP headers if defined.
if refreshDelayInMinutes > 0 && refreshDelayInMinutes > intervalMinutes {
intervalMinutes = refreshDelayInMinutes
}
// Limit the max interval value for misconfigured feeds.
switch config.Opts.PollingScheduler() {
case SchedulerRoundRobin:
intervalMinutes = min(intervalMinutes, config.Opts.SchedulerRoundRobinMaxInterval())
case SchedulerEntryFrequency:
intervalMinutes = min(intervalMinutes, config.Opts.SchedulerEntryFrequencyMaxInterval())
}
f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
return intervalMinutes
}
// FeedCreationRequest represents the request to create a feed.

23
internal/model/feed_test.go
View file

@ -144,6 +144,29 @@ func TestFeedScheduleNextCheckRoundRobinWithRefreshDelayBelowMinInterval(t *test
checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinWithRefreshDelayBelowMinInterval")
}
func TestFeedScheduleNextCheckRoundRobinWithRefreshDelayAboveMaxInterval(t *testing.T) {
os.Clearenv()
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
timeBefore := time.Now()
feed := &Feed{}
feed.ScheduleNextCheck(0, config.Opts.SchedulerRoundRobinMaxInterval()+30)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
}
expectedInterval := config.Opts.SchedulerRoundRobinMaxInterval()
checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinWithRefreshDelayAboveMaxInterval")
}
func TestFeedScheduleNextCheckRoundRobinMinInterval(t *testing.T) {
minInterval := 1
os.Clearenv()

6
internal/reader/handler/handler.go
View file

@ -240,12 +240,13 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
if responseHandler.IsRateLimited() {
retryDelayInSeconds := responseHandler.ParseRetryDelay()
refreshDelayInMinutes = retryDelayInSeconds / 60
originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
calculatedNextCheckIntervalInMinutes := originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
slog.Warn("Feed is rate limited",
slog.String("feed_url", originalFeed.FeedURL),
slog.Int("retry_delay_in_seconds", retryDelayInSeconds),
slog.Int("refresh_delay_in_minutes", refreshDelayInMinutes),
slog.Int("calculated_next_check_interval_in_minutes", calculatedNextCheckIntervalInMinutes),
slog.Time("new_next_check_at", originalFeed.NextCheckAt),
)
}
@ -316,7 +317,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
refreshDelayInMinutes = max(feedTTLValue, cacheControlMaxAgeValue, expiresValue)
// Set the next check at with updated arguments.
originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
calculatedNextCheckIntervalInMinutes := originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
slog.Debug("Updated next check date",
slog.Int64("user_id", userID),
@ -326,6 +327,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
slog.Int("cache_control_max_age_in_minutes", cacheControlMaxAgeValue),
slog.Int("expires_in_minutes", expiresValue),
slog.Int("refresh_delay_in_minutes", refreshDelayInMinutes),
slog.Int("calculated_next_check_interval_in_minutes", calculatedNextCheckIntervalInMinutes),
slog.Time("new_next_check_at", originalFeed.NextCheckAt),
)

7
miniflux.1
View file

@ -1,5 +1,5 @@
.\" Manpage for miniflux.
.TH "MINIFLUX" "1" "December 7, 2024" "\ \&" "\ \&"
.TH "MINIFLUX" "1" "April 11, 2025" "\ \&" "\ \&"
.SH NAME
miniflux \- Minimalist and opinionated feed reader
@ -533,6 +533,11 @@ Minimum interval in minutes for the entry frequency scheduler\&.
.br
Default is 5 minutes\&.
.TP
.B SCHEDULER_ROUND_ROBIN_MAX_INTERVAL
Maximum interval in minutes for the round robin scheduler\&.
.br
Default is 1440 minutes (24 hours)\&.
.TP
.B SCHEDULER_ROUND_ROBIN_MIN_INTERVAL
Minimum interval in minutes for the round robin scheduler\&.
.br