From c87c93d85f06a9286496c5a2e75c5bf70f3ee63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Fri, 11 Apr 2025 15:32:19 -0700 Subject: [PATCH] feat(config): add `SCHEDULER_ROUND_ROBIN_MAX_INTERVAL` option Add option to cap maximum refresh interval when RSS TTL, Retry-After, Cache-Control, or Expires headers specify excessively high values. --- internal/config/config_test.go | 35 ++++++++++++++++++++++++++++++ internal/config/options.go | 8 +++++++ internal/config/parser.go | 2 ++ internal/model/feed.go | 15 +++++++++---- internal/model/feed_test.go | 23 ++++++++++++++++++++ internal/reader/handler/handler.go | 6 +++-- miniflux.1 | 7 +++++- 7 files changed, 89 insertions(+), 7 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6d1ecbdc..f925e2c9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1028,6 +1028,41 @@ func TestSchedulerRoundRobin(t *testing.T) { } } +func TestDefaultSchedulerRoundRobinMaxIntervalValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := defaultSchedulerRoundRobinMaxInterval + result := opts.SchedulerRoundRobinMaxInterval() + + if result != expected { + t.Fatalf(`Unexpected SCHEDULER_ROUND_ROBIN_MAX_INTERVAL value, got %v instead of %v`, result, expected) + } +} + +func TestSchedulerRoundRobinMaxInterval(t *testing.T) { + os.Clearenv() + os.Setenv("SCHEDULER_ROUND_ROBIN_MAX_INTERVAL", "150") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 150 + result := opts.SchedulerRoundRobinMaxInterval() + + if result != expected { + t.Fatalf(`Unexpected SCHEDULER_ROUND_ROBIN_MAX_INTERVAL value, got %v instead of %v`, result, expected) + } +} + func TestPollingParsingErrorLimit(t *testing.T) { os.Clearenv() os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100") diff --git a/internal/config/options.go b/internal/config/options.go index bfe0cf62..9605dfe3 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -37,6 +37,7 @@ const ( defaultSchedulerEntryFrequencyMaxInterval = 24 * 60 defaultSchedulerEntryFrequencyFactor = 1 defaultSchedulerRoundRobinMinInterval = 60 + defaultSchedulerRoundRobinMaxInterval = 1440 defaultPollingParsingErrorLimit = 3 defaultRunMigrations = false defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" @@ -137,6 +138,7 @@ type Options struct { schedulerEntryFrequencyMaxInterval int schedulerEntryFrequencyFactor int schedulerRoundRobinMinInterval int + schedulerRoundRobinMaxInterval int pollingParsingErrorLimit int workerPoolSize int createAdmin bool @@ -220,6 +222,7 @@ func NewOptions() *Options { schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval, schedulerEntryFrequencyFactor: defaultSchedulerEntryFrequencyFactor, schedulerRoundRobinMinInterval: defaultSchedulerRoundRobinMinInterval, + schedulerRoundRobinMaxInterval: defaultSchedulerRoundRobinMaxInterval, pollingParsingErrorLimit: defaultPollingParsingErrorLimit, workerPoolSize: defaultWorkerPoolSize, createAdmin: defaultCreateAdmin, @@ -433,6 +436,10 @@ func (o *Options) SchedulerRoundRobinMinInterval() int { return o.schedulerRoundRobinMinInterval } +func (o *Options) SchedulerRoundRobinMaxInterval() int { + return o.schedulerRoundRobinMaxInterval +} + // PollingParsingErrorLimit returns the limit of errors when to stop polling. func (o *Options) PollingParsingErrorLimit() int { return o.pollingParsingErrorLimit @@ -778,6 +785,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": o.schedulerEntryFrequencyMinInterval, "SCHEDULER_ENTRY_FREQUENCY_FACTOR": o.schedulerEntryFrequencyFactor, "SCHEDULER_ROUND_ROBIN_MIN_INTERVAL": o.schedulerRoundRobinMinInterval, + "SCHEDULER_ROUND_ROBIN_MAX_INTERVAL": o.schedulerRoundRobinMaxInterval, "SCHEDULER_SERVICE": o.schedulerService, "SERVER_TIMING_HEADER": o.serverTimingHeader, "WATCHDOG": o.watchdog, diff --git a/internal/config/parser.go b/internal/config/parser.go index abf8e5ef..29ef5018 100644 --- a/internal/config/parser.go +++ b/internal/config/parser.go @@ -160,6 +160,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.schedulerEntryFrequencyFactor = parseInt(value, defaultSchedulerEntryFrequencyFactor) case "SCHEDULER_ROUND_ROBIN_MIN_INTERVAL": p.opts.schedulerRoundRobinMinInterval = parseInt(value, defaultSchedulerRoundRobinMinInterval) + case "SCHEDULER_ROUND_ROBIN_MAX_INTERVAL": + p.opts.schedulerRoundRobinMaxInterval = parseInt(value, defaultSchedulerRoundRobinMaxInterval) case "POLLING_PARSING_ERROR_LIMIT": p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit) case "PROXY_IMAGES": diff --git a/internal/model/feed.go b/internal/model/feed.go index 7f022bed..6f289b47 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -117,9 +117,7 @@ func (f *Feed) CheckedNow() { } // ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration. -func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) { - f.TTL = refreshDelayInMinutes - +func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) int { // Default to the global config Polling Frequency. intervalMinutes := config.Opts.SchedulerRoundRobinMinInterval() @@ -133,12 +131,21 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) { } } - // If the feed has a TTL or a Retry-After defined, we use it to make sure we don't check it too often. + // Use the RSS TTL field, Retry-After, Cache-Control or Expires HTTP headers if defined. if refreshDelayInMinutes > 0 && refreshDelayInMinutes > intervalMinutes { intervalMinutes = refreshDelayInMinutes } + // Limit the max interval value for misconfigured feeds. + switch config.Opts.PollingScheduler() { + case SchedulerRoundRobin: + intervalMinutes = min(intervalMinutes, config.Opts.SchedulerRoundRobinMaxInterval()) + case SchedulerEntryFrequency: + intervalMinutes = min(intervalMinutes, config.Opts.SchedulerEntryFrequencyMaxInterval()) + } + f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes)) + return intervalMinutes } // FeedCreationRequest represents the request to create a feed. diff --git a/internal/model/feed_test.go b/internal/model/feed_test.go index 673b828e..d6887a93 100644 --- a/internal/model/feed_test.go +++ b/internal/model/feed_test.go @@ -144,6 +144,29 @@ func TestFeedScheduleNextCheckRoundRobinWithRefreshDelayBelowMinInterval(t *test checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinWithRefreshDelayBelowMinInterval") } +func TestFeedScheduleNextCheckRoundRobinWithRefreshDelayAboveMaxInterval(t *testing.T) { + os.Clearenv() + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + timeBefore := time.Now() + feed := &Feed{} + + feed.ScheduleNextCheck(0, config.Opts.SchedulerRoundRobinMaxInterval()+30) + + if feed.NextCheckAt.IsZero() { + t.Error(`The next_check_at must be set`) + } + + expectedInterval := config.Opts.SchedulerRoundRobinMaxInterval() + checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinWithRefreshDelayAboveMaxInterval") +} + func TestFeedScheduleNextCheckRoundRobinMinInterval(t *testing.T) { minInterval := 1 os.Clearenv() diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 15301a7a..43f6c764 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -240,12 +240,13 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool if responseHandler.IsRateLimited() { retryDelayInSeconds := responseHandler.ParseRetryDelay() refreshDelayInMinutes = retryDelayInSeconds / 60 - originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes) + calculatedNextCheckIntervalInMinutes := originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes) slog.Warn("Feed is rate limited", slog.String("feed_url", originalFeed.FeedURL), slog.Int("retry_delay_in_seconds", retryDelayInSeconds), slog.Int("refresh_delay_in_minutes", refreshDelayInMinutes), + slog.Int("calculated_next_check_interval_in_minutes", calculatedNextCheckIntervalInMinutes), slog.Time("new_next_check_at", originalFeed.NextCheckAt), ) } @@ -316,7 +317,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool refreshDelayInMinutes = max(feedTTLValue, cacheControlMaxAgeValue, expiresValue) // Set the next check at with updated arguments. - originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes) + calculatedNextCheckIntervalInMinutes := originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes) slog.Debug("Updated next check date", slog.Int64("user_id", userID), @@ -326,6 +327,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool slog.Int("cache_control_max_age_in_minutes", cacheControlMaxAgeValue), slog.Int("expires_in_minutes", expiresValue), slog.Int("refresh_delay_in_minutes", refreshDelayInMinutes), + slog.Int("calculated_next_check_interval_in_minutes", calculatedNextCheckIntervalInMinutes), slog.Time("new_next_check_at", originalFeed.NextCheckAt), ) diff --git a/miniflux.1 b/miniflux.1 index 4ac89a82..00d2171d 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -1,5 +1,5 @@ .\" Manpage for miniflux. -.TH "MINIFLUX" "1" "December 7, 2024" "\ \&" "\ \&" +.TH "MINIFLUX" "1" "April 11, 2025" "\ \&" "\ \&" .SH NAME miniflux \- Minimalist and opinionated feed reader @@ -533,6 +533,11 @@ Minimum interval in minutes for the entry frequency scheduler\&. .br Default is 5 minutes\&. .TP +.B SCHEDULER_ROUND_ROBIN_MAX_INTERVAL +Maximum interval in minutes for the round robin scheduler\&. +.br +Default is 1440 minutes (24 hours)\&. +.TP .B SCHEDULER_ROUND_ROBIN_MIN_INTERVAL Minimum interval in minutes for the round robin scheduler\&. .br