oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-09-15 18:57:04 +00:00
Code Wiki Activity

feat: take Retry-After header into consideration for rate limited feeds

Browse source
This commit is contained in:
Frédéric Guillot 2024-10-05 22:26:05 -07:00 • committed by GitHub
parent e555e442fb
commit e1050e21b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 143 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

21
internal/model/feed.go
View file

@ -112,12 +112,13 @@ func (f *Feed) CheckedNow() {
}
// ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration.
func (f *Feed) ScheduleNextCheck(weeklyCount int, newTTL int) {
f.TTL = newTTL
func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) {
f.TTL = refreshDelayInMinutes
// Default to the global config Polling Frequency.
var intervalMinutes int
switch config.Opts.PollingScheduler() {
case SchedulerEntryFrequency:
intervalMinutes := config.Opts.SchedulerRoundRobinMinInterval()
if config.Opts.PollingScheduler() == SchedulerEntryFrequency {
if weeklyCount <= 0 {
intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
} else {
@ -125,13 +126,13 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, newTTL int) {
intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
}
default:
intervalMinutes = config.Opts.SchedulerRoundRobinMinInterval()
}
// If the feed has a TTL defined, we use it to make sure we don't check it too often.
if newTTL > intervalMinutes && newTTL > 0 {
intervalMinutes = newTTL
// If the feed has a TTL or a Retry-After defined, we use it to make sure we don't check it too often.
if refreshDelayInMinutes > 0 && refreshDelayInMinutes > intervalMinutes {
intervalMinutes = refreshDelayInMinutes
}
f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
}

70
internal/model/feed_test.go
View file

@ -14,7 +14,7 @@ import (
const (
largeWeeklyCount = 10080
noNewTTL = 0
noRefreshDelay = 0
)
func TestFeedCategorySetter(t *testing.T) {
@ -76,7 +76,7 @@ func checkTargetInterval(t *testing.T, feed *Feed, targetInterval int, timeBefor
}
}
func TestFeedScheduleNextCheckDefault(t *testing.T) {
func TestFeedScheduleNextCheckRoundRobinDefault(t *testing.T) {
os.Clearenv()
var err error
@ -88,15 +88,60 @@ func TestFeedScheduleNextCheckDefault(t *testing.T) {
timeBefore := time.Now()
feed := &Feed{}
weeklyCount := 10
feed.ScheduleNextCheck(weeklyCount, noNewTTL)
feed.ScheduleNextCheck(0, noRefreshDelay)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
}
targetInterval := config.Opts.SchedulerRoundRobinMinInterval()
checkTargetInterval(t, feed, targetInterval, timeBefore, "default SchedulerRoundRobinMinInterval")
checkTargetInterval(t, feed, targetInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinDefault")
}
func TestFeedScheduleNextCheckRoundRobinWithRefreshDelayAboveMinInterval(t *testing.T) {
os.Clearenv()
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
timeBefore := time.Now()
feed := &Feed{}
feed.ScheduleNextCheck(0, config.Opts.SchedulerRoundRobinMinInterval()+30)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
}
expectedInterval := config.Opts.SchedulerRoundRobinMinInterval() + 30
checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinWithRefreshDelayAboveMinInterval")
}
func TestFeedScheduleNextCheckRoundRobinWithRefreshDelayBelowMinInterval(t *testing.T) {
os.Clearenv()
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
timeBefore := time.Now()
feed := &Feed{}
feed.ScheduleNextCheck(0, config.Opts.SchedulerRoundRobinMinInterval()-30)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
}
expectedInterval := config.Opts.SchedulerRoundRobinMinInterval()
checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinWithRefreshDelayBelowMinInterval")
}
func TestFeedScheduleNextCheckRoundRobinMinInterval(t *testing.T) {
@ -114,15 +159,14 @@ func TestFeedScheduleNextCheckRoundRobinMinInterval(t *testing.T) {
timeBefore := time.Now()
feed := &Feed{}
weeklyCount := 100
feed.ScheduleNextCheck(weeklyCount, noNewTTL)
feed.ScheduleNextCheck(0, noRefreshDelay)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
}
targetInterval := minInterval
checkTargetInterval(t, feed, targetInterval, timeBefore, "round robin min interval")
expectedInterval := minInterval
checkTargetInterval(t, feed, expectedInterval, timeBefore, "TestFeedScheduleNextCheckRoundRobinMinInterval")
}
func TestFeedScheduleNextCheckEntryFrequencyMaxInterval(t *testing.T) {
@ -144,7 +188,7 @@ func TestFeedScheduleNextCheckEntryFrequencyMaxInterval(t *testing.T) {
feed := &Feed{}
// Use a very small weekly count to trigger the max interval
weeklyCount := 1
feed.ScheduleNextCheck(weeklyCount, noNewTTL)
feed.ScheduleNextCheck(weeklyCount, noRefreshDelay)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
@ -173,7 +217,7 @@ func TestFeedScheduleNextCheckEntryFrequencyMaxIntervalZeroWeeklyCount(t *testin
feed := &Feed{}
// Use a very small weekly count to trigger the max interval
weeklyCount := 0
feed.ScheduleNextCheck(weeklyCount, noNewTTL)
feed.ScheduleNextCheck(weeklyCount, noRefreshDelay)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
@ -202,7 +246,7 @@ func TestFeedScheduleNextCheckEntryFrequencyMinInterval(t *testing.T) {
feed := &Feed{}
// Use a very large weekly count to trigger the min interval
weeklyCount := largeWeeklyCount
feed.ScheduleNextCheck(weeklyCount, noNewTTL)
feed.ScheduleNextCheck(weeklyCount, noRefreshDelay)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)
@ -228,7 +272,7 @@ func TestFeedScheduleNextCheckEntryFrequencyFactor(t *testing.T) {
timeBefore := time.Now()
feed := &Feed{}
weeklyCount := 7
feed.ScheduleNextCheck(weeklyCount, noNewTTL)
feed.ScheduleNextCheck(weeklyCount, noRefreshDelay)
if feed.NextCheckAt.IsZero() {
t.Error(`The next_check_at must be set`)