feat: take Retry-After header into consideration for rate limited feeds

internal/reader/fetcher/response_handler.go

@@ -13,7 +13,9 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
+	"time"
 
 	"miniflux.app/v2/internal/locale"
 )
@@ -51,6 +53,26 @@ func (r *ResponseHandler) ETag() string {
 	return r.httpResponse.Header.Get("ETag")
 }
 
+func (r *ResponseHandler) ParseRetryDelay() int {
+	retryAfterHeaderValue := r.httpResponse.Header.Get("Retry-After")
+	if retryAfterHeaderValue != "" {
+		// First, try to parse as an integer (number of seconds)
+		if seconds, err := strconv.Atoi(retryAfterHeaderValue); err == nil {
+			return seconds
+		}
+
+		// If not an integer, try to parse as an HTTP-date
+		if t, err := time.Parse(time.RFC1123, retryAfterHeaderValue); err == nil {
+			return int(time.Until(t).Seconds())
+		}
+	}
+	return 0
+}
+
+func (r *ResponseHandler) IsRateLimited() bool {
+	return r.httpResponse.StatusCode == http.StatusTooManyRequests
+}
+
 func (r *ResponseHandler) IsModified(lastEtagValue, lastModifiedValue string) bool {
 	if r.httpResponse.StatusCode == http.StatusNotModified {
 		return false

internal/reader/fetcher/response_handler_test.go

@@ -6,6 +6,7 @@ package fetcher // import "miniflux.app/v2/internal/reader/fetcher"
 import (
 	"net/http"
 	"testing"
+	"time"
 )
 
 func TestIsModified(t *testing.T) {
@@ -67,3 +68,37 @@ func TestIsModified(t *testing.T) {
 		})
 	}
 }
+
+func TestRetryDelay(t *testing.T) {
+	var testCases = map[string]struct {
+		RetryAfterHeader string
+		ExpectedDelay    int
+	}{
+		"Empty header": {
+			RetryAfterHeader: "",
+			ExpectedDelay:    0,
+		},
+		"Integer value": {
+			RetryAfterHeader: "42",
+			ExpectedDelay:    42,
+		},
+		"HTTP-date": {
+			RetryAfterHeader: time.Now().Add(42 * time.Second).Format(time.RFC1123),
+			ExpectedDelay:    41,
+		},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(tt *testing.T) {
+			header := http.Header{}
+			header.Add("Retry-After", tc.RetryAfterHeader)
+			rh := ResponseHandler{
+				httpResponse: &http.Response{
+					Header: header,
+				},
+			}
+			if tc.ExpectedDelay != rh.ParseRetryDelay() {
+				tt.Errorf("Expected %d, got %d for scenario %q", tc.ExpectedDelay, rh.ParseRetryDelay(), name)
+			}
+		})
+	}
+}

internal/reader/handler/handler.go

@@ -210,7 +210,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 	}
 
 	weeklyEntryCount := 0
-	newTTL := 0
+	refreshDelayInMinutes := 0
 	if config.Opts.PollingScheduler() == model.SchedulerEntryFrequency {
 		var weeklyCountErr error
 		weeklyEntryCount, weeklyCountErr = store.WeeklyFeedEntryCount(userID, feedID)
@@ -220,7 +220,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 	}
 
 	originalFeed.CheckedNow()
-	originalFeed.ScheduleNextCheck(weeklyEntryCount, newTTL)
+	originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
 
 	requestBuilder := fetcher.NewRequestBuilder()
 	requestBuilder.WithUsernameAndPassword(originalFeed.Username, originalFeed.Password)
@@ -241,6 +241,19 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 	responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(originalFeed.FeedURL))
 	defer responseHandler.Close()
 
+	if responseHandler.IsRateLimited() {
+		retryDelayInSeconds := responseHandler.ParseRetryDelay()
+		refreshDelayInMinutes = retryDelayInSeconds / 60
+		originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
+
+		slog.Warn("Feed is rate limited",
+			slog.String("feed_url", originalFeed.FeedURL),
+			slog.Int("retry_delay_in_seconds", retryDelayInSeconds),
+			slog.Int("refresh_delay_in_minutes", refreshDelayInMinutes),
+			slog.Time("new_next_check_at", originalFeed.NextCheckAt),
+		)
+	}
+
 	if localizedError := responseHandler.LocalizedError(); localizedError != nil {
 		slog.Warn("Unable to fetch feed", slog.String("feed_url", originalFeed.FeedURL), slog.Any("error", localizedError.Error()))
 		originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
@@ -283,15 +296,15 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 		}
 
 		// If the feed has a TTL defined, we use it to make sure we don't check it too often.
-		newTTL = updatedFeed.TTL
+		refreshDelayInMinutes = updatedFeed.TTL
 
 		// Set the next check at with updated arguments.
-		originalFeed.ScheduleNextCheck(weeklyEntryCount, newTTL)
+		originalFeed.ScheduleNextCheck(weeklyEntryCount, refreshDelayInMinutes)
 
 		slog.Debug("Updated next check date",
 			slog.Int64("user_id", userID),
 			slog.Int64("feed_id", feedID),
-			slog.Int("ttl", newTTL),
+			slog.Int("refresh_delay_in_minutes", refreshDelayInMinutes),
 			slog.Time("new_next_check_at", originalFeed.NextCheckAt),
 		)