From 535fd050b7e5544b15cc8516b88b3fb067c7e106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Sun, 6 Apr 2025 14:45:56 -0700 Subject: [PATCH] feat: add proxy rotation functionality --- internal/api/subscription.go | 6 +- internal/cli/cli.go | 9 +++ internal/config/config_test.go | 68 +++++++++++++++++++++ internal/config/options.go | 65 ++++++++++++++++---- internal/config/parser.go | 7 ++- internal/googlereader/handler.go | 4 +- internal/proxyrotator/proxyrotator.go | 60 ++++++++++++++++++ internal/proxyrotator/proxyrotator_test.go | 71 ++++++++++++++++++++++ internal/reader/fetcher/request_builder.go | 38 +++++++----- internal/reader/handler/handler.go | 16 +++-- internal/reader/icon/checker.go | 6 +- internal/reader/processor/bilibili.go | 4 +- internal/reader/processor/nebula.go | 4 +- internal/reader/processor/odysee.go | 4 +- internal/reader/processor/processor.go | 11 ++-- internal/reader/processor/youtube.go | 7 ++- internal/ui/feed_edit.go | 2 +- internal/ui/opml_upload.go | 4 +- internal/ui/subscription_add.go | 2 +- internal/ui/subscription_bookmarklet.go | 2 +- internal/ui/subscription_submit.go | 10 +-- miniflux.1 | 7 ++- 22 files changed, 351 insertions(+), 56 deletions(-) create mode 100644 internal/proxyrotator/proxyrotator.go create mode 100644 internal/proxyrotator/proxyrotator_test.go diff --git a/internal/api/subscription.go b/internal/api/subscription.go index 37e68ac0..e5099528 100644 --- a/internal/api/subscription.go +++ b/internal/api/subscription.go @@ -11,6 +11,7 @@ import ( "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/subscription" "miniflux.app/v2/internal/validator" @@ -36,11 +37,12 @@ func (h *handler) discoverSubscriptions(w http.ResponseWriter, r *http.Request) requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(subscriptionDiscoveryRequest.FetchViaProxy) requestBuilder.WithUserAgent(subscriptionDiscoveryRequest.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(subscriptionDiscoveryRequest.Cookie) requestBuilder.WithUsernameAndPassword(subscriptionDiscoveryRequest.Username, subscriptionDiscoveryRequest.Password) - requestBuilder.UseProxy(subscriptionDiscoveryRequest.FetchViaProxy) requestBuilder.IgnoreTLSErrors(subscriptionDiscoveryRequest.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(subscriptionDiscoveryRequest.DisableHTTP2) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index fc074717..de1aa71e 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -13,6 +13,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/database" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/storage" "miniflux.app/v2/internal/ui/static" "miniflux.app/v2/internal/version" @@ -228,6 +229,14 @@ func Parse() { createAdminUserFromEnvironmentVariables(store) } + if config.Opts.HasHTTPClientProxiesConfigured() { + slog.Info("Initializing proxy rotation", slog.Int("proxies_count", len(config.Opts.HTTPClientProxies()))) + proxyrotator.ProxyRotatorInstance, err = proxyrotator.NewProxyRotator(config.Opts.HTTPClientProxies()) + if err != nil { + printErrorAndExit(fmt.Errorf("unable to initialize proxy rotator: %v", err)) + } + } + if flagRefreshFeeds { refreshFeeds(store) return diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bfe40450..6d1ecbdc 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2186,3 +2186,71 @@ func TestParseConfigDumpOutput(t *testing.T) { t.Fatal(err) } } + +func TestHTTPClientProxies(t *testing.T) { + os.Clearenv() + os.Setenv("HTTP_CLIENT_PROXIES", "http://proxy1.example.com,http://proxy2.example.com") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := []string{"http://proxy1.example.com", "http://proxy2.example.com"} + result := opts.HTTPClientProxies() + + if len(expected) != len(result) { + t.Fatalf(`Unexpected HTTP_CLIENT_PROXIES value, got %v instead of %v`, result, expected) + } + + for i, proxy := range expected { + if result[i] != proxy { + t.Fatalf(`Unexpected HTTP_CLIENT_PROXIES value at index %d, got %q instead of %q`, i, result[i], proxy) + } + } +} + +func TestDefaultHTTPClientProxiesValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := []string{} + result := opts.HTTPClientProxies() + + if len(expected) != len(result) { + t.Fatalf(`Unexpected default HTTP_CLIENT_PROXIES value, got %v instead of %v`, result, expected) + } +} + +func TestHTTPClientProxy(t *testing.T) { + os.Clearenv() + os.Setenv("HTTP_CLIENT_PROXY", "http://proxy.example.com") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := "http://proxy.example.com" + if opts.HTTPClientProxyURL() == nil || opts.HTTPClientProxyURL().String() != expected { + t.Fatalf(`Unexpected HTTP_CLIENT_PROXY value, got %v instead of %v`, opts.HTTPClientProxyURL(), expected) + } +} + +func TestInvalidHTTPClientProxy(t *testing.T) { + os.Clearenv() + os.Setenv("HTTP_CLIENT_PROXY", "sche|me://invalid-proxy-url") + + parser := NewParser() + _, err := parser.ParseEnvironmentVariables() + if err == nil { + t.Fatalf(`Expected error for invalid HTTP_CLIENT_PROXY value, but got none`) + } +} diff --git a/internal/config/options.go b/internal/config/options.go index 015a725e..bfe0cf62 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -5,6 +5,7 @@ package config // import "miniflux.app/v2/internal/config" import ( "fmt" + "net/url" "sort" "strings" "time" @@ -163,7 +164,8 @@ type Options struct { pocketConsumerKey string httpClientTimeout int httpClientMaxBodySize int64 - httpClientProxy string + httpClientProxyURL *url.URL + httpClientProxies []string httpClientUserAgent string httpServerTimeout int authProxyHeader string @@ -243,7 +245,8 @@ func NewOptions() *Options { pocketConsumerKey: defaultPocketConsumerKey, httpClientTimeout: defaultHTTPClientTimeout, httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024, - httpClientProxy: defaultHTTPClientProxy, + httpClientProxyURL: nil, + httpClientProxies: []string{}, httpClientUserAgent: defaultHTTPClientUserAgent, httpServerTimeout: defaultHTTPServerTimeout, authProxyHeader: defaultAuthProxyHeader, @@ -587,9 +590,24 @@ func (o *Options) HTTPClientMaxBodySize() int64 { return o.httpClientMaxBodySize } -// HTTPClientProxy returns the proxy URL for HTTP client. -func (o *Options) HTTPClientProxy() string { - return o.httpClientProxy +// HTTPClientProxyURL returns the client HTTP proxy URL if configured. +func (o *Options) HTTPClientProxyURL() *url.URL { + return o.httpClientProxyURL +} + +// HasHTTPClientProxyURLConfigured returns true if the client HTTP proxy URL if configured. +func (o *Options) HasHTTPClientProxyURLConfigured() bool { + return o.httpClientProxyURL != nil +} + +// HTTPClientProxies returns the list of proxies. +func (o *Options) HTTPClientProxies() []string { + return o.httpClientProxies +} + +// HTTPClientProxiesString returns true if the list of rotating proxies are configured. +func (o *Options) HasHTTPClientProxiesConfigured() bool { + return len(o.httpClientProxies) > 0 } // HTTPServerTimeout returns the time limit in seconds before the HTTP server cancel the request. @@ -597,11 +615,6 @@ func (o *Options) HTTPServerTimeout() int { return o.httpServerTimeout } -// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured. -func (o *Options) HasHTTPClientProxyConfigured() bool { - return o.httpClientProxy != "" -} - // AuthProxyHeader returns an HTTP header name that contains username for // authentication using auth proxy. func (o *Options) AuthProxyHeader() string { @@ -664,6 +677,33 @@ func (o *Options) FilterEntryMaxAgeDays() int { // SortedOptions returns options as a list of key value pairs, sorted by keys. func (o *Options) SortedOptions(redactSecret bool) []*Option { + var clientProxyURLRedacted string + if o.httpClientProxyURL != nil { + if redactSecret { + clientProxyURLRedacted = o.httpClientProxyURL.Redacted() + } else { + clientProxyURLRedacted = o.httpClientProxyURL.String() + } + } + + var clientProxyURLsRedacted string + if len(o.httpClientProxies) > 0 { + if redactSecret { + var proxyURLs []string + for range o.httpClientProxies { + proxyURLs = append(proxyURLs, "") + } + clientProxyURLsRedacted = strings.Join(proxyURLs, ",") + } else { + clientProxyURLsRedacted = strings.Join(o.httpClientProxies, ",") + } + } + + var mediaProxyPrivateKeyValue string + if len(o.mediaProxyPrivateKey) > 0 { + mediaProxyPrivateKeyValue = "" + } + var keyValues = map[string]interface{}{ "ADMIN_PASSWORD": redactSecretValue(o.adminPassword, redactSecret), "ADMIN_USERNAME": o.adminUsername, @@ -694,7 +734,8 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "FETCH_BILIBILI_WATCH_TIME": o.fetchBilibiliWatchTime, "HTTPS": o.HTTPS, "HTTP_CLIENT_MAX_BODY_SIZE": o.httpClientMaxBodySize, - "HTTP_CLIENT_PROXY": o.httpClientProxy, + "HTTP_CLIENT_PROXIES": clientProxyURLsRedacted, + "HTTP_CLIENT_PROXY": clientProxyURLRedacted, "HTTP_CLIENT_TIMEOUT": o.httpClientTimeout, "HTTP_CLIENT_USER_AGENT": o.httpClientUserAgent, "HTTP_SERVER_TIMEOUT": o.httpServerTimeout, @@ -729,7 +770,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout, "MEDIA_PROXY_RESOURCE_TYPES": o.mediaProxyResourceTypes, "MEDIA_PROXY_MODE": o.mediaProxyMode, - "MEDIA_PROXY_PRIVATE_KEY": redactSecretValue(string(o.mediaProxyPrivateKey), redactSecret), + "MEDIA_PROXY_PRIVATE_KEY": mediaProxyPrivateKeyValue, "MEDIA_PROXY_CUSTOM_URL": o.mediaProxyCustomURL, "ROOT_URL": o.rootURL, "RUN_MIGRATIONS": o.runMigrations, diff --git a/internal/config/parser.go b/internal/config/parser.go index a3b30190..abf8e5ef 100644 --- a/internal/config/parser.go +++ b/internal/config/parser.go @@ -240,7 +240,12 @@ func (p *Parser) parseLines(lines []string) (err error) { case "HTTP_CLIENT_MAX_BODY_SIZE": p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024) case "HTTP_CLIENT_PROXY": - p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy) + p.opts.httpClientProxyURL, err = url.Parse(parseString(value, defaultHTTPClientProxy)) + if err != nil { + return fmt.Errorf("config: invalid HTTP_CLIENT_PROXY value: %w", err) + } + case "HTTP_CLIENT_PROXIES": + p.opts.httpClientProxies = parseStringList(value, []string{}) case "HTTP_CLIENT_USER_AGENT": p.opts.httpClientUserAgent = parseString(value, defaultHTTPClientUserAgent) case "HTTP_SERVER_TIMEOUT": diff --git a/internal/googlereader/handler.go b/internal/googlereader/handler.go index f4569380..9764edbf 100644 --- a/internal/googlereader/handler.go +++ b/internal/googlereader/handler.go @@ -20,6 +20,7 @@ import ( "miniflux.app/v2/internal/integration" "miniflux.app/v2/internal/mediaproxy" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" mff "miniflux.app/v2/internal/reader/handler" mfs "miniflux.app/v2/internal/reader/subscription" @@ -683,7 +684,8 @@ func (h *handler) quickAddHandler(w http.ResponseWriter, r *http.Request) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) var rssBridgeURL string if intg, err := h.store.Integration(userID); err == nil && intg != nil && intg.RSSBridgeEnabled { diff --git a/internal/proxyrotator/proxyrotator.go b/internal/proxyrotator/proxyrotator.go new file mode 100644 index 00000000..03c791fc --- /dev/null +++ b/internal/proxyrotator/proxyrotator.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package proxyrotator // import "miniflux.app/v2/internal/proxyrotator" + +import ( + "net/url" + "sync" +) + +var ProxyRotatorInstance *ProxyRotator + +// ProxyRotator manages a list of proxies and rotates through them. +type ProxyRotator struct { + proxies []*url.URL + currentIndex int + mutex sync.Mutex +} + +// NewProxyRotator creates a new ProxyRotator with the given proxy URLs. +func NewProxyRotator(proxyURLs []string) (*ProxyRotator, error) { + parsedProxies := make([]*url.URL, 0, len(proxyURLs)) + + for _, p := range proxyURLs { + proxyURL, err := url.Parse(p) + if err != nil { + return nil, err + } + parsedProxies = append(parsedProxies, proxyURL) + } + + return &ProxyRotator{ + proxies: parsedProxies, + currentIndex: 0, + mutex: sync.Mutex{}, + }, nil +} + +// GetNextProxy returns the next proxy in the rotation. +func (pr *ProxyRotator) GetNextProxy() *url.URL { + pr.mutex.Lock() + defer pr.mutex.Unlock() + + if len(pr.proxies) == 0 { + return nil + } + + proxy := pr.proxies[pr.currentIndex] + pr.currentIndex = (pr.currentIndex + 1) % len(pr.proxies) + + return proxy +} + +// HasProxies checks if there are any proxies available in the rotator. +func (pr *ProxyRotator) HasProxies() bool { + pr.mutex.Lock() + defer pr.mutex.Unlock() + + return len(pr.proxies) > 0 +} diff --git a/internal/proxyrotator/proxyrotator_test.go b/internal/proxyrotator/proxyrotator_test.go new file mode 100644 index 00000000..06afae4e --- /dev/null +++ b/internal/proxyrotator/proxyrotator_test.go @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package proxyrotator // import "miniflux.app/v2/internal/proxyrotator" + +import ( + "testing" +) + +func TestProxyRotator(t *testing.T) { + proxyURLs := []string{ + "http://proxy1.example.com", + "http://proxy2.example.com", + "http://proxy3.example.com", + } + + rotator, err := NewProxyRotator(proxyURLs) + if err != nil { + t.Fatalf("Failed to create ProxyRotator: %v", err) + } + + if !rotator.HasProxies() { + t.Fatalf("Expected rotator to have proxies") + } + + seenProxies := make(map[string]bool) + for range len(proxyURLs) * 2 { + proxy := rotator.GetNextProxy() + if proxy == nil { + t.Fatalf("Expected a proxy, got nil") + } + + seenProxies[proxy.String()] = true + } + + if len(seenProxies) != len(proxyURLs) { + t.Fatalf("Expected to see all proxies, but saw: %v", seenProxies) + } +} + +func TestProxyRotatorEmpty(t *testing.T) { + rotator, err := NewProxyRotator([]string{}) + if err != nil { + t.Fatalf("Failed to create ProxyRotator: %v", err) + } + + if rotator.HasProxies() { + t.Fatalf("Expected rotator to have no proxies") + } + + proxy := rotator.GetNextProxy() + if proxy != nil { + t.Fatalf("Expected no proxy, got: %v", proxy) + } +} + +func TestProxyRotatorInvalidURL(t *testing.T) { + invalidProxyURLs := []string{ + "http://validproxy.example.com", + "test|test://invalidproxy.example.com", + } + + rotator, err := NewProxyRotator(invalidProxyURLs) + if err == nil { + t.Fatalf("Expected an error when creating ProxyRotator with invalid URLs, but got none") + } + + if rotator != nil { + t.Fatalf("Expected rotator to be nil when initialization fails, but got: %v", rotator) + } +} diff --git a/internal/reader/fetcher/request_builder.go b/internal/reader/fetcher/request_builder.go index 5ed10a51..a71ea1a4 100644 --- a/internal/reader/fetcher/request_builder.go +++ b/internal/reader/fetcher/request_builder.go @@ -11,6 +11,8 @@ import ( "net/http" "net/url" "time" + + "miniflux.app/v2/internal/proxyrotator" ) const ( @@ -21,12 +23,13 @@ const ( type RequestBuilder struct { headers http.Header - clientProxyURL string + clientProxyURL *url.URL useClientProxy bool clientTimeout int withoutRedirects bool ignoreTLSErrors bool disableHTTP2 bool + proxyRotator *proxyrotator.ProxyRotator } func NewRequestBuilder() *RequestBuilder { @@ -78,12 +81,17 @@ func (r *RequestBuilder) WithUsernameAndPassword(username, password string) *Req return r } -func (r *RequestBuilder) WithProxy(proxyURL string) *RequestBuilder { +func (r *RequestBuilder) WithProxyRotator(proxyRotator *proxyrotator.ProxyRotator) *RequestBuilder { + r.proxyRotator = proxyRotator + return r +} + +func (r *RequestBuilder) WithCustomApplicationProxyURL(proxyURL *url.URL) *RequestBuilder { r.clientProxyURL = proxyURL return r } -func (r *RequestBuilder) UseProxy(value bool) *RequestBuilder { +func (r *RequestBuilder) UseCustomApplicationProxyURL(value bool) *RequestBuilder { r.useClientProxy = value return r } @@ -151,15 +159,17 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} } - if r.useClientProxy && r.clientProxyURL != "" { - if proxyURL, err := url.Parse(r.clientProxyURL); err != nil { - slog.Warn("Unable to parse proxy URL", - slog.String("proxy_url", r.clientProxyURL), - slog.Any("error", err), - ) - } else { - transport.Proxy = http.ProxyURL(proxyURL) - } + var clientProxyURL *url.URL + if r.useClientProxy && r.clientProxyURL != nil { + clientProxyURL = r.clientProxyURL + } else if r.proxyRotator != nil && r.proxyRotator.HasProxies() { + clientProxyURL = r.proxyRotator.GetNextProxy() + } + + var clientProxyURLRedacted string + if clientProxyURL != nil { + transport.Proxy = http.ProxyURL(clientProxyURL) + clientProxyURLRedacted = clientProxyURL.Redacted() } client := &http.Client{ @@ -189,8 +199,8 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro slog.String("url", req.URL.String()), slog.Any("headers", req.Header), slog.Bool("without_redirects", r.withoutRedirects), - slog.Bool("with_proxy", r.useClientProxy), - slog.String("proxy_url", r.clientProxyURL), + slog.Bool("use_app_client_proxy", r.useClientProxy), + slog.String("client_proxy_url", clientProxyURLRedacted), slog.Bool("ignore_tls_errors", r.ignoreTLSErrors), slog.Bool("disable_http2", r.disableHTTP2), )) diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 937d7b78..4c211790 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -12,6 +12,7 @@ import ( "miniflux.app/v2/internal/integration" "miniflux.app/v2/internal/locale" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/icon" "miniflux.app/v2/internal/reader/parser" @@ -83,8 +84,9 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feedCreationRequest.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy) requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2) @@ -109,8 +111,9 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feedCreationRequest.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy) requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2) @@ -212,8 +215,9 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(originalFeed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(originalFeed.FetchViaProxy) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(originalFeed.FetchViaProxy) requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(originalFeed.DisableHTTP2) diff --git a/internal/reader/icon/checker.go b/internal/reader/icon/checker.go index a639844b..45a1b519 100644 --- a/internal/reader/icon/checker.go +++ b/internal/reader/icon/checker.go @@ -8,6 +8,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/storage" ) @@ -29,8 +30,9 @@ func (c *IconChecker) fetchAndStoreIcon() { requestBuilder.WithUserAgent(c.feed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(c.feed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(c.feed.FetchViaProxy) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(c.feed.FetchViaProxy) requestBuilder.IgnoreTLSErrors(c.feed.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(c.feed.DisableHTTP2) diff --git a/internal/reader/processor/bilibili.go b/internal/reader/processor/bilibili.go index 4450fcc3..fffc5bd3 100644 --- a/internal/reader/processor/bilibili.go +++ b/internal/reader/processor/bilibili.go @@ -11,6 +11,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" ) @@ -45,7 +46,8 @@ func extractBilibiliVideoID(websiteURL string) (string, string, error) { func fetchBilibiliWatchTime(websiteURL string) (int, error) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) idType, videoID, extractErr := extractBilibiliVideoID(websiteURL) if extractErr != nil { diff --git a/internal/reader/processor/nebula.go b/internal/reader/processor/nebula.go index afe7e302..9193b50f 100644 --- a/internal/reader/processor/nebula.go +++ b/internal/reader/processor/nebula.go @@ -14,6 +14,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" ) @@ -33,7 +34,8 @@ func shouldFetchNebulaWatchTime(entry *model.Entry) bool { func fetchNebulaWatchTime(websiteURL string) (int, error) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) defer responseHandler.Close() diff --git a/internal/reader/processor/odysee.go b/internal/reader/processor/odysee.go index 4748dc5c..4e4a82b4 100644 --- a/internal/reader/processor/odysee.go +++ b/internal/reader/processor/odysee.go @@ -14,6 +14,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" ) @@ -33,7 +34,8 @@ func shouldFetchOdyseeWatchTime(entry *model.Entry) bool { func fetchOdyseeWatchTime(websiteURL string) (int, error) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) defer responseHandler.Close() diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go index d4485d91..2c433728 100644 --- a/internal/reader/processor/processor.go +++ b/internal/reader/processor/processor.go @@ -14,6 +14,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/metric" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/readingtime" "miniflux.app/v2/internal/reader/rewrite" @@ -78,8 +79,9 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64, requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(feed.FetchViaProxy) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy) requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(feed.DisableHTTP2) @@ -145,8 +147,9 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(feed.FetchViaProxy) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy) requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(feed.DisableHTTP2) diff --git a/internal/reader/processor/youtube.go b/internal/reader/processor/youtube.go index c67c02d9..fcb07d4b 100644 --- a/internal/reader/processor/youtube.go +++ b/internal/reader/processor/youtube.go @@ -18,6 +18,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" ) @@ -52,7 +53,8 @@ func fetchYouTubeWatchTimeForSingleEntry(websiteURL string) (int, error) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) defer responseHandler.Close() @@ -132,7 +134,8 @@ func fetchYouTubeWatchTimeFromApiInBulk(videoIDs []string) (map[string]time.Dura requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String())) defer responseHandler.Close() diff --git a/internal/ui/feed_edit.go b/internal/ui/feed_edit.go index c58d60a3..d3ade628 100644 --- a/internal/ui/feed_edit.go +++ b/internal/ui/feed_edit.go @@ -82,7 +82,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) { view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent()) - view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured()) html.OK(w, r, view.Render("edit_feed")) } diff --git a/internal/ui/opml_upload.go b/internal/ui/opml_upload.go index ed05cc2f..6dd58410 100644 --- a/internal/ui/opml_upload.go +++ b/internal/ui/opml_upload.go @@ -13,6 +13,7 @@ import ( "miniflux.app/v2/internal/http/response/html" "miniflux.app/v2/internal/http/route" "miniflux.app/v2/internal/locale" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/opml" "miniflux.app/v2/internal/ui/session" @@ -95,7 +96,8 @@ func (h *handler) fetchOPML(w http.ResponseWriter, r *http.Request) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(opmlFileURL)) defer responseHandler.Close() diff --git a/internal/ui/subscription_add.go b/internal/ui/subscription_add.go index ca56a973..0347c91b 100644 --- a/internal/ui/subscription_add.go +++ b/internal/ui/subscription_add.go @@ -36,7 +36,7 @@ func (h *handler) showAddSubscriptionPage(w http.ResponseWriter, r *http.Request view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent()) view.Set("form", &form.SubscriptionForm{CategoryID: 0}) - view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured()) html.OK(w, r, view.Render("add_subscription")) } diff --git a/internal/ui/subscription_bookmarklet.go b/internal/ui/subscription_bookmarklet.go index dfc4dfd2..bc198439 100644 --- a/internal/ui/subscription_bookmarklet.go +++ b/internal/ui/subscription_bookmarklet.go @@ -53,7 +53,7 @@ func (h *handler) bookmarklet(w http.ResponseWriter, r *http.Request) { view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent()) - view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured()) html.OK(w, r, view.Render("add_subscription")) } diff --git a/internal/ui/subscription_submit.go b/internal/ui/subscription_submit.go index fd8e43ef..4ba9f8c1 100644 --- a/internal/ui/subscription_submit.go +++ b/internal/ui/subscription_submit.go @@ -12,6 +12,7 @@ import ( "miniflux.app/v2/internal/http/route" "miniflux.app/v2/internal/locale" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" "miniflux.app/v2/internal/reader/fetcher" feedHandler "miniflux.app/v2/internal/reader/handler" "miniflux.app/v2/internal/reader/subscription" @@ -41,7 +42,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { v.Set("countUnread", h.store.CountUnreadEntries(user.ID)) v.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) v.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent()) - v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) + v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured()) subscriptionForm := form.NewSubscriptionForm(r) if validationErr := subscriptionForm.Validate(); validationErr != nil { @@ -58,11 +59,12 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(subscriptionForm.FetchViaProxy) requestBuilder.WithUserAgent(subscriptionForm.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(subscriptionForm.Cookie) requestBuilder.WithUsernameAndPassword(subscriptionForm.Username, subscriptionForm.Password) - requestBuilder.UseProxy(subscriptionForm.FetchViaProxy) requestBuilder.IgnoreTLSErrors(subscriptionForm.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(subscriptionForm.DisableHTTP2) @@ -149,7 +151,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { view.Set("user", user) view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) - view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured()) html.OK(w, r, view.Render("choose_subscription")) } diff --git a/miniflux.1 b/miniflux.1 index 05093890..4ac89a82 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -293,8 +293,13 @@ Maximum body size for HTTP requests in Mebibyte (MiB)\&. .br Default is 15 MiB\&. .TP +.B HTTP_CLIENT_PROXIES +Enable proxy rotation for outgoing requests by providing a comma-separated list of proxy URLs\&. +.br +Default is empty\&. +.TP .B HTTP_CLIENT_PROXY -Proxy URL for HTTP client\&. +Proxy URL to use when the "Fetch via proxy" feed option is enabled\&. .br Default is empty\&. .TP