oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-08-11 17:51:01 +00:00
Code Wiki Activity

feat: add proxy rotation functionality

Browse source
This commit is contained in:
Frédéric Guillot 2025-04-06 14:45:56 -07:00
parent d20e8a4e2c
commit c24209dde4
22 changed files with 351 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

38
internal/reader/fetcher/request_builder.go
View file

@ -11,6 +11,8 @@ import (
"net/http"
"net/url"
"time"
"miniflux.app/v2/internal/proxyrotator"
)
const (
@ -21,12 +23,13 @@ const (
type RequestBuilder struct {
headers http.Header
clientProxyURL string
clientProxyURL *url.URL
useClientProxy bool
clientTimeout int
withoutRedirects bool
ignoreTLSErrors bool
disableHTTP2 bool
proxyRotator *proxyrotator.ProxyRotator
}
func NewRequestBuilder() *RequestBuilder {
@ -78,12 +81,17 @@ func (r *RequestBuilder) WithUsernameAndPassword(username, password string) *Req
return r
}
func (r *RequestBuilder) WithProxy(proxyURL string) *RequestBuilder {
func (r *RequestBuilder) WithProxyRotator(proxyRotator *proxyrotator.ProxyRotator) *RequestBuilder {
r.proxyRotator = proxyRotator
return r
}
func (r *RequestBuilder) WithCustomApplicationProxyURL(proxyURL *url.URL) *RequestBuilder {
r.clientProxyURL = proxyURL
return r
}
func (r *RequestBuilder) UseProxy(value bool) *RequestBuilder {
func (r *RequestBuilder) UseCustomApplicationProxyURL(value bool) *RequestBuilder {
r.useClientProxy = value
return r
}
@ -151,15 +159,17 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
}
if r.useClientProxy && r.clientProxyURL != "" {
if proxyURL, err := url.Parse(r.clientProxyURL); err != nil {
slog.Warn("Unable to parse proxy URL",
slog.String("proxy_url", r.clientProxyURL),
slog.Any("error", err),
)
} else {
transport.Proxy = http.ProxyURL(proxyURL)
}
var clientProxyURL *url.URL
if r.useClientProxy && r.clientProxyURL != nil {
clientProxyURL = r.clientProxyURL
} else if r.proxyRotator != nil && r.proxyRotator.HasProxies() {
clientProxyURL = r.proxyRotator.GetNextProxy()
}
var clientProxyURLRedacted string
if clientProxyURL != nil {
transport.Proxy = http.ProxyURL(clientProxyURL)
clientProxyURLRedacted = clientProxyURL.Redacted()
}
client := &http.Client{
@ -189,8 +199,8 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
slog.String("url", req.URL.String()),
slog.Any("headers", req.Header),
slog.Bool("without_redirects", r.withoutRedirects),
slog.Bool("with_proxy", r.useClientProxy),
slog.String("proxy_url", r.clientProxyURL),
slog.Bool("use_app_client_proxy", r.useClientProxy),
slog.String("client_proxy_url", clientProxyURLRedacted),
slog.Bool("ignore_tls_errors", r.ignoreTLSErrors),
slog.Bool("disable_http2", r.disableHTTP2),
))

16
internal/reader/handler/handler.go
View file

@ -12,6 +12,7 @@ import (
"miniflux.app/v2/internal/integration"
"miniflux.app/v2/internal/locale"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
"miniflux.app/v2/internal/reader/icon"
"miniflux.app/v2/internal/reader/parser"
@ -83,8 +84,9 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(feedCreationRequest.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy)
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2)
@ -109,8 +111,9 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(feedCreationRequest.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy)
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2)
@ -212,8 +215,9 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(originalFeed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(originalFeed.FetchViaProxy)
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(originalFeed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(originalFeed.DisableHTTP2)

6
internal/reader/icon/checker.go
View file

@ -8,6 +8,7 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
"miniflux.app/v2/internal/storage"
)
@ -29,8 +30,9 @@ func (c *IconChecker) fetchAndStoreIcon() {
requestBuilder.WithUserAgent(c.feed.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(c.feed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(c.feed.FetchViaProxy)
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(c.feed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(c.feed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(c.feed.DisableHTTP2)

4
internal/reader/processor/bilibili.go
View file

@ -11,6 +11,7 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
)
@ -45,7 +46,8 @@ func extractBilibiliVideoID(websiteURL string) (string, string, error) {
func fetchBilibiliWatchTime(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
idType, videoID, extractErr := extractBilibiliVideoID(websiteURL)
if extractErr != nil {

4
internal/reader/processor/nebula.go
View file

@ -14,6 +14,7 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
)
@ -33,7 +34,8 @@ func shouldFetchNebulaWatchTime(entry *model.Entry) bool {
func fetchNebulaWatchTime(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
defer responseHandler.Close()

4
internal/reader/processor/odysee.go
View file

@ -14,6 +14,7 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
)
@ -33,7 +34,8 @@ func shouldFetchOdyseeWatchTime(entry *model.Entry) bool {
func fetchOdyseeWatchTime(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
defer responseHandler.Close()

11
internal/reader/processor/processor.go
View file

@ -14,6 +14,7 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/metric"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
"miniflux.app/v2/internal/reader/readingtime"
"miniflux.app/v2/internal/reader/rewrite"
@ -78,8 +79,9 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(feed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(feed.FetchViaProxy)
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(feed.DisableHTTP2)
@ -145,8 +147,9 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(feed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(feed.FetchViaProxy)
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(feed.DisableHTTP2)

7
internal/reader/processor/youtube.go
View file

@ -18,6 +18,7 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
)
@ -52,7 +53,8 @@ func fetchYouTubeWatchTimeForSingleEntry(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
defer responseHandler.Close()
@ -132,7 +134,8 @@ func fetchYouTubeWatchTimeFromApiInBulk(videoIDs []string) (map[string]time.Dura
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
defer responseHandler.Close()