mirror of
https://github.com/miniflux/v2.git
synced 2025-06-27 16:36:00 +00:00
feat: add proxy rotation functionality
This commit is contained in:
parent
d20e8a4e2c
commit
535fd050b7
22 changed files with 351 additions and 56 deletions
|
@ -11,6 +11,7 @@ import (
|
||||||
"miniflux.app/v2/internal/http/request"
|
"miniflux.app/v2/internal/http/request"
|
||||||
"miniflux.app/v2/internal/http/response/json"
|
"miniflux.app/v2/internal/http/response/json"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/reader/subscription"
|
"miniflux.app/v2/internal/reader/subscription"
|
||||||
"miniflux.app/v2/internal/validator"
|
"miniflux.app/v2/internal/validator"
|
||||||
|
@ -36,11 +37,12 @@ func (h *handler) discoverSubscriptions(w http.ResponseWriter, r *http.Request)
|
||||||
|
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(subscriptionDiscoveryRequest.FetchViaProxy)
|
||||||
requestBuilder.WithUserAgent(subscriptionDiscoveryRequest.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(subscriptionDiscoveryRequest.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(subscriptionDiscoveryRequest.Cookie)
|
requestBuilder.WithCookie(subscriptionDiscoveryRequest.Cookie)
|
||||||
requestBuilder.WithUsernameAndPassword(subscriptionDiscoveryRequest.Username, subscriptionDiscoveryRequest.Password)
|
requestBuilder.WithUsernameAndPassword(subscriptionDiscoveryRequest.Username, subscriptionDiscoveryRequest.Password)
|
||||||
requestBuilder.UseProxy(subscriptionDiscoveryRequest.FetchViaProxy)
|
|
||||||
requestBuilder.IgnoreTLSErrors(subscriptionDiscoveryRequest.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(subscriptionDiscoveryRequest.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(subscriptionDiscoveryRequest.DisableHTTP2)
|
requestBuilder.DisableHTTP2(subscriptionDiscoveryRequest.DisableHTTP2)
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/database"
|
"miniflux.app/v2/internal/database"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/storage"
|
"miniflux.app/v2/internal/storage"
|
||||||
"miniflux.app/v2/internal/ui/static"
|
"miniflux.app/v2/internal/ui/static"
|
||||||
"miniflux.app/v2/internal/version"
|
"miniflux.app/v2/internal/version"
|
||||||
|
@ -228,6 +229,14 @@ func Parse() {
|
||||||
createAdminUserFromEnvironmentVariables(store)
|
createAdminUserFromEnvironmentVariables(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.Opts.HasHTTPClientProxiesConfigured() {
|
||||||
|
slog.Info("Initializing proxy rotation", slog.Int("proxies_count", len(config.Opts.HTTPClientProxies())))
|
||||||
|
proxyrotator.ProxyRotatorInstance, err = proxyrotator.NewProxyRotator(config.Opts.HTTPClientProxies())
|
||||||
|
if err != nil {
|
||||||
|
printErrorAndExit(fmt.Errorf("unable to initialize proxy rotator: %v", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if flagRefreshFeeds {
|
if flagRefreshFeeds {
|
||||||
refreshFeeds(store)
|
refreshFeeds(store)
|
||||||
return
|
return
|
||||||
|
|
|
@ -2186,3 +2186,71 @@ func TestParseConfigDumpOutput(t *testing.T) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientProxies(t *testing.T) {
|
||||||
|
os.Clearenv()
|
||||||
|
os.Setenv("HTTP_CLIENT_PROXIES", "http://proxy1.example.com,http://proxy2.example.com")
|
||||||
|
|
||||||
|
parser := NewParser()
|
||||||
|
opts, err := parser.ParseEnvironmentVariables()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf(`Parsing failure: %v`, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := []string{"http://proxy1.example.com", "http://proxy2.example.com"}
|
||||||
|
result := opts.HTTPClientProxies()
|
||||||
|
|
||||||
|
if len(expected) != len(result) {
|
||||||
|
t.Fatalf(`Unexpected HTTP_CLIENT_PROXIES value, got %v instead of %v`, result, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, proxy := range expected {
|
||||||
|
if result[i] != proxy {
|
||||||
|
t.Fatalf(`Unexpected HTTP_CLIENT_PROXIES value at index %d, got %q instead of %q`, i, result[i], proxy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultHTTPClientProxiesValue(t *testing.T) {
|
||||||
|
os.Clearenv()
|
||||||
|
|
||||||
|
parser := NewParser()
|
||||||
|
opts, err := parser.ParseEnvironmentVariables()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf(`Parsing failure: %v`, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := []string{}
|
||||||
|
result := opts.HTTPClientProxies()
|
||||||
|
|
||||||
|
if len(expected) != len(result) {
|
||||||
|
t.Fatalf(`Unexpected default HTTP_CLIENT_PROXIES value, got %v instead of %v`, result, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientProxy(t *testing.T) {
|
||||||
|
os.Clearenv()
|
||||||
|
os.Setenv("HTTP_CLIENT_PROXY", "http://proxy.example.com")
|
||||||
|
|
||||||
|
parser := NewParser()
|
||||||
|
opts, err := parser.ParseEnvironmentVariables()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf(`Parsing failure: %v`, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := "http://proxy.example.com"
|
||||||
|
if opts.HTTPClientProxyURL() == nil || opts.HTTPClientProxyURL().String() != expected {
|
||||||
|
t.Fatalf(`Unexpected HTTP_CLIENT_PROXY value, got %v instead of %v`, opts.HTTPClientProxyURL(), expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInvalidHTTPClientProxy(t *testing.T) {
|
||||||
|
os.Clearenv()
|
||||||
|
os.Setenv("HTTP_CLIENT_PROXY", "sche|me://invalid-proxy-url")
|
||||||
|
|
||||||
|
parser := NewParser()
|
||||||
|
_, err := parser.ParseEnvironmentVariables()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf(`Expected error for invalid HTTP_CLIENT_PROXY value, but got none`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@ package config // import "miniflux.app/v2/internal/config"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/url"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
@ -163,7 +164,8 @@ type Options struct {
|
||||||
pocketConsumerKey string
|
pocketConsumerKey string
|
||||||
httpClientTimeout int
|
httpClientTimeout int
|
||||||
httpClientMaxBodySize int64
|
httpClientMaxBodySize int64
|
||||||
httpClientProxy string
|
httpClientProxyURL *url.URL
|
||||||
|
httpClientProxies []string
|
||||||
httpClientUserAgent string
|
httpClientUserAgent string
|
||||||
httpServerTimeout int
|
httpServerTimeout int
|
||||||
authProxyHeader string
|
authProxyHeader string
|
||||||
|
@ -243,7 +245,8 @@ func NewOptions() *Options {
|
||||||
pocketConsumerKey: defaultPocketConsumerKey,
|
pocketConsumerKey: defaultPocketConsumerKey,
|
||||||
httpClientTimeout: defaultHTTPClientTimeout,
|
httpClientTimeout: defaultHTTPClientTimeout,
|
||||||
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
|
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
|
||||||
httpClientProxy: defaultHTTPClientProxy,
|
httpClientProxyURL: nil,
|
||||||
|
httpClientProxies: []string{},
|
||||||
httpClientUserAgent: defaultHTTPClientUserAgent,
|
httpClientUserAgent: defaultHTTPClientUserAgent,
|
||||||
httpServerTimeout: defaultHTTPServerTimeout,
|
httpServerTimeout: defaultHTTPServerTimeout,
|
||||||
authProxyHeader: defaultAuthProxyHeader,
|
authProxyHeader: defaultAuthProxyHeader,
|
||||||
|
@ -587,9 +590,24 @@ func (o *Options) HTTPClientMaxBodySize() int64 {
|
||||||
return o.httpClientMaxBodySize
|
return o.httpClientMaxBodySize
|
||||||
}
|
}
|
||||||
|
|
||||||
// HTTPClientProxy returns the proxy URL for HTTP client.
|
// HTTPClientProxyURL returns the client HTTP proxy URL if configured.
|
||||||
func (o *Options) HTTPClientProxy() string {
|
func (o *Options) HTTPClientProxyURL() *url.URL {
|
||||||
return o.httpClientProxy
|
return o.httpClientProxyURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasHTTPClientProxyURLConfigured returns true if the client HTTP proxy URL if configured.
|
||||||
|
func (o *Options) HasHTTPClientProxyURLConfigured() bool {
|
||||||
|
return o.httpClientProxyURL != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPClientProxies returns the list of proxies.
|
||||||
|
func (o *Options) HTTPClientProxies() []string {
|
||||||
|
return o.httpClientProxies
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPClientProxiesString returns true if the list of rotating proxies are configured.
|
||||||
|
func (o *Options) HasHTTPClientProxiesConfigured() bool {
|
||||||
|
return len(o.httpClientProxies) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// HTTPServerTimeout returns the time limit in seconds before the HTTP server cancel the request.
|
// HTTPServerTimeout returns the time limit in seconds before the HTTP server cancel the request.
|
||||||
|
@ -597,11 +615,6 @@ func (o *Options) HTTPServerTimeout() int {
|
||||||
return o.httpServerTimeout
|
return o.httpServerTimeout
|
||||||
}
|
}
|
||||||
|
|
||||||
// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured.
|
|
||||||
func (o *Options) HasHTTPClientProxyConfigured() bool {
|
|
||||||
return o.httpClientProxy != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// AuthProxyHeader returns an HTTP header name that contains username for
|
// AuthProxyHeader returns an HTTP header name that contains username for
|
||||||
// authentication using auth proxy.
|
// authentication using auth proxy.
|
||||||
func (o *Options) AuthProxyHeader() string {
|
func (o *Options) AuthProxyHeader() string {
|
||||||
|
@ -664,6 +677,33 @@ func (o *Options) FilterEntryMaxAgeDays() int {
|
||||||
|
|
||||||
// SortedOptions returns options as a list of key value pairs, sorted by keys.
|
// SortedOptions returns options as a list of key value pairs, sorted by keys.
|
||||||
func (o *Options) SortedOptions(redactSecret bool) []*Option {
|
func (o *Options) SortedOptions(redactSecret bool) []*Option {
|
||||||
|
var clientProxyURLRedacted string
|
||||||
|
if o.httpClientProxyURL != nil {
|
||||||
|
if redactSecret {
|
||||||
|
clientProxyURLRedacted = o.httpClientProxyURL.Redacted()
|
||||||
|
} else {
|
||||||
|
clientProxyURLRedacted = o.httpClientProxyURL.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var clientProxyURLsRedacted string
|
||||||
|
if len(o.httpClientProxies) > 0 {
|
||||||
|
if redactSecret {
|
||||||
|
var proxyURLs []string
|
||||||
|
for range o.httpClientProxies {
|
||||||
|
proxyURLs = append(proxyURLs, "<redacted>")
|
||||||
|
}
|
||||||
|
clientProxyURLsRedacted = strings.Join(proxyURLs, ",")
|
||||||
|
} else {
|
||||||
|
clientProxyURLsRedacted = strings.Join(o.httpClientProxies, ",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var mediaProxyPrivateKeyValue string
|
||||||
|
if len(o.mediaProxyPrivateKey) > 0 {
|
||||||
|
mediaProxyPrivateKeyValue = "<binary-data>"
|
||||||
|
}
|
||||||
|
|
||||||
var keyValues = map[string]interface{}{
|
var keyValues = map[string]interface{}{
|
||||||
"ADMIN_PASSWORD": redactSecretValue(o.adminPassword, redactSecret),
|
"ADMIN_PASSWORD": redactSecretValue(o.adminPassword, redactSecret),
|
||||||
"ADMIN_USERNAME": o.adminUsername,
|
"ADMIN_USERNAME": o.adminUsername,
|
||||||
|
@ -694,7 +734,8 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
|
||||||
"FETCH_BILIBILI_WATCH_TIME": o.fetchBilibiliWatchTime,
|
"FETCH_BILIBILI_WATCH_TIME": o.fetchBilibiliWatchTime,
|
||||||
"HTTPS": o.HTTPS,
|
"HTTPS": o.HTTPS,
|
||||||
"HTTP_CLIENT_MAX_BODY_SIZE": o.httpClientMaxBodySize,
|
"HTTP_CLIENT_MAX_BODY_SIZE": o.httpClientMaxBodySize,
|
||||||
"HTTP_CLIENT_PROXY": o.httpClientProxy,
|
"HTTP_CLIENT_PROXIES": clientProxyURLsRedacted,
|
||||||
|
"HTTP_CLIENT_PROXY": clientProxyURLRedacted,
|
||||||
"HTTP_CLIENT_TIMEOUT": o.httpClientTimeout,
|
"HTTP_CLIENT_TIMEOUT": o.httpClientTimeout,
|
||||||
"HTTP_CLIENT_USER_AGENT": o.httpClientUserAgent,
|
"HTTP_CLIENT_USER_AGENT": o.httpClientUserAgent,
|
||||||
"HTTP_SERVER_TIMEOUT": o.httpServerTimeout,
|
"HTTP_SERVER_TIMEOUT": o.httpServerTimeout,
|
||||||
|
@ -729,7 +770,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
|
||||||
"MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout,
|
"MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout,
|
||||||
"MEDIA_PROXY_RESOURCE_TYPES": o.mediaProxyResourceTypes,
|
"MEDIA_PROXY_RESOURCE_TYPES": o.mediaProxyResourceTypes,
|
||||||
"MEDIA_PROXY_MODE": o.mediaProxyMode,
|
"MEDIA_PROXY_MODE": o.mediaProxyMode,
|
||||||
"MEDIA_PROXY_PRIVATE_KEY": redactSecretValue(string(o.mediaProxyPrivateKey), redactSecret),
|
"MEDIA_PROXY_PRIVATE_KEY": mediaProxyPrivateKeyValue,
|
||||||
"MEDIA_PROXY_CUSTOM_URL": o.mediaProxyCustomURL,
|
"MEDIA_PROXY_CUSTOM_URL": o.mediaProxyCustomURL,
|
||||||
"ROOT_URL": o.rootURL,
|
"ROOT_URL": o.rootURL,
|
||||||
"RUN_MIGRATIONS": o.runMigrations,
|
"RUN_MIGRATIONS": o.runMigrations,
|
||||||
|
|
|
@ -240,7 +240,12 @@ func (p *Parser) parseLines(lines []string) (err error) {
|
||||||
case "HTTP_CLIENT_MAX_BODY_SIZE":
|
case "HTTP_CLIENT_MAX_BODY_SIZE":
|
||||||
p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024)
|
p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024)
|
||||||
case "HTTP_CLIENT_PROXY":
|
case "HTTP_CLIENT_PROXY":
|
||||||
p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy)
|
p.opts.httpClientProxyURL, err = url.Parse(parseString(value, defaultHTTPClientProxy))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("config: invalid HTTP_CLIENT_PROXY value: %w", err)
|
||||||
|
}
|
||||||
|
case "HTTP_CLIENT_PROXIES":
|
||||||
|
p.opts.httpClientProxies = parseStringList(value, []string{})
|
||||||
case "HTTP_CLIENT_USER_AGENT":
|
case "HTTP_CLIENT_USER_AGENT":
|
||||||
p.opts.httpClientUserAgent = parseString(value, defaultHTTPClientUserAgent)
|
p.opts.httpClientUserAgent = parseString(value, defaultHTTPClientUserAgent)
|
||||||
case "HTTP_SERVER_TIMEOUT":
|
case "HTTP_SERVER_TIMEOUT":
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"miniflux.app/v2/internal/integration"
|
"miniflux.app/v2/internal/integration"
|
||||||
"miniflux.app/v2/internal/mediaproxy"
|
"miniflux.app/v2/internal/mediaproxy"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
mff "miniflux.app/v2/internal/reader/handler"
|
mff "miniflux.app/v2/internal/reader/handler"
|
||||||
mfs "miniflux.app/v2/internal/reader/subscription"
|
mfs "miniflux.app/v2/internal/reader/subscription"
|
||||||
|
@ -683,7 +684,8 @@ func (h *handler) quickAddHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
|
||||||
var rssBridgeURL string
|
var rssBridgeURL string
|
||||||
if intg, err := h.store.Integration(userID); err == nil && intg != nil && intg.RSSBridgeEnabled {
|
if intg, err := h.store.Integration(userID); err == nil && intg != nil && intg.RSSBridgeEnabled {
|
||||||
|
|
60
internal/proxyrotator/proxyrotator.go
Normal file
60
internal/proxyrotator/proxyrotator.go
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package proxyrotator // import "miniflux.app/v2/internal/proxyrotator"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/url"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ProxyRotatorInstance *ProxyRotator
|
||||||
|
|
||||||
|
// ProxyRotator manages a list of proxies and rotates through them.
|
||||||
|
type ProxyRotator struct {
|
||||||
|
proxies []*url.URL
|
||||||
|
currentIndex int
|
||||||
|
mutex sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewProxyRotator creates a new ProxyRotator with the given proxy URLs.
|
||||||
|
func NewProxyRotator(proxyURLs []string) (*ProxyRotator, error) {
|
||||||
|
parsedProxies := make([]*url.URL, 0, len(proxyURLs))
|
||||||
|
|
||||||
|
for _, p := range proxyURLs {
|
||||||
|
proxyURL, err := url.Parse(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parsedProxies = append(parsedProxies, proxyURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ProxyRotator{
|
||||||
|
proxies: parsedProxies,
|
||||||
|
currentIndex: 0,
|
||||||
|
mutex: sync.Mutex{},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetNextProxy returns the next proxy in the rotation.
|
||||||
|
func (pr *ProxyRotator) GetNextProxy() *url.URL {
|
||||||
|
pr.mutex.Lock()
|
||||||
|
defer pr.mutex.Unlock()
|
||||||
|
|
||||||
|
if len(pr.proxies) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy := pr.proxies[pr.currentIndex]
|
||||||
|
pr.currentIndex = (pr.currentIndex + 1) % len(pr.proxies)
|
||||||
|
|
||||||
|
return proxy
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasProxies checks if there are any proxies available in the rotator.
|
||||||
|
func (pr *ProxyRotator) HasProxies() bool {
|
||||||
|
pr.mutex.Lock()
|
||||||
|
defer pr.mutex.Unlock()
|
||||||
|
|
||||||
|
return len(pr.proxies) > 0
|
||||||
|
}
|
71
internal/proxyrotator/proxyrotator_test.go
Normal file
71
internal/proxyrotator/proxyrotator_test.go
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package proxyrotator // import "miniflux.app/v2/internal/proxyrotator"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProxyRotator(t *testing.T) {
|
||||||
|
proxyURLs := []string{
|
||||||
|
"http://proxy1.example.com",
|
||||||
|
"http://proxy2.example.com",
|
||||||
|
"http://proxy3.example.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
rotator, err := NewProxyRotator(proxyURLs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create ProxyRotator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !rotator.HasProxies() {
|
||||||
|
t.Fatalf("Expected rotator to have proxies")
|
||||||
|
}
|
||||||
|
|
||||||
|
seenProxies := make(map[string]bool)
|
||||||
|
for range len(proxyURLs) * 2 {
|
||||||
|
proxy := rotator.GetNextProxy()
|
||||||
|
if proxy == nil {
|
||||||
|
t.Fatalf("Expected a proxy, got nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
seenProxies[proxy.String()] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(seenProxies) != len(proxyURLs) {
|
||||||
|
t.Fatalf("Expected to see all proxies, but saw: %v", seenProxies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProxyRotatorEmpty(t *testing.T) {
|
||||||
|
rotator, err := NewProxyRotator([]string{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create ProxyRotator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rotator.HasProxies() {
|
||||||
|
t.Fatalf("Expected rotator to have no proxies")
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy := rotator.GetNextProxy()
|
||||||
|
if proxy != nil {
|
||||||
|
t.Fatalf("Expected no proxy, got: %v", proxy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProxyRotatorInvalidURL(t *testing.T) {
|
||||||
|
invalidProxyURLs := []string{
|
||||||
|
"http://validproxy.example.com",
|
||||||
|
"test|test://invalidproxy.example.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
rotator, err := NewProxyRotator(invalidProxyURLs)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Expected an error when creating ProxyRotator with invalid URLs, but got none")
|
||||||
|
}
|
||||||
|
|
||||||
|
if rotator != nil {
|
||||||
|
t.Fatalf("Expected rotator to be nil when initialization fails, but got: %v", rotator)
|
||||||
|
}
|
||||||
|
}
|
|
@ -11,6 +11,8 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -21,12 +23,13 @@ const (
|
||||||
|
|
||||||
type RequestBuilder struct {
|
type RequestBuilder struct {
|
||||||
headers http.Header
|
headers http.Header
|
||||||
clientProxyURL string
|
clientProxyURL *url.URL
|
||||||
useClientProxy bool
|
useClientProxy bool
|
||||||
clientTimeout int
|
clientTimeout int
|
||||||
withoutRedirects bool
|
withoutRedirects bool
|
||||||
ignoreTLSErrors bool
|
ignoreTLSErrors bool
|
||||||
disableHTTP2 bool
|
disableHTTP2 bool
|
||||||
|
proxyRotator *proxyrotator.ProxyRotator
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRequestBuilder() *RequestBuilder {
|
func NewRequestBuilder() *RequestBuilder {
|
||||||
|
@ -78,12 +81,17 @@ func (r *RequestBuilder) WithUsernameAndPassword(username, password string) *Req
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *RequestBuilder) WithProxy(proxyURL string) *RequestBuilder {
|
func (r *RequestBuilder) WithProxyRotator(proxyRotator *proxyrotator.ProxyRotator) *RequestBuilder {
|
||||||
|
r.proxyRotator = proxyRotator
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RequestBuilder) WithCustomApplicationProxyURL(proxyURL *url.URL) *RequestBuilder {
|
||||||
r.clientProxyURL = proxyURL
|
r.clientProxyURL = proxyURL
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *RequestBuilder) UseProxy(value bool) *RequestBuilder {
|
func (r *RequestBuilder) UseCustomApplicationProxyURL(value bool) *RequestBuilder {
|
||||||
r.useClientProxy = value
|
r.useClientProxy = value
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
@ -151,15 +159,17 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
|
||||||
transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
|
transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.useClientProxy && r.clientProxyURL != "" {
|
var clientProxyURL *url.URL
|
||||||
if proxyURL, err := url.Parse(r.clientProxyURL); err != nil {
|
if r.useClientProxy && r.clientProxyURL != nil {
|
||||||
slog.Warn("Unable to parse proxy URL",
|
clientProxyURL = r.clientProxyURL
|
||||||
slog.String("proxy_url", r.clientProxyURL),
|
} else if r.proxyRotator != nil && r.proxyRotator.HasProxies() {
|
||||||
slog.Any("error", err),
|
clientProxyURL = r.proxyRotator.GetNextProxy()
|
||||||
)
|
}
|
||||||
} else {
|
|
||||||
transport.Proxy = http.ProxyURL(proxyURL)
|
var clientProxyURLRedacted string
|
||||||
}
|
if clientProxyURL != nil {
|
||||||
|
transport.Proxy = http.ProxyURL(clientProxyURL)
|
||||||
|
clientProxyURLRedacted = clientProxyURL.Redacted()
|
||||||
}
|
}
|
||||||
|
|
||||||
client := &http.Client{
|
client := &http.Client{
|
||||||
|
@ -189,8 +199,8 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
|
||||||
slog.String("url", req.URL.String()),
|
slog.String("url", req.URL.String()),
|
||||||
slog.Any("headers", req.Header),
|
slog.Any("headers", req.Header),
|
||||||
slog.Bool("without_redirects", r.withoutRedirects),
|
slog.Bool("without_redirects", r.withoutRedirects),
|
||||||
slog.Bool("with_proxy", r.useClientProxy),
|
slog.Bool("use_app_client_proxy", r.useClientProxy),
|
||||||
slog.String("proxy_url", r.clientProxyURL),
|
slog.String("client_proxy_url", clientProxyURLRedacted),
|
||||||
slog.Bool("ignore_tls_errors", r.ignoreTLSErrors),
|
slog.Bool("ignore_tls_errors", r.ignoreTLSErrors),
|
||||||
slog.Bool("disable_http2", r.disableHTTP2),
|
slog.Bool("disable_http2", r.disableHTTP2),
|
||||||
))
|
))
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"miniflux.app/v2/internal/integration"
|
"miniflux.app/v2/internal/integration"
|
||||||
"miniflux.app/v2/internal/locale"
|
"miniflux.app/v2/internal/locale"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/reader/icon"
|
"miniflux.app/v2/internal/reader/icon"
|
||||||
"miniflux.app/v2/internal/reader/parser"
|
"miniflux.app/v2/internal/reader/parser"
|
||||||
|
@ -83,8 +84,9 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
|
||||||
requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(feedCreationRequest.Cookie)
|
requestBuilder.WithCookie(feedCreationRequest.Cookie)
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy)
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy)
|
||||||
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2)
|
requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2)
|
||||||
|
|
||||||
|
@ -109,8 +111,9 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
|
||||||
requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(feedCreationRequest.Cookie)
|
requestBuilder.WithCookie(feedCreationRequest.Cookie)
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy)
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy)
|
||||||
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2)
|
requestBuilder.DisableHTTP2(feedCreationRequest.DisableHTTP2)
|
||||||
|
|
||||||
|
@ -212,8 +215,9 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(originalFeed.Cookie)
|
requestBuilder.WithCookie(originalFeed.Cookie)
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
requestBuilder.UseProxy(originalFeed.FetchViaProxy)
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(originalFeed.FetchViaProxy)
|
||||||
requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(originalFeed.DisableHTTP2)
|
requestBuilder.DisableHTTP2(originalFeed.DisableHTTP2)
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/storage"
|
"miniflux.app/v2/internal/storage"
|
||||||
)
|
)
|
||||||
|
@ -29,8 +30,9 @@ func (c *IconChecker) fetchAndStoreIcon() {
|
||||||
requestBuilder.WithUserAgent(c.feed.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(c.feed.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(c.feed.Cookie)
|
requestBuilder.WithCookie(c.feed.Cookie)
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
requestBuilder.UseProxy(c.feed.FetchViaProxy)
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(c.feed.FetchViaProxy)
|
||||||
requestBuilder.IgnoreTLSErrors(c.feed.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(c.feed.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(c.feed.DisableHTTP2)
|
requestBuilder.DisableHTTP2(c.feed.DisableHTTP2)
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -45,7 +46,8 @@ func extractBilibiliVideoID(websiteURL string) (string, string, error) {
|
||||||
func fetchBilibiliWatchTime(websiteURL string) (int, error) {
|
func fetchBilibiliWatchTime(websiteURL string) (int, error) {
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
|
||||||
idType, videoID, extractErr := extractBilibiliVideoID(websiteURL)
|
idType, videoID, extractErr := extractBilibiliVideoID(websiteURL)
|
||||||
if extractErr != nil {
|
if extractErr != nil {
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -33,7 +34,8 @@ func shouldFetchNebulaWatchTime(entry *model.Entry) bool {
|
||||||
func fetchNebulaWatchTime(websiteURL string) (int, error) {
|
func fetchNebulaWatchTime(websiteURL string) (int, error) {
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
|
||||||
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
|
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
|
||||||
defer responseHandler.Close()
|
defer responseHandler.Close()
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -33,7 +34,8 @@ func shouldFetchOdyseeWatchTime(entry *model.Entry) bool {
|
||||||
func fetchOdyseeWatchTime(websiteURL string) (int, error) {
|
func fetchOdyseeWatchTime(websiteURL string) (int, error) {
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
|
||||||
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
|
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
|
||||||
defer responseHandler.Close()
|
defer responseHandler.Close()
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/metric"
|
"miniflux.app/v2/internal/metric"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/reader/readingtime"
|
"miniflux.app/v2/internal/reader/readingtime"
|
||||||
"miniflux.app/v2/internal/reader/rewrite"
|
"miniflux.app/v2/internal/reader/rewrite"
|
||||||
|
@ -78,8 +79,9 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
|
||||||
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(feed.Cookie)
|
requestBuilder.WithCookie(feed.Cookie)
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
requestBuilder.UseProxy(feed.FetchViaProxy)
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy)
|
||||||
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(feed.DisableHTTP2)
|
requestBuilder.DisableHTTP2(feed.DisableHTTP2)
|
||||||
|
|
||||||
|
@ -145,8 +147,9 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
|
||||||
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(feed.Cookie)
|
requestBuilder.WithCookie(feed.Cookie)
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
requestBuilder.UseProxy(feed.FetchViaProxy)
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy)
|
||||||
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(feed.DisableHTTP2)
|
requestBuilder.DisableHTTP2(feed.DisableHTTP2)
|
||||||
|
|
||||||
|
|
|
@ -18,6 +18,7 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -52,7 +53,8 @@ func fetchYouTubeWatchTimeForSingleEntry(websiteURL string) (int, error) {
|
||||||
|
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
|
||||||
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
|
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
|
||||||
defer responseHandler.Close()
|
defer responseHandler.Close()
|
||||||
|
@ -132,7 +134,8 @@ func fetchYouTubeWatchTimeFromApiInBulk(videoIDs []string) (map[string]time.Dura
|
||||||
|
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
|
||||||
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
|
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
|
||||||
defer responseHandler.Close()
|
defer responseHandler.Close()
|
||||||
|
|
|
@ -82,7 +82,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
|
||||||
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
||||||
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
||||||
view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
||||||
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
|
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured())
|
||||||
|
|
||||||
html.OK(w, r, view.Render("edit_feed"))
|
html.OK(w, r, view.Render("edit_feed"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"miniflux.app/v2/internal/http/response/html"
|
"miniflux.app/v2/internal/http/response/html"
|
||||||
"miniflux.app/v2/internal/http/route"
|
"miniflux.app/v2/internal/http/route"
|
||||||
"miniflux.app/v2/internal/locale"
|
"miniflux.app/v2/internal/locale"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/reader/opml"
|
"miniflux.app/v2/internal/reader/opml"
|
||||||
"miniflux.app/v2/internal/ui/session"
|
"miniflux.app/v2/internal/ui/session"
|
||||||
|
@ -95,7 +96,8 @@ func (h *handler) fetchOPML(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
|
||||||
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(opmlFileURL))
|
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(opmlFileURL))
|
||||||
defer responseHandler.Close()
|
defer responseHandler.Close()
|
||||||
|
|
|
@ -36,7 +36,7 @@ func (h *handler) showAddSubscriptionPage(w http.ResponseWriter, r *http.Request
|
||||||
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
||||||
view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
||||||
view.Set("form", &form.SubscriptionForm{CategoryID: 0})
|
view.Set("form", &form.SubscriptionForm{CategoryID: 0})
|
||||||
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
|
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured())
|
||||||
|
|
||||||
html.OK(w, r, view.Render("add_subscription"))
|
html.OK(w, r, view.Render("add_subscription"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,7 +53,7 @@ func (h *handler) bookmarklet(w http.ResponseWriter, r *http.Request) {
|
||||||
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
||||||
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
||||||
view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
||||||
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
|
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured())
|
||||||
|
|
||||||
html.OK(w, r, view.Render("add_subscription"))
|
html.OK(w, r, view.Render("add_subscription"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"miniflux.app/v2/internal/http/route"
|
"miniflux.app/v2/internal/http/route"
|
||||||
"miniflux.app/v2/internal/locale"
|
"miniflux.app/v2/internal/locale"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/proxyrotator"
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
feedHandler "miniflux.app/v2/internal/reader/handler"
|
feedHandler "miniflux.app/v2/internal/reader/handler"
|
||||||
"miniflux.app/v2/internal/reader/subscription"
|
"miniflux.app/v2/internal/reader/subscription"
|
||||||
|
@ -41,7 +42,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
|
||||||
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
||||||
v.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
v.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
||||||
v.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
v.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent())
|
||||||
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
|
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured())
|
||||||
|
|
||||||
subscriptionForm := form.NewSubscriptionForm(r)
|
subscriptionForm := form.NewSubscriptionForm(r)
|
||||||
if validationErr := subscriptionForm.Validate(); validationErr != nil {
|
if validationErr := subscriptionForm.Validate(); validationErr != nil {
|
||||||
|
@ -58,11 +59,12 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
requestBuilder := fetcher.NewRequestBuilder()
|
requestBuilder := fetcher.NewRequestBuilder()
|
||||||
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
|
||||||
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
|
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
|
||||||
|
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
|
||||||
|
requestBuilder.UseCustomApplicationProxyURL(subscriptionForm.FetchViaProxy)
|
||||||
requestBuilder.WithUserAgent(subscriptionForm.UserAgent, config.Opts.HTTPClientUserAgent())
|
requestBuilder.WithUserAgent(subscriptionForm.UserAgent, config.Opts.HTTPClientUserAgent())
|
||||||
requestBuilder.WithCookie(subscriptionForm.Cookie)
|
requestBuilder.WithCookie(subscriptionForm.Cookie)
|
||||||
requestBuilder.WithUsernameAndPassword(subscriptionForm.Username, subscriptionForm.Password)
|
requestBuilder.WithUsernameAndPassword(subscriptionForm.Username, subscriptionForm.Password)
|
||||||
requestBuilder.UseProxy(subscriptionForm.FetchViaProxy)
|
|
||||||
requestBuilder.IgnoreTLSErrors(subscriptionForm.AllowSelfSignedCertificates)
|
requestBuilder.IgnoreTLSErrors(subscriptionForm.AllowSelfSignedCertificates)
|
||||||
requestBuilder.DisableHTTP2(subscriptionForm.DisableHTTP2)
|
requestBuilder.DisableHTTP2(subscriptionForm.DisableHTTP2)
|
||||||
|
|
||||||
|
@ -149,7 +151,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
|
||||||
view.Set("user", user)
|
view.Set("user", user)
|
||||||
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
||||||
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
||||||
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
|
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyURLConfigured())
|
||||||
|
|
||||||
html.OK(w, r, view.Render("choose_subscription"))
|
html.OK(w, r, view.Render("choose_subscription"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -293,8 +293,13 @@ Maximum body size for HTTP requests in Mebibyte (MiB)\&.
|
||||||
.br
|
.br
|
||||||
Default is 15 MiB\&.
|
Default is 15 MiB\&.
|
||||||
.TP
|
.TP
|
||||||
|
.B HTTP_CLIENT_PROXIES
|
||||||
|
Enable proxy rotation for outgoing requests by providing a comma-separated list of proxy URLs\&.
|
||||||
|
.br
|
||||||
|
Default is empty\&.
|
||||||
|
.TP
|
||||||
.B HTTP_CLIENT_PROXY
|
.B HTTP_CLIENT_PROXY
|
||||||
Proxy URL for HTTP client\&.
|
Proxy URL to use when the "Fetch via proxy" feed option is enabled\&.
|
||||||
.br
|
.br
|
||||||
Default is empty\&.
|
Default is empty\&.
|
||||||
.TP
|
.TP
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue