oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-08-31 18:31:01 +00:00
Code Wiki Activity

feat: add proxy rotation functionality

Browse source
This commit is contained in:
Frédéric Guillot 2025-04-06 14:45:56 -07:00
parent d20e8a4e2c
commit c24209dde4
22 changed files with 351 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

68
internal/config/config_test.go
View file

@ -2186,3 +2186,71 @@ func TestParseConfigDumpOutput(t *testing.T) {
t.Fatal(err)
}
}
func TestHTTPClientProxies(t *testing.T) {
os.Clearenv()
os.Setenv("HTTP_CLIENT_PROXIES", "http://proxy1.example.com,http://proxy2.example.com")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{"http://proxy1.example.com", "http://proxy2.example.com"}
result := opts.HTTPClientProxies()
if len(expected) != len(result) {
t.Fatalf(`Unexpected HTTP_CLIENT_PROXIES value, got %v instead of %v`, result, expected)
}
for i, proxy := range expected {
if result[i] != proxy {
t.Fatalf(`Unexpected HTTP_CLIENT_PROXIES value at index %d, got %q instead of %q`, i, result[i], proxy)
}
}
}
func TestDefaultHTTPClientProxiesValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{}
result := opts.HTTPClientProxies()
if len(expected) != len(result) {
t.Fatalf(`Unexpected default HTTP_CLIENT_PROXIES value, got %v instead of %v`, result, expected)
}
}
func TestHTTPClientProxy(t *testing.T) {
os.Clearenv()
os.Setenv("HTTP_CLIENT_PROXY", "http://proxy.example.com")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := "http://proxy.example.com"
if opts.HTTPClientProxyURL() == nil || opts.HTTPClientProxyURL().String() != expected {
t.Fatalf(`Unexpected HTTP_CLIENT_PROXY value, got %v instead of %v`, opts.HTTPClientProxyURL(), expected)
}
}
func TestInvalidHTTPClientProxy(t *testing.T) {
os.Clearenv()
os.Setenv("HTTP_CLIENT_PROXY", "sche|me://invalid-proxy-url")
parser := NewParser()
_, err := parser.ParseEnvironmentVariables()
if err == nil {
t.Fatalf(`Expected error for invalid HTTP_CLIENT_PROXY value, but got none`)
}
}

65
internal/config/options.go
View file

@ -5,6 +5,7 @@ package config // import "miniflux.app/v2/internal/config"
import (
"fmt"
"net/url"
"sort"
"strings"
"time"
@ -163,7 +164,8 @@ type Options struct {
pocketConsumerKey string
httpClientTimeout int
httpClientMaxBodySize int64
httpClientProxy string
httpClientProxyURL *url.URL
httpClientProxies []string
httpClientUserAgent string
httpServerTimeout int
authProxyHeader string
@ -243,7 +245,8 @@ func NewOptions() *Options {
pocketConsumerKey: defaultPocketConsumerKey,
httpClientTimeout: defaultHTTPClientTimeout,
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
httpClientProxy: defaultHTTPClientProxy,
httpClientProxyURL: nil,
httpClientProxies: []string{},
httpClientUserAgent: defaultHTTPClientUserAgent,
httpServerTimeout: defaultHTTPServerTimeout,
authProxyHeader: defaultAuthProxyHeader,
@ -587,9 +590,24 @@ func (o *Options) HTTPClientMaxBodySize() int64 {
return o.httpClientMaxBodySize
}
// HTTPClientProxy returns the proxy URL for HTTP client.
func (o *Options) HTTPClientProxy() string {
return o.httpClientProxy
// HTTPClientProxyURL returns the client HTTP proxy URL if configured.
func (o *Options) HTTPClientProxyURL() *url.URL {
return o.httpClientProxyURL
}
// HasHTTPClientProxyURLConfigured returns true if the client HTTP proxy URL if configured.
func (o *Options) HasHTTPClientProxyURLConfigured() bool {
return o.httpClientProxyURL != nil
}
// HTTPClientProxies returns the list of proxies.
func (o *Options) HTTPClientProxies() []string {
return o.httpClientProxies
}
// HTTPClientProxiesString returns true if the list of rotating proxies are configured.
func (o *Options) HasHTTPClientProxiesConfigured() bool {
return len(o.httpClientProxies) > 0
}
// HTTPServerTimeout returns the time limit in seconds before the HTTP server cancel the request.
@ -597,11 +615,6 @@ func (o *Options) HTTPServerTimeout() int {
return o.httpServerTimeout
}
// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured.
func (o *Options) HasHTTPClientProxyConfigured() bool {
return o.httpClientProxy != ""
}
// AuthProxyHeader returns an HTTP header name that contains username for
// authentication using auth proxy.
func (o *Options) AuthProxyHeader() string {
@ -664,6 +677,33 @@ func (o *Options) FilterEntryMaxAgeDays() int {
// SortedOptions returns options as a list of key value pairs, sorted by keys.
func (o *Options) SortedOptions(redactSecret bool) []*Option {
var clientProxyURLRedacted string
if o.httpClientProxyURL != nil {
if redactSecret {
clientProxyURLRedacted = o.httpClientProxyURL.Redacted()
} else {
clientProxyURLRedacted = o.httpClientProxyURL.String()
}
}
var clientProxyURLsRedacted string
if len(o.httpClientProxies) > 0 {
if redactSecret {
var proxyURLs []string
for range o.httpClientProxies {
proxyURLs = append(proxyURLs, "<redacted>")
}
clientProxyURLsRedacted = strings.Join(proxyURLs, ",")
} else {
clientProxyURLsRedacted = strings.Join(o.httpClientProxies, ",")
}
}
var mediaProxyPrivateKeyValue string
if len(o.mediaProxyPrivateKey) > 0 {
mediaProxyPrivateKeyValue = "<binary-data>"
}
var keyValues = map[string]interface{}{
"ADMIN_PASSWORD": redactSecretValue(o.adminPassword, redactSecret),
"ADMIN_USERNAME": o.adminUsername,
@ -694,7 +734,8 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"FETCH_BILIBILI_WATCH_TIME": o.fetchBilibiliWatchTime,
"HTTPS": o.HTTPS,
"HTTP_CLIENT_MAX_BODY_SIZE": o.httpClientMaxBodySize,
"HTTP_CLIENT_PROXY": o.httpClientProxy,
"HTTP_CLIENT_PROXIES": clientProxyURLsRedacted,
"HTTP_CLIENT_PROXY": clientProxyURLRedacted,
"HTTP_CLIENT_TIMEOUT": o.httpClientTimeout,
"HTTP_CLIENT_USER_AGENT": o.httpClientUserAgent,
"HTTP_SERVER_TIMEOUT": o.httpServerTimeout,
@ -729,7 +770,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout,
"MEDIA_PROXY_RESOURCE_TYPES": o.mediaProxyResourceTypes,
"MEDIA_PROXY_MODE": o.mediaProxyMode,
"MEDIA_PROXY_PRIVATE_KEY": redactSecretValue(string(o.mediaProxyPrivateKey), redactSecret),
"MEDIA_PROXY_PRIVATE_KEY": mediaProxyPrivateKeyValue,
"MEDIA_PROXY_CUSTOM_URL": o.mediaProxyCustomURL,
"ROOT_URL": o.rootURL,
"RUN_MIGRATIONS": o.runMigrations,

7
internal/config/parser.go
View file

@ -240,7 +240,12 @@ func (p *Parser) parseLines(lines []string) (err error) {
case "HTTP_CLIENT_MAX_BODY_SIZE":
p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024)
case "HTTP_CLIENT_PROXY":
p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy)
p.opts.httpClientProxyURL, err = url.Parse(parseString(value, defaultHTTPClientProxy))
if err != nil {
return fmt.Errorf("config: invalid HTTP_CLIENT_PROXY value: %w", err)
}
case "HTTP_CLIENT_PROXIES":
p.opts.httpClientProxies = parseStringList(value, []string{})
case "HTTP_CLIENT_USER_AGENT":
p.opts.httpClientUserAgent = parseString(value, defaultHTTPClientUserAgent)
case "HTTP_SERVER_TIMEOUT":