oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-07-22 17:18:37 +00:00
Code Wiki Activity

feat: implement proxy URL per feed

Browse source
This commit is contained in:
Frédéric Guillot 2025-04-06 18:04:50 -07:00
parent c45b51d1f8
commit d17f8a7bfa
48 changed files with 333 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

19
internal/reader/fetcher/request_builder.go
View file

@ -6,6 +6,7 @@ package fetcher // import "miniflux.app/v2/internal/reader/fetcher"
import (
"crypto/tls"
"encoding/base64"
"fmt"
"log/slog"
"net"
"net/http"
@ -30,6 +31,7 @@ type RequestBuilder struct {
ignoreTLSErrors bool
disableHTTP2 bool
proxyRotator *proxyrotator.ProxyRotator
feedProxyURL string
}
func NewRequestBuilder() *RequestBuilder {
@ -96,6 +98,11 @@ func (r *RequestBuilder) UseCustomApplicationProxyURL(value bool) *RequestBuilde
return r
}
func (r *RequestBuilder) WithCustomFeedProxyURL(proxyURL string) *RequestBuilder {
r.feedProxyURL = proxyURL
return r
}
func (r *RequestBuilder) WithTimeout(timeout int) *RequestBuilder {
r.clientTimeout = timeout
return r
@ -160,9 +167,17 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
}
var clientProxyURL *url.URL
if r.useClientProxy && r.clientProxyURL != nil {
switch {
case r.feedProxyURL != "":
var err error
clientProxyURL, err = url.Parse(r.feedProxyURL)
if err != nil {
return nil, fmt.Errorf(`fetcher: invalid feed proxy URL %q: %w`, r.feedProxyURL, err)
}
case r.useClientProxy && r.clientProxyURL != nil:
clientProxyURL = r.clientProxyURL
} else if r.proxyRotator != nil && r.proxyRotator.HasProxies() {
case r.proxyRotator != nil && r.proxyRotator.HasProxies():
clientProxyURL = r.proxyRotator.GetNextProxy()
}

7
internal/reader/handler/handler.go
View file

@ -30,6 +30,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
slog.Debug("Begin feed creation process from subscription discovery",
slog.Int64("user_id", userID),
slog.String("feed_url", feedCreationRequest.FeedURL),
slog.String("proxy_url", feedCreationRequest.ProxyURL),
)
if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) {
@ -65,6 +66,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
subscription.FeedURL = feedCreationRequest.FeedURL
subscription.DisableHTTP2 = feedCreationRequest.DisableHTTP2
subscription.WithCategoryID(feedCreationRequest.CategoryID)
subscription.ProxyURL = feedCreationRequest.ProxyURL
subscription.CheckedNow()
processor.ProcessFeedEntries(store, subscription, userID, true)
@ -85,6 +87,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
requestBuilder.WithCookie(feedCreationRequest.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomFeedProxyURL(feedCreationRequest.ProxyURL)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
@ -100,6 +103,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
slog.Debug("Begin feed creation process",
slog.Int64("user_id", userID),
slog.String("feed_url", feedCreationRequest.FeedURL),
slog.String("proxy_url", feedCreationRequest.ProxyURL),
)
if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) {
@ -112,6 +116,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
requestBuilder.WithCookie(feedCreationRequest.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomFeedProxyURL(feedCreationRequest.ProxyURL)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feedCreationRequest.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates)
@ -160,6 +165,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
subscription.EtagHeader = responseHandler.ETag()
subscription.LastModifiedHeader = responseHandler.LastModified()
subscription.FeedURL = responseHandler.EffectiveURL()
subscription.ProxyURL = feedCreationRequest.ProxyURL
subscription.WithCategoryID(feedCreationRequest.CategoryID)
subscription.CheckedNow()
@ -216,6 +222,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
requestBuilder.WithCookie(originalFeed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomFeedProxyURL(originalFeed.ProxyURL)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(originalFeed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates)

1
internal/reader/icon/checker.go
View file

@ -31,6 +31,7 @@ func (c *IconChecker) fetchAndStoreIcon() {
requestBuilder.WithCookie(c.feed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomFeedProxyURL(c.feed.ProxyURL)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(c.feed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(c.feed.AllowSelfSignedCertificates)

1
internal/reader/processor/bilibili.go
View file

@ -47,7 +47,6 @@ func fetchBilibiliWatchTime(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
idType, videoID, extractErr := extractBilibiliVideoID(websiteURL)
if extractErr != nil {

1
internal/reader/processor/nebula.go
View file

@ -35,7 +35,6 @@ func fetchNebulaWatchTime(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
defer responseHandler.Close()

1
internal/reader/processor/odysee.go
View file

@ -35,7 +35,6 @@ func fetchOdyseeWatchTime(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
defer responseHandler.Close()

2
internal/reader/processor/processor.go
View file

@ -80,6 +80,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
requestBuilder.WithCookie(feed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomFeedProxyURL(feed.ProxyURL)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
@ -148,6 +149,7 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
requestBuilder.WithCookie(feed.Cookie)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomFeedProxyURL(feed.ProxyURL)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)

2
internal/reader/processor/youtube.go
View file

@ -54,7 +54,6 @@ func fetchYouTubeWatchTimeForSingleEntry(websiteURL string) (int, error) {
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL))
defer responseHandler.Close()
@ -135,7 +134,6 @@ func fetchYouTubeWatchTimeFromApiInBulk(videoIDs []string) (map[string]time.Dura
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL())
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
defer responseHandler.Close()