diff --git a/internal/reader/fetcher/request_builder.go b/internal/reader/fetcher/request_builder.go index 06c1bf24..4a602d56 100644 --- a/internal/reader/fetcher/request_builder.go +++ b/internal/reader/fetcher/request_builder.go @@ -18,21 +18,21 @@ import ( ) const ( - defaultHTTPClientTimeout = 20 - defaultHTTPClientMaxBodySize = 15 * 1024 * 1024 - defaultAcceptHeader = "application/xml, application/atom+xml, application/rss+xml, application/rdf+xml, application/feed+json, text/html, */*;q=0.9" + defaultHTTPClientTimeout = 20 + defaultAcceptHeader = "application/xml, application/atom+xml, application/rss+xml, application/rdf+xml, application/feed+json, text/html, */*;q=0.9" ) type RequestBuilder struct { - headers http.Header - clientProxyURL *url.URL - clientTimeout int - useClientProxy bool - withoutRedirects bool - ignoreTLSErrors bool - disableHTTP2 bool - proxyRotator *proxyrotator.ProxyRotator - feedProxyURL string + headers http.Header + clientProxyURL *url.URL + clientTimeout int + useClientProxy bool + withoutRedirects bool + ignoreTLSErrors bool + disableHTTP2 bool + disableCompression bool + proxyRotator *proxyrotator.ProxyRotator + feedProxyURL string } func NewRequestBuilder() *RequestBuilder { @@ -124,6 +124,11 @@ func (r *RequestBuilder) IgnoreTLSErrors(value bool) *RequestBuilder { return r } +func (r *RequestBuilder) WithoutCompression() *RequestBuilder { + r.disableCompression = true + return r +} + func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, error) { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -197,8 +202,18 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro } req.Header = r.headers - req.Header.Set("Accept-Encoding", "br, gzip") - req.Header.Set("Accept", defaultAcceptHeader) + if r.disableCompression { + req.Header.Set("Accept-Encoding", "identity") + } else { + req.Header.Set("Accept-Encoding", "br, gzip") + } + + // Set default Accept header if not already set. + // Note that for the media proxy requests, we need to forward the browser Accept header. + if req.Header.Get("Accept") == "" { + req.Header.Set("Accept", defaultAcceptHeader) + } + req.Header.Set("Connection", "close") slog.Debug("Making outgoing request", slog.Group("request", diff --git a/internal/ui/proxy.go b/internal/ui/proxy.go index 602735e8..8e9ee787 100644 --- a/internal/ui/proxy.go +++ b/internal/ui/proxy.go @@ -88,6 +88,9 @@ func (h *handler) mediaProxy(w http.ResponseWriter, r *http.Request) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.MediaProxyHTTPClientTimeout()) + // Disable compression for the media proxy requests (not implemented). + requestBuilder.WithoutCompression() + if referer := rewrite.GetRefererForURL(mediaURL); referer != "" { requestBuilder.WithHeader("Referer", referer) }