mirror of
https://github.com/miniflux/v2.git
synced 2025-08-31 18:31:01 +00:00
Refactor HTTP Client and LocalizedError packages
This commit is contained in:
Frédéric Guillot
parent
120aabfbce
commit
14e25ab9fe
104 changed files with 1277 additions and 10672 deletions
168
internal/reader/fetcher/request_builder.go
Normal file
168
internal/reader/fetcher/request_builder.go
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package fetcher // import "miniflux.app/v2/internal/reader/fetcher"
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHTTPClientTimeout = 20
|
||||
defaultHTTPClientMaxBodySize = 15 * 1024 * 1024
|
||||
)
|
||||
|
||||
type RequestBuilder struct {
|
||||
headers http.Header
|
||||
clientProxyURL string
|
||||
useClientProxy bool
|
||||
clientTimeout int
|
||||
withoutRedirects bool
|
||||
ignoreTLSErrors bool
|
||||
}
|
||||
|
||||
func NewRequestBuilder() *RequestBuilder {
|
||||
return &RequestBuilder{
|
||||
headers: make(http.Header),
|
||||
clientTimeout: defaultHTTPClientTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithHeader(key, value string) *RequestBuilder {
|
||||
r.headers.Set(key, value)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithETag(etag string) *RequestBuilder {
|
||||
if etag != "" {
|
||||
r.headers.Set("If-None-Match", etag)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithLastModified(lastModified string) *RequestBuilder {
|
||||
if lastModified != "" {
|
||||
r.headers.Set("If-Modified-Since", lastModified)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithUserAgent(userAgent string) *RequestBuilder {
|
||||
if userAgent != "" {
|
||||
r.headers.Set("User-Agent", userAgent)
|
||||
} else {
|
||||
r.headers.Del("User-Agent")
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithCookie(cookie string) *RequestBuilder {
|
||||
if cookie != "" {
|
||||
r.headers.Set("Cookie", cookie)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithUsernameAndPassword(username, password string) *RequestBuilder {
|
||||
if username != "" && password != "" {
|
||||
r.headers.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(username+":"+password)))
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithProxy(proxyURL string) *RequestBuilder {
|
||||
r.clientProxyURL = proxyURL
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) UseProxy(value bool) *RequestBuilder {
|
||||
r.useClientProxy = value
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithTimeout(timeout int) *RequestBuilder {
|
||||
r.clientTimeout = timeout
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) WithoutRedirects() *RequestBuilder {
|
||||
r.withoutRedirects = true
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) IgnoreTLSErrors(value bool) *RequestBuilder {
|
||||
r.ignoreTLSErrors = value
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, error) {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
// Default is 30s.
|
||||
Timeout: 10 * time.Second,
|
||||
|
||||
// Default is 30s.
|
||||
KeepAlive: 15 * time.Second,
|
||||
}).DialContext,
|
||||
|
||||
// Default is 100.
|
||||
MaxIdleConns: 50,
|
||||
|
||||
// Default is 90s.
|
||||
IdleConnTimeout: 10 * time.Second,
|
||||
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: r.ignoreTLSErrors,
|
||||
},
|
||||
}
|
||||
|
||||
if r.useClientProxy && r.clientProxyURL != "" {
|
||||
if proxyURL, err := url.Parse(r.clientProxyURL); err != nil {
|
||||
slog.Warn("Unable to parse proxy URL",
|
||||
slog.String("proxy_url", r.clientProxyURL),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
} else {
|
||||
transport.Proxy = http.ProxyURL(proxyURL)
|
||||
}
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: time.Duration(r.clientTimeout) * time.Second,
|
||||
}
|
||||
|
||||
if r.withoutRedirects {
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
}
|
||||
|
||||
client.Transport = transport
|
||||
|
||||
req, err := http.NewRequest("GET", requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header = r.headers
|
||||
req.Header.Set("Accept", "*/*")
|
||||
req.Header.Set("Connection", "close")
|
||||
|
||||
slog.Debug("Making outgoing request", slog.Group("request",
|
||||
slog.String("method", req.Method),
|
||||
slog.String("url", req.URL.String()),
|
||||
slog.Any("headers", req.Header),
|
||||
slog.Bool("without_redirects", r.withoutRedirects),
|
||||
slog.Bool("with_proxy", r.useClientProxy),
|
||||
slog.String("proxy_url", r.clientProxyURL),
|
||||
))
|
||||
|
||||
return client.Do(req)
|
||||
}
|
||||
147
internal/reader/fetcher/response_handler.go
Normal file
147
internal/reader/fetcher/response_handler.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package fetcher // import "miniflux.app/v2/internal/reader/fetcher"
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"miniflux.app/v2/internal/locale"
|
||||
)
|
||||
|
||||
type ResponseHandler struct {
|
||||
httpResponse *http.Response
|
||||
clientErr error
|
||||
}
|
||||
|
||||
func NewResponseHandler(httpResponse *http.Response, clientErr error) *ResponseHandler {
|
||||
return &ResponseHandler{httpResponse: httpResponse, clientErr: clientErr}
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) EffectiveURL() string {
|
||||
return r.httpResponse.Request.URL.String()
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) ContentType() string {
|
||||
return r.httpResponse.Header.Get("Content-Type")
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) LastModified() string {
|
||||
// Ignore caching headers for feeds that do not want any cache.
|
||||
if r.httpResponse.Header.Get("Expires") == "0" {
|
||||
return ""
|
||||
}
|
||||
return r.httpResponse.Header.Get("Last-Modified")
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) ETag() string {
|
||||
// Ignore caching headers for feeds that do not want any cache.
|
||||
if r.httpResponse.Header.Get("Expires") == "0" {
|
||||
return ""
|
||||
}
|
||||
return r.httpResponse.Header.Get("ETag")
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) IsModified(lastEtagValue, lastModifiedValue string) bool {
|
||||
if r.httpResponse.StatusCode == http.StatusNotModified {
|
||||
return false
|
||||
}
|
||||
|
||||
if r.ETag() != "" && r.ETag() == lastEtagValue {
|
||||
return false
|
||||
}
|
||||
|
||||
if r.LastModified() != "" && r.LastModified() == lastModifiedValue {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) Close() {
|
||||
if r.httpResponse != nil && r.httpResponse.Body != nil && r.clientErr == nil {
|
||||
r.httpResponse.Body.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) Body(maxBodySize int64) io.ReadCloser {
|
||||
return http.MaxBytesReader(nil, r.httpResponse.Body, maxBodySize)
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) ReadBody(maxBodySize int64) ([]byte, *locale.LocalizedErrorWrapper) {
|
||||
limitedReader := http.MaxBytesReader(nil, r.httpResponse.Body, maxBodySize)
|
||||
|
||||
buffer, err := io.ReadAll(limitedReader)
|
||||
if err != nil && err != io.EOF {
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: response body too large: %w", err), "error.http_response_too_large")
|
||||
}
|
||||
|
||||
return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: unable to read response body: %w", err), "error.http_body_read", err)
|
||||
}
|
||||
|
||||
if len(buffer) == 0 {
|
||||
return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: empty response body"), "error.http_empty_response_body")
|
||||
}
|
||||
|
||||
return buffer, nil
|
||||
}
|
||||
|
||||
func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper {
|
||||
if r.clientErr != nil {
|
||||
switch r.clientErr.(type) {
|
||||
case x509.CertificateInvalidError, x509.UnknownAuthorityError, x509.HostnameError:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.tls_error", r.clientErr.Error())
|
||||
case *net.OpError:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_operation", r.clientErr.Error())
|
||||
case net.Error:
|
||||
networkErr := r.clientErr.(net.Error)
|
||||
if networkErr.Timeout() {
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if errors.Is(r.clientErr, io.EOF) {
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_empty_response")
|
||||
}
|
||||
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr.Error())
|
||||
}
|
||||
|
||||
switch r.httpResponse.StatusCode {
|
||||
case http.StatusUnauthorized:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: access unauthorized (401 status code)"), "error.http_not_authorized")
|
||||
case http.StatusForbidden:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: access forbidden (403 status code)"), "error.http_forbidden")
|
||||
case http.StatusTooManyRequests:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: too many requests (429 status code)"), "error.http_too_many_requests")
|
||||
case http.StatusNotFound, http.StatusGone:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: resource not found (%d status code)", r.httpResponse.StatusCode), "error.http_resource_not_found")
|
||||
case http.StatusInternalServerError:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: remote server error (%d status code)", r.httpResponse.StatusCode), "error.http_internal_server_error")
|
||||
case http.StatusBadGateway:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: bad gateway (%d status code)", r.httpResponse.StatusCode), "error.http_bad_gateway")
|
||||
case http.StatusServiceUnavailable:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: service unavailable (%d status code)", r.httpResponse.StatusCode), "error.http_service_unavailable")
|
||||
case http.StatusGatewayTimeout:
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: gateway timeout (%d status code)", r.httpResponse.StatusCode), "error.http_gateway_timeout")
|
||||
}
|
||||
|
||||
if r.httpResponse.StatusCode >= 400 {
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: unexpected status code (%d status code)", r.httpResponse.StatusCode), "error.http_unexpected_status_code", r.httpResponse.StatusCode)
|
||||
}
|
||||
|
||||
if r.httpResponse.StatusCode != 304 {
|
||||
// Content-Length = -1 when no Content-Length header is sent.
|
||||
if r.httpResponse.ContentLength == 0 {
|
||||
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: empty response body"), "error.http_empty_response_body")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue