miniflux-v2/internal/reader/fetcher/response_handler.go

// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package fetcher // import "miniflux.app/v2/internal/reader/fetcher"

import (
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"miniflux.app/v2/internal/locale"
)

type ResponseHandler struct {
	httpResponse *http.Response
	clientErr    error
}

func NewResponseHandler(httpResponse *http.Response, clientErr error) *ResponseHandler {
	return &ResponseHandler{httpResponse: httpResponse, clientErr: clientErr}
}

func (r *ResponseHandler) EffectiveURL() string {
	return r.httpResponse.Request.URL.String()
}

func (r *ResponseHandler) ContentType() string {
	return r.httpResponse.Header.Get("Content-Type")
}

func (r *ResponseHandler) LastModified() string {
	// Ignore caching headers for feeds that do not want any cache.
	if r.httpResponse.Header.Get("Expires") == "0" {
		return ""
	}
	return r.httpResponse.Header.Get("Last-Modified")
}

func (r *ResponseHandler) ETag() string {
	// Ignore caching headers for feeds that do not want any cache.
	if r.httpResponse.Header.Get("Expires") == "0" {
		return ""
	}
	return r.httpResponse.Header.Get("ETag")
}

func (r *ResponseHandler) Expires() time.Duration {
	expiresHeaderValue := r.httpResponse.Header.Get("Expires")
	if expiresHeaderValue != "" {
		t, err := time.Parse(time.RFC1123, expiresHeaderValue)
		if err == nil {
			// This rounds up to the next minute by rounding down and just adding a minute.
			return time.Until(t).Truncate(time.Minute) + time.Minute
		}
	}
	return 0
}

func (r *ResponseHandler) CacheControlMaxAge() time.Duration {
	cacheControlHeaderValue := r.httpResponse.Header.Get("Cache-Control")
	if cacheControlHeaderValue != "" {
		for directive := range strings.SplitSeq(cacheControlHeaderValue, ",") {
			if after, ok := strings.CutPrefix(strings.TrimSpace(directive), "max-age="); ok {
				if maxAge, err := strconv.Atoi(after); err == nil {
					return time.Duration(maxAge) * time.Second
				}
			}
		}
	}
	return 0
}

func (r *ResponseHandler) ParseRetryDelay() time.Duration {
	retryAfterHeaderValue := r.httpResponse.Header.Get("Retry-After")
	if retryAfterHeaderValue != "" {
		// First, try to parse as an integer (number of seconds)
		if seconds, err := strconv.Atoi(retryAfterHeaderValue); err == nil {
			return time.Duration(seconds) * time.Second
		}

		// If not an integer, try to parse as an HTTP-date
		if t, err := time.Parse(time.RFC1123, retryAfterHeaderValue); err == nil {
			return time.Until(t).Truncate(time.Second)
		}
	}
	return 0
}

func (r *ResponseHandler) IsRateLimited() bool {
	return r.httpResponse != nil && r.httpResponse.StatusCode == http.StatusTooManyRequests
}

func (r *ResponseHandler) IsModified(lastEtagValue, lastModifiedValue string) bool {
	if r.httpResponse.StatusCode == http.StatusNotModified {
		return false
	}

	if r.ETag() != "" {
		return r.ETag() != lastEtagValue
	}

	if r.LastModified() != "" {
		return r.LastModified() != lastModifiedValue
	}

	return true
}

func (r *ResponseHandler) IsRedirect() bool {
	return r.httpResponse != nil && r.httpResponse.StatusCode >= 300 && r.httpResponse.StatusCode < 400
}

func (r *ResponseHandler) Close() {
	if r.httpResponse != nil && r.httpResponse.Body != nil && r.clientErr == nil {
		r.httpResponse.Body.Close()
	}
}

func (r *ResponseHandler) getReader(maxBodySize int64) io.ReadCloser {
	contentEncoding := strings.ToLower(r.httpResponse.Header.Get("Content-Encoding"))
	slog.Debug("Request response",
		slog.String("effective_url", r.EffectiveURL()),
		slog.String("content_length", r.httpResponse.Header.Get("Content-Length")),
		slog.String("content_encoding", contentEncoding),
		slog.String("content_type", r.httpResponse.Header.Get("Content-Type")),
	)

	reader := r.httpResponse.Body
	switch contentEncoding {
	case "br":
		reader = NewBrotliReadCloser(reader)
	case "gzip":
		reader = NewGzipReadCloser(reader)
	}
	return http.MaxBytesReader(nil, reader, maxBodySize)
}

func (r *ResponseHandler) Body(maxBodySize int64) io.ReadCloser {
	return r.getReader(maxBodySize)
}

func (r *ResponseHandler) ReadBody(maxBodySize int64) ([]byte, *locale.LocalizedErrorWrapper) {
	limitedReader := r.getReader(maxBodySize)

	buffer, err := io.ReadAll(limitedReader)
	if err != nil && err != io.EOF {
		if err, ok := err.(*http.MaxBytesError); ok {
			return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: response body too large: %d bytes", err.Limit), "error.http_response_too_large")
		}

		return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: unable to read response body: %w", err), "error.http_body_read", err)
	}

	if len(buffer) == 0 {
		return nil, locale.NewLocalizedErrorWrapper(errors.New("fetcher: empty response body"), "error.http_empty_response_body")
	}

	return buffer, nil
}

func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper {
	if r.clientErr != nil {
		err := fmt.Errorf("fetcher: %w", r.clientErr)
		switch {
		case isSSLError(r.clientErr):
			return locale.NewLocalizedErrorWrapper(err, "error.tls_error", r.clientErr)
		case isNetworkError(r.clientErr):
			return locale.NewLocalizedErrorWrapper(err, "error.network_operation", r.clientErr)
		case os.IsTimeout(r.clientErr):
			return locale.NewLocalizedErrorWrapper(err, "error.network_timeout", r.clientErr)
		case errors.Is(r.clientErr, io.EOF):
			return locale.NewLocalizedErrorWrapper(err, "error.http_empty_response")
		default:
			return locale.NewLocalizedErrorWrapper(err, "error.http_client_error", r.clientErr)
		}
	}

	switch r.httpResponse.StatusCode {
	case http.StatusUnauthorized:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: access unauthorized (401 status code)"), "error.http_not_authorized")
	case http.StatusForbidden:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: access forbidden (403 status code)"), "error.http_forbidden")
	case http.StatusTooManyRequests:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: too many requests (429 status code)"), "error.http_too_many_requests")
	case http.StatusNotFound:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: resource not found (404 status code)"), "error.http_resource_not_found")
	case http.StatusGone:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: resource not found (410 status code)"), "error.http_resource_not_found")
	case http.StatusInternalServerError:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: remote server error (500 status code)"), "error.http_internal_server_error")
	case http.StatusBadGateway:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: bad gateway (502 status code)"), "error.http_bad_gateway")
	case http.StatusServiceUnavailable:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: service unavailable (503 status code)"), "error.http_service_unavailable")
	case http.StatusGatewayTimeout:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: gateway timeout (504 status code)"), "error.http_gateway_timeout")
	}

	if r.httpResponse.StatusCode >= 400 {
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: unexpected status code (%d status code)", r.httpResponse.StatusCode), "error.http_unexpected_status_code", r.httpResponse.StatusCode)
	}

	if r.httpResponse.StatusCode != 304 {
		// Content-Length = -1 when no Content-Length header is sent.
		if r.httpResponse.ContentLength == 0 {
			return locale.NewLocalizedErrorWrapper(errors.New("fetcher: empty response body"), "error.http_empty_response_body")
		}
	}

	return nil
}

func isNetworkError(err error) bool {
	if _, ok := err.(*url.Error); ok {
		return true
	}
	if err == io.EOF {
		return true
	}
	var opErr *net.OpError
	if ok := errors.As(err, &opErr); ok {
		return true
	}
	return false
}

func isSSLError(err error) bool {
	var certErr x509.UnknownAuthorityError
	if errors.As(err, &certErr) {
		return true
	}

	var hostErr x509.HostnameError
	if errors.As(err, &hostErr) {
		return true
	}

	var algErr x509.InsecureAlgorithmError
	return errors.As(err, &algErr)
}