diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go index 000ebd12..f93e0b58 100644 --- a/internal/reader/processor/processor.go +++ b/internal/reader/processor/processor.go @@ -50,6 +50,17 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64, slog.Int64("feed_id", feed.ID), ) + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) + requestBuilder.WithCookie(feed.Cookie) + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + requestBuilder.WithCustomFeedProxyURL(feed.ProxyURL) + requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) + requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates) + requestBuilder.DisableHTTP2(feed.DisableHTTP2) + // Processing older entries first ensures that their creation timestamp is lower than newer entries. for _, entry := range slices.Backward(feed.Entries) { slog.Debug("Processing entry", @@ -95,17 +106,6 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64, startTime := time.Now() - requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) - requestBuilder.WithCookie(feed.Cookie) - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) - requestBuilder.WithCustomFeedProxyURL(feed.ProxyURL) - requestBuilder.WithCustomApplicationProxyURL(config.Opts.HTTPClientProxyURL()) - requestBuilder.UseCustomApplicationProxyURL(feed.FetchViaProxy) - requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates) - requestBuilder.DisableHTTP2(feed.DisableHTTP2) - scrapedPageBaseURL, extractedContent, scraperErr := scraper.ScrapeWebsite( requestBuilder, entry.URL,