diff --git a/internal/reader/processor/nebula.go b/internal/reader/processor/nebula.go index 0759f6fe..c47d218f 100644 --- a/internal/reader/processor/nebula.go +++ b/internal/reader/processor/nebula.go @@ -4,18 +4,9 @@ package processor // import "miniflux.app/v2/internal/reader/processor" import ( - "errors" - "fmt" - "log/slog" - "net/url" - "strconv" - - "github.com/PuerkitoBio/goquery" - "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxyrotator" - "miniflux.app/v2/internal/reader/fetcher" + "miniflux.app/v2/internal/urllib" ) func shouldFetchNebulaWatchTime(entry *model.Entry) bool { @@ -23,42 +14,9 @@ func shouldFetchNebulaWatchTime(entry *model.Entry) bool { return false } - u, err := url.Parse(entry.URL) - if err != nil { - return false - } - - return u.Hostname() == "nebula.tv" + return urllib.DomainWithoutWWW(entry.URL) == "nebula.tv" } func fetchNebulaWatchTime(websiteURL string) (int, error) { - requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) - - responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) - defer responseHandler.Close() - - if localizedError := responseHandler.LocalizedError(); localizedError != nil { - slog.Warn("Unable to fetch Nebula watch time", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error())) - return 0, localizedError.Error() - } - - doc, docErr := goquery.NewDocumentFromReader(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())) - if docErr != nil { - return 0, docErr - } - - durs, exists := doc.FindMatcher(goquery.Single(`meta[property="video:duration"]`)).Attr("content") - // durs contains video watch time in seconds - if !exists { - return 0, errors.New("duration has not found") - } - - dur, err := strconv.ParseInt(durs, 10, 64) - if err != nil { - return 0, fmt.Errorf("unable to parse duration %s: %v", durs, err) - } - - return int(dur / 60), nil + return fetchWatchTime(websiteURL, `meta[property="video:duration"]`, false) } diff --git a/internal/reader/processor/odysee.go b/internal/reader/processor/odysee.go index 132d2926..5523bb3f 100644 --- a/internal/reader/processor/odysee.go +++ b/internal/reader/processor/odysee.go @@ -4,18 +4,9 @@ package processor // import "miniflux.app/v2/internal/reader/processor" import ( - "errors" - "fmt" - "log/slog" - "net/url" - "strconv" - - "github.com/PuerkitoBio/goquery" - "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxyrotator" - "miniflux.app/v2/internal/reader/fetcher" + "miniflux.app/v2/internal/urllib" ) func shouldFetchOdyseeWatchTime(entry *model.Entry) bool { @@ -23,42 +14,9 @@ func shouldFetchOdyseeWatchTime(entry *model.Entry) bool { return false } - u, err := url.Parse(entry.URL) - if err != nil { - return false - } - - return u.Hostname() == "odysee.com" + return urllib.DomainWithoutWWW(entry.URL) == "odysee.com" } func fetchOdyseeWatchTime(websiteURL string) (int, error) { - requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) - - responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) - defer responseHandler.Close() - - if localizedError := responseHandler.LocalizedError(); localizedError != nil { - slog.Warn("Unable to fetch Odysee watch time", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error())) - return 0, localizedError.Error() - } - - doc, docErr := goquery.NewDocumentFromReader(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())) - if docErr != nil { - return 0, docErr - } - - durs, exists := doc.FindMatcher(goquery.Single(`meta[property="og:video:duration"]`)).Attr("content") - // durs contains video watch time in seconds - if !exists { - return 0, errors.New("duration has not found") - } - - dur, err := strconv.ParseInt(durs, 10, 64) - if err != nil { - return 0, fmt.Errorf("unable to parse duration %s: %v", durs, err) - } - - return int(dur / 60), nil + return fetchWatchTime(websiteURL, `meta[property="og:video:duration"]`, false) } diff --git a/internal/reader/processor/reading_time.go b/internal/reader/processor/reading_time.go index 3d0a3cda..40de4a49 100644 --- a/internal/reader/processor/reading_time.go +++ b/internal/reader/processor/reading_time.go @@ -4,13 +4,60 @@ package processor // import "miniflux.app/v2/internal/reader/processor" import ( + "errors" + "fmt" "log/slog" + "strconv" + "github.com/PuerkitoBio/goquery" + "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/proxyrotator" + "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/readingtime" "miniflux.app/v2/internal/storage" ) +func fetchWatchTime(websiteURL, query string, isoDate bool) (int, error) { + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance) + + responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) + defer responseHandler.Close() + + if localizedError := responseHandler.LocalizedError(); localizedError != nil { + slog.Warn("Unable to fetch watch time", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error())) + return 0, localizedError.Error() + } + + doc, docErr := goquery.NewDocumentFromReader(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())) + if docErr != nil { + return 0, docErr + } + + duration, exists := doc.FindMatcher(goquery.Single(query)).Attr("content") + if !exists { + return 0, errors.New("duration has not found") + } + + ret := 0 + if isoDate { + parsedDuration, err := parseISO8601(duration) + if err != nil { + return 0, fmt.Errorf("unable to parse iso duration %s: %v", duration, err) + } + ret = int(parsedDuration.Minutes()) + } else { + parsedDuration, err := strconv.ParseInt(duration, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse duration %s: %v", duration, err) + } + ret = int(parsedDuration / 60) + } + return ret, nil +} + func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *model.Entry, entryIsNew bool, user *model.User) { if !user.ShowReadingTime { slog.Debug("Skip reading time estimation for this user", slog.Int64("user_id", user.ID))