feat(processor): fetch YouTube watch time in bulk using the API

internal/reader/processor/youtube.go

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-package processor
+package processor // import "miniflux.app/v2/internal/reader/processor
 
 import (
 	"encoding/json"
@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
@@ -25,24 +26,30 @@ var (
 	iso8601Regex = regexp.MustCompile(`^P((?P<year>\d+)Y)?((?P<month>\d+)M)?((?P<week>\d+)W)?((?P<day>\d+)D)?(T((?P<hour>\d+)H)?((?P<minute>\d+)M)?((?P<second>\d+)S)?)?$`)
 )
 
-func shouldFetchYouTubeWatchTime(entry *model.Entry) bool {
-	if !config.Opts.FetchYouTubeWatchTime() {
-		return false
-	}
-	matches := youtubeRegex.FindStringSubmatch(entry.URL)
-	urlMatchesYouTubePattern := len(matches) == 2
-	return urlMatchesYouTubePattern
+func isYouTubeVideoURL(websiteURL string) bool {
+	return len(youtubeRegex.FindStringSubmatch(websiteURL)) == 2
 }
 
-func fetchYouTubeWatchTime(websiteURL string) (int, error) {
-	if config.Opts.YouTubeApiKey() == "" {
-		return fetchYouTubeWatchTimeFromWebsite(websiteURL)
-	} else {
-		return fetchYouTubeWatchTimeFromApi(websiteURL)
+func getVideoIDFromYouTubeURL(websiteURL string) string {
+	parsedWebsiteURL, err := url.Parse(websiteURL)
+	if err != nil {
+		return ""
 	}
+
+	return parsedWebsiteURL.Query().Get("v")
 }
 
-func fetchYouTubeWatchTimeFromWebsite(websiteURL string) (int, error) {
+func shouldFetchYouTubeWatchTimeForSingleEntry(entry *model.Entry) bool {
+	return config.Opts.FetchYouTubeWatchTime() && config.Opts.YouTubeApiKey() == "" && isYouTubeVideoURL(entry.URL)
+}
+
+func shouldFetchYouTubeWatchTimeInBulk() bool {
+	return config.Opts.FetchYouTubeWatchTime() && config.Opts.YouTubeApiKey() != ""
+}
+
+func fetchYouTubeWatchTimeForSingleEntry(websiteURL string) (int, error) {
+	slog.Debug("Fetching YouTube watch time for a single entry", slog.String("website_url", websiteURL))
+
 	requestBuilder := fetcher.NewRequestBuilder()
 	requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
 	requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
@@ -60,31 +67,59 @@ func fetchYouTubeWatchTimeFromWebsite(websiteURL string) (int, error) {
 		return 0, docErr
 	}
 
-	durs, exists := doc.FindMatcher(goquery.Single(`meta[itemprop="duration"]`)).Attr("content")
+	htmlDuration, exists := doc.FindMatcher(goquery.Single(`meta[itemprop="duration"]`)).Attr("content")
 	if !exists {
-		return 0, errors.New("duration has not found")
+		return 0, errors.New("youtube: duration has not found")
 	}
 
-	dur, err := parseISO8601(durs)
+	parsedDuration, err := parseISO8601(htmlDuration)
 	if err != nil {
-		return 0, fmt.Errorf("unable to parse duration %s: %v", durs, err)
+		return 0, fmt.Errorf("youtube: unable to parse duration %s: %v", htmlDuration, err)
 	}
 
-	return int(dur.Minutes()), nil
+	return int(parsedDuration.Minutes()), nil
 }
 
-func fetchYouTubeWatchTimeFromApi(websiteURL string) (int, error) {
-	requestBuilder := fetcher.NewRequestBuilder()
-	requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
-	requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
+func fetchYouTubeWatchTimeInBulk(entries []*model.Entry) {
+	var videosEntriesMapping = make(map[string]*model.Entry)
+	var videoIDs []string
 
-	parsedWebsiteURL, err := url.Parse(websiteURL)
-	if err != nil {
-		return 0, fmt.Errorf("unable to parse URL: %v", err)
+	for _, entry := range entries {
+		if !isYouTubeVideoURL(entry.URL) {
+			continue
+		}
+
+		youtubeVideoID := getVideoIDFromYouTubeURL(entry.URL)
+		if youtubeVideoID == "" {
+			continue
+		}
+
+		videosEntriesMapping[getVideoIDFromYouTubeURL(entry.URL)] = entry
+		videoIDs = append(videoIDs, youtubeVideoID)
 	}
 
+	if len(videoIDs) == 0 {
+		return
+	}
+
+	watchTimeMap, err := fetchYouTubeWatchTimeFromApiInBulk(videoIDs)
+	if err != nil {
+		slog.Warn("Unable to fetch YouTube watch time in bulk", slog.Any("error", err))
+		return
+	}
+
+	for videoID, watchTime := range watchTimeMap {
+		if entry, ok := videosEntriesMapping[videoID]; ok {
+			entry.ReadingTime = int(watchTime.Minutes())
+		}
+	}
+}
+
+func fetchYouTubeWatchTimeFromApiInBulk(videoIDs []string) (map[string]time.Duration, error) {
+	slog.Debug("Fetching YouTube watch time in bulk", slog.Any("video_ids", videoIDs))
+
 	apiQuery := url.Values{}
-	apiQuery.Set("id", parsedWebsiteURL.Query().Get("v"))
+	apiQuery.Set("id", strings.Join(videoIDs, ","))
 	apiQuery.Set("key", config.Opts.YouTubeApiKey())
 	apiQuery.Set("part", "contentDetails")
 
@@ -95,37 +130,33 @@ func fetchYouTubeWatchTimeFromApi(websiteURL string) (int, error) {
 		RawQuery: apiQuery.Encode(),
 	}
 
+	requestBuilder := fetcher.NewRequestBuilder()
+	requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
+	requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
+
 	responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
 	defer responseHandler.Close()
 
 	if localizedError := responseHandler.LocalizedError(); localizedError != nil {
-		slog.Warn("Unable to fetch contentDetails from YouTube API", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error()))
-		return 0, localizedError.Error()
-	}
-
-	var videos struct {
-		Items []struct {
-			ContentDetails struct {
-				Duration string `json:"duration"`
-			} `json:"contentDetails"`
-		} `json:"items"`
+		slog.Warn("Unable to fetch contentDetails from YouTube API", slog.Any("error", localizedError.Error()))
+		return nil, localizedError.Error()
 	}
 
+	var videos youtubeVideoListResponse
 	if err := json.NewDecoder(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())).Decode(&videos); err != nil {
-		return 0, fmt.Errorf("unable to decode JSON: %v", err)
+		return nil, fmt.Errorf("youtube: unable to decode JSON: %v", err)
 	}
 
-	if n := len(videos.Items); n != 1 {
-		return 0, fmt.Errorf("invalid items length: %d", n)
+	watchTimeMap := make(map[string]time.Duration)
+	for _, video := range videos.Items {
+		duration, err := parseISO8601(video.ContentDetails.Duration)
+		if err != nil {
+			slog.Warn("Unable to parse ISO8601 duration", slog.Any("error", err))
+			continue
+		}
+		watchTimeMap[video.ID] = duration
 	}
-
-	durs := videos.Items[0].ContentDetails.Duration
-	dur, err := parseISO8601(durs)
-	if err != nil {
-		return 0, fmt.Errorf("unable to parse duration %s: %v", durs, err)
-	}
-
-	return int(dur.Minutes()), nil
+	return watchTimeMap, nil
 }
 
 func parseISO8601(from string) (time.Duration, error) {
@@ -135,7 +166,7 @@ func parseISO8601(from string) (time.Duration, error) {
 	if iso8601Regex.MatchString(from) {
 		match = iso8601Regex.FindStringSubmatch(from)
 	} else {
-		return 0, errors.New("could not parse duration string")
+		return 0, errors.New("youtube: could not parse duration string")
 	}
 
 	for i, name := range iso8601Regex.SubexpNames() {
@@ -157,9 +188,18 @@ func parseISO8601(from string) (time.Duration, error) {
 		case "second":
 			d += time.Duration(val) * time.Second
 		default:
-			return 0, fmt.Errorf("unknown field %s", name)
+			return 0, fmt.Errorf("youtube: unknown field %s", name)
 		}
 	}
 
 	return d, nil
 }
+
+type youtubeVideoListResponse struct {
+	Items []struct {
+		ID             string `json:"id"`
+		ContentDetails struct {
+			Duration string `json:"duration"`
+		} `json:"contentDetails"`
+	} `json:"items"`
+}