Merge branch 'main' into sitemap

internal/reader/readingtime/readingtime.go

@@ -7,33 +7,37 @@ package readingtime
 import (
 	"math"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 
 	"miniflux.app/v2/internal/reader/sanitizer"
-
-	"github.com/abadojack/whatlanggo"
 )
 
 // EstimateReadingTime returns the estimated reading time of an article in minute.
 func EstimateReadingTime(content string, defaultReadingSpeed, cjkReadingSpeed int) int {
 	sanitizedContent := sanitizer.StripTags(content)
+	truncationPoint := min(len(sanitizedContent), 50)
 
-	// Litterature on language detection says that around 100 signes is enough, we're safe here.
-	truncationPoint := min(len(sanitizedContent), 250)
-
-	// We're only interested in identifying Japanse/Chinese/Korean
-	options := whatlanggo.Options{
-		Whitelist: map[whatlanggo.Lang]bool{
-			whatlanggo.Jpn: true,
-			whatlanggo.Cmn: true,
-			whatlanggo.Kor: true,
-		},
-	}
-	langInfo := whatlanggo.DetectWithOptions(sanitizedContent[:truncationPoint], options)
-
-	if langInfo.IsReliable() {
+	if isCJK(sanitizedContent[:truncationPoint]) {
 		return int(math.Ceil(float64(utf8.RuneCountInString(sanitizedContent)) / float64(cjkReadingSpeed)))
 	}
-	nbOfWords := len(strings.Fields(sanitizedContent))
-	return int(math.Ceil(float64(nbOfWords) / float64(defaultReadingSpeed)))
+	return int(math.Ceil(float64(len(strings.Fields(sanitizedContent))) / float64(defaultReadingSpeed)))
+}
+
+func isCJK(text string) bool {
+	totalCJK := 0
+
+	for _, r := range text[:min(len(text), 50)] {
+		if unicode.Is(unicode.Han, r) ||
+			unicode.Is(unicode.Hangul, r) ||
+			unicode.Is(unicode.Hiragana, r) ||
+			unicode.Is(unicode.Katakana, r) ||
+			unicode.Is(unicode.Yi, r) ||
+			unicode.Is(unicode.Bopomofo, r) {
+			totalCJK++
+		}
+	}
+
+	// if at least 50% of the text is CJK, odds are that the text is in CJK.
+	return totalCJK > len(text)/50
 }