oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-07-22 17:18:37 +00:00
Code Wiki Activity

refactor(readingtime): replace whatlanggo with an ad-hoc implementation

Browse source 
The package `github.com/abadojack/whatlanggo` is unmaintained since 5 years, is
overkill for simply detecting CJK, and is quite slow.
This commit is contained in:
Julien Voisin 2024-12-26 22:21:07 +00:00 • committed by GitHub
parent e22520fc55
commit 8df4b780a8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 22 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

40
internal/reader/readingtime/readingtime.go
View file

@ -7,33 +7,37 @@ package readingtime
import (
"math"
"strings"
"unicode"
"unicode/utf8"
"miniflux.app/v2/internal/reader/sanitizer"
"github.com/abadojack/whatlanggo"
)
// EstimateReadingTime returns the estimated reading time of an article in minute.
func EstimateReadingTime(content string, defaultReadingSpeed, cjkReadingSpeed int) int {
sanitizedContent := sanitizer.StripTags(content)
truncationPoint := min(len(sanitizedContent), 50)
// Litterature on language detection says that around 100 signes is enough, we're safe here.
truncationPoint := min(len(sanitizedContent), 250)
// We're only interested in identifying Japanse/Chinese/Korean
options := whatlanggo.Options{
Whitelist: map[whatlanggo.Lang]bool{
whatlanggo.Jpn: true,
whatlanggo.Cmn: true,
whatlanggo.Kor: true,
},
}
langInfo := whatlanggo.DetectWithOptions(sanitizedContent[:truncationPoint], options)
if langInfo.IsReliable() {
if isCJK(sanitizedContent[:truncationPoint]) {
return int(math.Ceil(float64(utf8.RuneCountInString(sanitizedContent)) / float64(cjkReadingSpeed)))
}
nbOfWords := len(strings.Fields(sanitizedContent))
return int(math.Ceil(float64(nbOfWords) / float64(defaultReadingSpeed)))
return int(math.Ceil(float64(len(strings.Fields(sanitizedContent))) / float64(defaultReadingSpeed)))
}
func isCJK(text string) bool {
totalCJK := 0
for _, r := range text[:min(len(text), 50)] {
if unicode.Is(unicode.Han, r) ||
unicode.Is(unicode.Hangul, r) ||
unicode.Is(unicode.Hiragana, r) ||
unicode.Is(unicode.Katakana, r) ||
unicode.Is(unicode.Yi, r) ||
unicode.Is(unicode.Bopomofo, r) {
totalCJK++
}
}
// if at least 50% of the text is CJK, odds are that the text is in CJK.
return totalCJK > len(text)/50
}