oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-06-27 16:36:00 +00:00
Code Wiki Activity

perf(readability): Simplify removeUnlikelyCandidates

Browse source 
- Use an array of strings instead of a regex, like done in ef13756b1a
- Extract the `shouldRemove` function from `removeUnlikelyCandidates`, as there
  is no reason to have it there instead of being a proper standalone function.
- Improve a condition, where the goquery selection would have its `id`
  attribute left unchecked if a `class` one was present, regardless of if
  `class` was a candidate to removal or not.
- Add some comments
This commit is contained in:
jvoisin 2025-06-27 16:39:53 +02:00
parent fcf86e33b9
commit daedd26e2b
1 changed files with 29 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

46
internal/reader/readability/readability.go
View file

@ -23,8 +23,9 @@ const (
var (
divToPElementsRegexp = regexp.MustCompile(`(?i)<(?:a|blockquote|dl|div|img|ol|p|pre|table|ul)[ />]`)
okMaybeItsACandidateRegexp = regexp.MustCompile(`and|article|body|column|main|shadow`)
unlikelyCandidatesRegexp = regexp.MustCompile(`banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
strongCandidates = [...]string{"popupbody", "-ad", "g-plus"}
maybeCandidate = [...]string{"and", "article", "body", "column", "main", "shadow"}
unlikelyCandidate = [...]string{"banner", "breadcrumbs", "combx", "comment", "community", "cover-wrap", "disqus", "extra", "foot", "header", "legends", "menu", "modal", "related", "remark", "replies", "rss", "shoutbox", "sidebar", "skyscraper", "social", "sponsor", "supplemental", "ad-break", "agegate", "pagination", "pager", "popup", "yom-remote"}
negativeRegexp = regexp.MustCompile(`hid|banner|combx|comment|com-|contact|foot|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby`)
positiveRegexp = regexp.MustCompile(`article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`)
@ -145,18 +146,33 @@ func getArticle(topCandidate *candidate, candidates candidateList) string {
output.WriteString("</div>")
return output.String()
}
func shouldRemoveCandidate(str string) bool {
str = strings.ToLower(str)
func removeUnlikelyCandidates(document *goquery.Document) {
var shouldRemove = func(str string) bool {
str = strings.ToLower(str)
if strings.Contains(str, "popupbody") || strings.Contains(str, "-ad") || strings.Contains(str, "g-plus") {
return true
} else if unlikelyCandidatesRegexp.MatchString(str) && !okMaybeItsACandidateRegexp.MatchString(str) {
// Those candidates have no false-positives, no need to check against `maybeCandidate`
for _, strong := range strongCandidates {
if strings.Contains(str, strong) {
return true
}
return false
}
for _, unlikely := range unlikelyCandidate {
if strings.Contains(str, unlikely) {
// Do we have a false positive?
for _, maybe := range maybeCandidate {
if strings.Contains(str, maybe) {
return false
}
}
// Nope, it's a true positive!
return true
}
}
return false
}
func removeUnlikelyCandidates(document *goquery.Document) {
document.Find("*").Each(func(i int, s *goquery.Selection) {
if s.Length() == 0 || s.Get(0).Data == "html" || s.Get(0).Data == "body" {
return
@ -167,14 +183,10 @@ func removeUnlikelyCandidates(document *goquery.Document) {
return
}
if class, ok := s.Attr("class"); ok {
if shouldRemove(class) {
s.Remove()
}
} else if id, ok := s.Attr("id"); ok {
if shouldRemove(id) {
s.Remove()
}
if class, ok := s.Attr("class"); ok && shouldRemoveCandidate(class) {
s.Remove()
} else if id, ok := s.Attr("id"); ok && shouldRemoveCandidate(id) {
s.Remove()
}
})
}