oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-06-27 16:36:00 +00:00
Code Wiki Activity

Merge ef13756b1a into fcf86e33b9

Browse source
This commit is contained in:
Julien Voisin 2025-06-27 14:27:30 +00:00 • committed by GitHub
commit 027ca94df6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 34 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

33
internal/reader/readability/readability.go
View file

@ -26,8 +26,8 @@ var (
okMaybeItsACandidateRegexp = regexp.MustCompile(`and|article|body|column|main|shadow`) okMaybeItsACandidateRegexp = regexp.MustCompile(`and|article|body|column|main|shadow`)
unlikelyCandidatesRegexp = regexp.MustCompile(`banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`) unlikelyCandidatesRegexp = regexp.MustCompile(`banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
negativeRegexp = regexp.MustCompile(`hid|banner|combx|comment|com-|contact|foot|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby`) positive = [...]string{"article", "blog", "body", "content", "entry", "h-entry", "hentry", "main", "page", "pagination", "post", "story", "text"}
positiveRegexp = regexp.MustCompile(`article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`) negative = [...]string{"author", "banner", "byline", "com-", "combx", "comment", "contact", "dateline", "foot", "hid", "masthead", "media", "meta", "modal", "outbrain", "promo", "related", "scroll", "share", "shopping", "shoutbox", "sidebar", "skyscraper", "sponsor", "tags", "tool", "widget", "writtenby"}
) )
type candidate struct { type candidate struct {
@ -294,26 +294,29 @@ func getClassWeight(s *goquery.Selection) float32 {
weight := 0 weight := 0
if class, ok := s.Attr("class"); ok { if class, ok := s.Attr("class"); ok {
class = strings.ToLower(class) weight += getWeight(class)
if negativeRegexp.MatchString(class) {
weight -= 25
} else if positiveRegexp.MatchString(class) {
weight += 25
} }
}
if id, ok := s.Attr("id"); ok { if id, ok := s.Attr("id"); ok {
id = strings.ToLower(id) weight += getWeight(id)
if negativeRegexp.MatchString(id) {
weight -= 25
} else if positiveRegexp.MatchString(id) {
weight += 25
}
} }
return float32(weight) return float32(weight)
} }
func getWeight(s string) int {
for _, pos := range negative {
if strings.Contains(s, pos) {
return -25
}
}
for _, pos := range positive {
if strings.Contains(s, pos) {
return +25
}
}
return 0
}
func transformMisusedDivsIntoParagraphs(document *goquery.Document) { func transformMisusedDivsIntoParagraphs(document *goquery.Document) {
document.Find("div").Each(func(i int, s *goquery.Selection) { document.Find("div").Each(func(i int, s *goquery.Selection) {
html, _ := s.Html() html, _ := s.Html()

16
internal/reader/readability/readability_test.go
View file

@ -204,3 +204,19 @@ func BenchmarkExtractContent(b *testing.B) {
} }
} }
} }
func BenchmarkGetWeight(b *testing.B) {
testCases := []string{
"p-3 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content",
"d-flex flex-column mb-3",
"AppHeader-search-control AppHeader-search-control-overflow",
"Button Button--iconOnly Button--invisible Button--medium mr-1 px-2 py-0 d-flex flex-items-center rounded-1 color-fg-muted",
"sr-only",
"validation-12753bbc-b4d1-4e10-bec6-92e585d1699d",
}
for range b.N {
for _, v := range testCases {
getWeight(v)
}
}
}