mirror of
https://github.com/miniflux/v2.git
synced 2025-07-22 17:18:37 +00:00
refactor(processor): move filters to a filter package
This commit is contained in:
Frédéric Guillot
parent
96c0ef4efd
commit
6282ac1f38
5 changed files with 109 additions and 97 deletions
|
|
@ -1,198 +0,0 @@
|
|||
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package processor // import "miniflux.app/v2/internal/reader/processor"
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"miniflux.app/v2/internal/model"
|
||||
)
|
||||
|
||||
// TODO factorize isBlockedEntry and isAllowedEntry
|
||||
|
||||
func isBlockedEntry(feed *model.Feed, entry *model.Entry, user *model.User) bool {
|
||||
if user.BlockFilterEntryRules != "" {
|
||||
rules := strings.Split(user.BlockFilterEntryRules, "\n")
|
||||
for _, rule := range rules {
|
||||
match := false
|
||||
parts := strings.SplitN(rule, "=", 2)
|
||||
if len(parts) != 2 {
|
||||
return false
|
||||
}
|
||||
part, pattern := parts[0], parts[1]
|
||||
|
||||
switch part {
|
||||
case "EntryDate":
|
||||
match = isDateMatchingPattern(pattern, entry.Date)
|
||||
case "EntryTitle":
|
||||
match, _ = regexp.MatchString(pattern, entry.Title)
|
||||
case "EntryURL":
|
||||
match, _ = regexp.MatchString(pattern, entry.URL)
|
||||
case "EntryCommentsURL":
|
||||
match, _ = regexp.MatchString(pattern, entry.CommentsURL)
|
||||
case "EntryContent":
|
||||
match, _ = regexp.MatchString(pattern, entry.Content)
|
||||
case "EntryAuthor":
|
||||
match, _ = regexp.MatchString(pattern, entry.Author)
|
||||
case "EntryTag":
|
||||
match = containsRegexPattern(pattern, entry.Tags)
|
||||
}
|
||||
|
||||
if match {
|
||||
slog.Debug("Blocking entry based on rule",
|
||||
slog.String("entry_url", entry.URL),
|
||||
slog.Int64("feed_id", feed.ID),
|
||||
slog.String("feed_url", feed.FeedURL),
|
||||
slog.String("rule", rule),
|
||||
)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if feed.BlocklistRules == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
compiledBlocklist, err := regexp.Compile(feed.BlocklistRules)
|
||||
if err != nil {
|
||||
slog.Debug("Failed on regexp compilation",
|
||||
slog.String("pattern", feed.BlocklistRules),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
return false
|
||||
}
|
||||
|
||||
containsBlockedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool {
|
||||
return compiledBlocklist.MatchString(tag)
|
||||
})
|
||||
|
||||
if compiledBlocklist.MatchString(entry.URL) || compiledBlocklist.MatchString(entry.Title) || compiledBlocklist.MatchString(entry.Author) || containsBlockedTag {
|
||||
slog.Debug("Blocking entry based on rule",
|
||||
slog.String("entry_url", entry.URL),
|
||||
slog.Int64("feed_id", feed.ID),
|
||||
slog.String("feed_url", feed.FeedURL),
|
||||
slog.String("rule", feed.BlocklistRules),
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func isAllowedEntry(feed *model.Feed, entry *model.Entry, user *model.User) bool {
|
||||
if user.KeepFilterEntryRules != "" {
|
||||
rules := strings.Split(user.KeepFilterEntryRules, "\n")
|
||||
for _, rule := range rules {
|
||||
match := false
|
||||
parts := strings.SplitN(rule, "=", 2)
|
||||
if len(parts) != 2 {
|
||||
return false
|
||||
}
|
||||
part, pattern := parts[0], parts[1]
|
||||
|
||||
switch part {
|
||||
case "EntryDate":
|
||||
match = isDateMatchingPattern(pattern, entry.Date)
|
||||
case "EntryTitle":
|
||||
match, _ = regexp.MatchString(pattern, entry.Title)
|
||||
case "EntryURL":
|
||||
match, _ = regexp.MatchString(pattern, entry.URL)
|
||||
case "EntryCommentsURL":
|
||||
match, _ = regexp.MatchString(pattern, entry.CommentsURL)
|
||||
case "EntryContent":
|
||||
match, _ = regexp.MatchString(pattern, entry.Content)
|
||||
case "EntryAuthor":
|
||||
match, _ = regexp.MatchString(pattern, entry.Author)
|
||||
case "EntryTag":
|
||||
match = containsRegexPattern(pattern, entry.Tags)
|
||||
}
|
||||
|
||||
if match {
|
||||
slog.Debug("Allowing entry based on rule",
|
||||
slog.String("entry_url", entry.URL),
|
||||
slog.Int64("feed_id", feed.ID),
|
||||
slog.String("feed_url", feed.FeedURL),
|
||||
slog.String("rule", rule),
|
||||
)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if feed.KeeplistRules == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
compiledKeeplist, err := regexp.Compile(feed.KeeplistRules)
|
||||
if err != nil {
|
||||
slog.Debug("Failed on regexp compilation",
|
||||
slog.String("pattern", feed.KeeplistRules),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
return false
|
||||
}
|
||||
containsAllowedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool {
|
||||
return compiledKeeplist.MatchString(tag)
|
||||
})
|
||||
|
||||
if compiledKeeplist.MatchString(entry.URL) || compiledKeeplist.MatchString(entry.Title) || compiledKeeplist.MatchString(entry.Author) || containsAllowedTag {
|
||||
slog.Debug("Allow entry based on rule",
|
||||
slog.String("entry_url", entry.URL),
|
||||
slog.Int64("feed_id", feed.ID),
|
||||
slog.String("feed_url", feed.FeedURL),
|
||||
slog.String("rule", feed.KeeplistRules),
|
||||
)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDateMatchingPattern(pattern string, entryDate time.Time) bool {
|
||||
if pattern == "future" {
|
||||
return entryDate.After(time.Now())
|
||||
}
|
||||
|
||||
parts := strings.SplitN(pattern, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return false
|
||||
}
|
||||
|
||||
operator, dateStr := parts[0], parts[1]
|
||||
|
||||
switch operator {
|
||||
case "before":
|
||||
targetDate, err := time.Parse("2006-01-02", dateStr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return entryDate.Before(targetDate)
|
||||
case "after":
|
||||
targetDate, err := time.Parse("2006-01-02", dateStr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return entryDate.After(targetDate)
|
||||
case "between":
|
||||
dates := strings.Split(dateStr, ",")
|
||||
if len(dates) != 2 {
|
||||
return false
|
||||
}
|
||||
startDate, err := time.Parse("2006-01-02", dates[0])
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
endDate, err := time.Parse("2006-01-02", dates[1])
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return entryDate.After(startDate) && entryDate.Before(endDate)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ import (
|
|||
"miniflux.app/v2/internal/model"
|
||||
"miniflux.app/v2/internal/proxyrotator"
|
||||
"miniflux.app/v2/internal/reader/fetcher"
|
||||
"miniflux.app/v2/internal/reader/filter"
|
||||
"miniflux.app/v2/internal/reader/readingtime"
|
||||
"miniflux.app/v2/internal/reader/rewrite"
|
||||
"miniflux.app/v2/internal/reader/sanitizer"
|
||||
|
|
@ -49,7 +50,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
|
|||
slog.Int64("feed_id", feed.ID),
|
||||
slog.String("feed_url", feed.FeedURL),
|
||||
)
|
||||
if isBlockedEntry(feed, entry, user) || !isAllowedEntry(feed, entry, user) || !isRecentEntry(entry) {
|
||||
if filter.IsBlockedEntry(feed, entry, user) || !filter.IsAllowedEntry(feed, entry, user) || !isRecentEntry(entry) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,86 +11,6 @@ import (
|
|||
"miniflux.app/v2/internal/model"
|
||||
)
|
||||
|
||||
func TestBlockingEntries(t *testing.T) {
|
||||
var scenarios = []struct {
|
||||
feed *model.Feed
|
||||
entry *model.Entry
|
||||
user *model.User
|
||||
expected bool
|
||||
}{
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{URL: "https://example.com"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{URL: "https://different.com"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Some Example"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Something different"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Something different", Tags: []string{"example", "something else"}}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Example", Tags: []string{"example", "something else"}}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Example", Tags: []string{"something different", "something else"}}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Something different", Tags: []string{"something different", "something else"}}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Something different", Author: "Example"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: "(?i)example"}, &model.Entry{Title: "Something different", Author: "Something different"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1}, &model.Entry{Title: "No rule defined"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{URL: "https://example.com", Title: "Some Example"}, &model.User{BlockFilterEntryRules: "EntryURL=(?i)example\nEntryTitle=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{URL: "https://different.com", Title: "Some Test"}, &model.User{BlockFilterEntryRules: "EntryURL=(?i)example\nEntryTitle=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{URL: "https://different.com", Title: "Some Example"}, &model.User{BlockFilterEntryRules: "EntryURL=(?i)example\nEntryTitle=(?i)Test"}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{CommentsURL: "https://example.com", Content: "Some Example"}, &model.User{BlockFilterEntryRules: "EntryCommentsURL=(?i)example\nEntryContent=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{CommentsURL: "https://different.com", Content: "Some Test"}, &model.User{BlockFilterEntryRules: "EntryCommentsURL=(?i)example\nEntryContent=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{CommentsURL: "https://different.com", Content: "Some Example"}, &model.User{BlockFilterEntryRules: "EntryCommentsURL=(?i)example\nEntryContent=(?i)Test"}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Example", Tags: []string{"example", "something else"}}, &model.User{BlockFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{BlockFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)example"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{BlockFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, false},
|
||||
}
|
||||
|
||||
for _, tc := range scenarios {
|
||||
result := isBlockedEntry(tc.feed, tc.entry, tc.user)
|
||||
if tc.expected != result {
|
||||
t.Errorf(`Unexpected result, got %v for entry %q`, result, tc.entry.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowEntries(t *testing.T) {
|
||||
var scenarios = []struct {
|
||||
feed *model.Feed
|
||||
entry *model.Entry
|
||||
user *model.User
|
||||
expected bool
|
||||
}{
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "https://example.com"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "https://different.com"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Some Example"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Something different"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1}, &model.Entry{Title: "No rule defined"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Something different", Tags: []string{"example", "something else"}}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Example", Tags: []string{"example", "something else"}}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Example", Tags: []string{"something different", "something else"}}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Something more", Tags: []string{"something different", "something else"}}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Something different", Author: "Example"}, &model.User{}, true},
|
||||
{&model.Feed{ID: 1, KeeplistRules: "(?i)example"}, &model.Entry{Title: "Something different", Author: "Something different"}, &model.User{}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{URL: "https://example.com", Title: "Some Example"}, &model.User{KeepFilterEntryRules: "EntryURL=(?i)example\nEntryTitle=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{URL: "https://different.com", Title: "Some Test"}, &model.User{KeepFilterEntryRules: "EntryURL=(?i)example\nEntryTitle=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{URL: "https://different.com", Title: "Some Example"}, &model.User{KeepFilterEntryRules: "EntryURL=(?i)example\nEntryTitle=(?i)Test"}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{CommentsURL: "https://example.com", Content: "Some Example"}, &model.User{KeepFilterEntryRules: "EntryCommentsURL=(?i)example\nEntryContent=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{CommentsURL: "https://different.com", Content: "Some Test"}, &model.User{KeepFilterEntryRules: "EntryCommentsURL=(?i)example\nEntryContent=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{CommentsURL: "https://different.com", Content: "Some Example"}, &model.User{KeepFilterEntryRules: "EntryCommentsURL=(?i)example\nEntryContent=(?i)Test"}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Example", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)example"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Now().Add(24 * time.Hour)}, &model.User{KeepFilterEntryRules: "EntryDate=future"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Now().Add(-24 * time.Hour)}, &model.User{KeepFilterEntryRules: "EntryDate=future"}, false},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 14, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=before:2024-03-15"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 16, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=after:2024-03-15"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 10, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=between:2024-03-01,2024-03-15"}, true},
|
||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 2, 28, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=between:2024-03-01,2024-03-15"}, false},
|
||||
}
|
||||
|
||||
for _, tc := range scenarios {
|
||||
result := isAllowedEntry(tc.feed, tc.entry, tc.user)
|
||||
if tc.expected != result {
|
||||
t.Errorf(`Unexpected result, got %v for entry %q`, result, tc.entry.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRecentEntry(t *testing.T) {
|
||||
parser := config.NewParser()
|
||||
var err error
|
||||
|
|
|
|||
|
|
@ -71,12 +71,3 @@ func minifyContent(content string) string {
|
|||
|
||||
return content
|
||||
}
|
||||
|
||||
func containsRegexPattern(pattern string, entries []string) bool {
|
||||
for _, entry := range entries {
|
||||
if matched, _ := regexp.MatchString(pattern, entry); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue