diff --git a/client/model.go b/client/model.go
index 1f7a28ea..d72d92f4 100644
--- a/client/model.go
+++ b/client/model.go
@@ -171,6 +171,7 @@ type Feed struct {
HideGlobally bool `json:"hide_globally"`
DisableHTTP2 bool `json:"disable_http2"`
ProxyURL string `json:"proxy_url"`
+ DeduplicateAgainstAll bool `json:"deduplicate_against_all"`
}
// FeedCreationRequest represents the request to create a feed.
@@ -193,6 +194,7 @@ type FeedCreationRequest struct {
HideGlobally bool `json:"hide_globally"`
DisableHTTP2 bool `json:"disable_http2"`
ProxyURL string `json:"proxy_url"`
+ DeduplicateAgainstAll bool `json:"deduplicate_against_all"`
}
// FeedModificationRequest represents the request to update a feed.
@@ -217,6 +219,7 @@ type FeedModificationRequest struct {
HideGlobally *bool `json:"hide_globally"`
DisableHTTP2 *bool `json:"disable_http2"`
ProxyURL *string `json:"proxy_url"`
+ DeduplicateAgainstAll *bool `json:"deduplicate_against_all"`
}
// FeedIcon represents the feed icon.
diff --git a/internal/model/feed.go b/internal/model/feed.go
index 6f289b47..9ec374f3 100644
--- a/internal/model/feed.go
+++ b/internal/model/feed.go
@@ -60,6 +60,7 @@ type Feed struct {
PushoverEnabled bool `json:"pushover_enabled"`
PushoverPriority int `json:"pushover_priority"`
ProxyURL string `json:"proxy_url"`
+ DeduplicateAgainstAll bool `json:"deduplicate_against_all"`
// Non-persisted attributes
Category *Category `json:"category,omitempty"`
@@ -170,6 +171,7 @@ type FeedCreationRequest struct {
UrlRewriteRules string `json:"urlrewrite_rules"`
DisableHTTP2 bool `json:"disable_http2"`
ProxyURL string `json:"proxy_url"`
+ DeduplicateAgainstAll bool `json:"deduplicate_against_all"`
}
type FeedCreationRequestFromSubscriptionDiscovery struct {
@@ -205,6 +207,7 @@ type FeedModificationRequest struct {
HideGlobally *bool `json:"hide_globally"`
DisableHTTP2 *bool `json:"disable_http2"`
ProxyURL *string `json:"proxy_url"`
+ DeduplicateAgainstAll *bool `json:"deduplicate_against_all"`
}
// Patch updates a feed with modified values.
@@ -300,6 +303,10 @@ func (f *FeedModificationRequest) Patch(feed *Feed) {
if f.ProxyURL != nil {
feed.ProxyURL = *f.ProxyURL
}
+
+ if f.DeduplicateAgainstAll != nil {
+ feed.DeduplicateAgainstAll = *f.DeduplicateAgainstAll
+ }
}
// Feeds is a list of feed
diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go
index 43f6c764..7afa30b8 100644
--- a/internal/reader/handler/handler.go
+++ b/internal/reader/handler/handler.go
@@ -336,7 +336,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
// We don't update existing entries when the crawler is enabled (we crawl only inexisting entries). Unless it is forced to refresh
updateExistingEntries := forceRefresh || !originalFeed.Crawler
- newEntries, storeErr := store.RefreshFeedEntries(originalFeed.UserID, originalFeed.ID, originalFeed.Entries, updateExistingEntries)
+ newEntries, storeErr := store.RefreshFeedEntries(originalFeed.UserID, originalFeed.ID, originalFeed.Entries, updateExistingEntries, originalFeed.DeduplicateAgainstAll)
if storeErr != nil {
localizedError := locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
user, storeErr := store.UserByID(userID)
diff --git a/internal/storage/entry.go b/internal/storage/entry.go
index c6add8e2..fcf7b3e7 100644
--- a/internal/storage/entry.go
+++ b/internal/storage/entry.go
@@ -225,6 +225,19 @@ func (s *Storage) entryExists(tx *sql.Tx, entry *model.Entry) (bool, error) {
return result, nil
}
+// entryExistsAnywhere checks if an entry already exists in other feeds based on its hash.
+func (s *Storage) entryExistsAnywhere(tx *sql.Tx, entry *model.Entry) (bool, error) {
+ var result bool
+
+ err := tx.QueryRow(`SELECT true FROM entries WHERE hash=$1`, entry.Hash).Scan(&result)
+
+ if err != nil && err != sql.ErrNoRows {
+ return result, fmt.Errorf(`store: unable to check if entry exists: %v`, err)
+ }
+
+ return result, nil
+}
+
func (s *Storage) IsNewEntry(feedID int64, entryHash string) bool {
var result bool
s.db.QueryRow(`SELECT true FROM entries WHERE feed_id=$1 AND hash=$2`, feedID, entryHash).Scan(&result)
@@ -268,7 +281,7 @@ func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
}
// RefreshFeedEntries updates feed entries while refreshing a feed.
-func (s *Storage) RefreshFeedEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool) (newEntries model.Entries, err error) {
+func (s *Storage) RefreshFeedEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool, deduplicateAgainstAll bool) (newEntries model.Entries, err error) {
var entryHashes []string
for _, entry := range entries {
@@ -280,7 +293,16 @@ func (s *Storage) RefreshFeedEntries(userID, feedID int64, entries model.Entries
return nil, fmt.Errorf(`store: unable to start transaction: %v`, err)
}
- entryExists, err := s.entryExists(tx, entry)
+ entryExists := false
+ if deduplicateAgainstAll {
+ entryExists, err = s.entryExistsAnywhere(tx, entry)
+ // maybe another feed was refreshed and has this entry as well,
+ // so we need to markd it as removed here.
+ updateExistingEntries = true
+ entry.Status = model.EntryStatusRemoved
+ } else {
+ entryExists, err = s.entryExists(tx, entry)
+ }
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
return nil, fmt.Errorf(`store: unable to rollback transaction: %v (rolled back due to: %v)`, rollbackErr, err)
diff --git a/internal/storage/feed_query_builder.go b/internal/storage/feed_query_builder.go
index 82e6594f..db55c15a 100644
--- a/internal/storage/feed_query_builder.go
+++ b/internal/storage/feed_query_builder.go
@@ -253,6 +253,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
&feed.PushoverEnabled,
&feed.PushoverPriority,
&feed.ProxyURL,
+ &feed.DeduplicateAgainstAll,
)
if err != nil {
diff --git a/internal/template/templates/views/edit_feed.html b/internal/template/templates/views/edit_feed.html
index 8921d18a..7dc53b23 100644
--- a/internal/template/templates/views/edit_feed.html
+++ b/internal/template/templates/views/edit_feed.html
@@ -72,6 +72,7 @@
+
diff --git a/internal/ui/feed_edit.go b/internal/ui/feed_edit.go
index d1dcdc6f..fc9a3435 100644
--- a/internal/ui/feed_edit.go
+++ b/internal/ui/feed_edit.go
@@ -71,6 +71,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
PushoverEnabled: feed.PushoverEnabled,
PushoverPriority: feed.PushoverPriority,
ProxyURL: feed.ProxyURL,
+ DeduplicateAgainstAll: feed.DeduplicateAgainstAll,
}
sess := session.New(h.store, request.SessionID(r))
diff --git a/internal/ui/form/feed.go b/internal/ui/form/feed.go
index fdb94b8e..820ccf21 100644
--- a/internal/ui/form/feed.go
+++ b/internal/ui/form/feed.go
@@ -43,6 +43,7 @@ type FeedForm struct {
PushoverEnabled bool
PushoverPriority int
ProxyURL string
+ DeduplicateAgainstAll bool
}
// Merge updates the fields of the given feed.
@@ -79,6 +80,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.PushoverEnabled = f.PushoverEnabled
feed.PushoverPriority = f.PushoverPriority
feed.ProxyURL = f.ProxyURL
+ feed.DeduplicateAgainstAll = f.DeduplicateAgainstAll
return feed
}
@@ -130,5 +132,6 @@ func NewFeedForm(r *http.Request) *FeedForm {
PushoverEnabled: r.FormValue("pushover_enabled") == "1",
PushoverPriority: pushoverPriority,
ProxyURL: r.FormValue("proxy_url"),
+ DeduplicateAgainstAll: r.FormValue("deduplicate_against_all") == "1",
}
}