diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go
index d45f54c8..cc8f5502 100644
--- a/internal/reader/processor/processor.go
+++ b/internal/reader/processor/processor.go
@@ -62,9 +62,8 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
entry.URL = cleanedURL
}
- pageBaseURL := ""
- rewrittenURL := rewriteEntryURL(feed, entry)
- entry.URL = rewrittenURL
+ webpageBaseURL := ""
+ entry.URL = rewriteEntryURL(feed, entry)
entryIsNew := store.IsNewEntry(feed.ID, entry.Hash)
if feed.Crawler && (entryIsNew || forceRefresh) {
slog.Debug("Scraping entry",
@@ -76,7 +75,6 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
slog.String("feed_url", feed.FeedURL),
slog.Bool("entry_is_new", entryIsNew),
slog.Bool("force_refresh", forceRefresh),
- slog.String("rewritten_url", rewrittenURL),
)
startTime := time.Now()
@@ -94,12 +92,12 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
scrapedPageBaseURL, extractedContent, scraperErr := scraper.ScrapeWebsite(
requestBuilder,
- rewrittenURL,
+ entry.URL,
feed.ScraperRules,
)
if scrapedPageBaseURL != "" {
- pageBaseURL = scrapedPageBaseURL
+ webpageBaseURL = scrapedPageBaseURL
}
if config.Opts.HasMetricsCollector() {
@@ -124,14 +122,14 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
}
}
- rewrite.Rewriter(rewrittenURL, entry, feed.RewriteRules)
+ rewrite.ApplyContentRewriteRules(entry, feed.RewriteRules)
- if pageBaseURL == "" {
- pageBaseURL = rewrittenURL
+ if webpageBaseURL == "" {
+ webpageBaseURL = entry.URL
}
// The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered out.
- entry.Content = sanitizer.SanitizeHTML(pageBaseURL, entry.Content, &sanitizer.SanitizerOptions{OpenLinksInNewTab: user.OpenExternalLinksInNewTab})
+ entry.Content = sanitizer.SanitizeHTML(webpageBaseURL, entry.Content, &sanitizer.SanitizerOptions{OpenLinksInNewTab: user.OpenExternalLinksInNewTab})
updateEntryReadingTime(store, feed, entry, entryIsNew, user)
@@ -148,7 +146,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) error {
startTime := time.Now()
- rewrittenEntryURL := rewriteEntryURL(feed, entry)
+ entry.URL = rewriteEntryURL(feed, entry)
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
@@ -161,9 +159,9 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(feed.DisableHTTP2)
- pageBaseURL, extractedContent, scraperErr := scraper.ScrapeWebsite(
+ webpageBaseURL, extractedContent, scraperErr := scraper.ScrapeWebsite(
requestBuilder,
- rewrittenEntryURL,
+ entry.URL,
feed.ScraperRules,
)
@@ -186,8 +184,8 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
}
}
- rewrite.Rewriter(rewrittenEntryURL, entry, entry.Feed.RewriteRules)
- entry.Content = sanitizer.SanitizeHTML(pageBaseURL, entry.Content, &sanitizer.SanitizerOptions{OpenLinksInNewTab: user.OpenExternalLinksInNewTab})
+ rewrite.ApplyContentRewriteRules(entry, entry.Feed.RewriteRules)
+ entry.Content = sanitizer.SanitizeHTML(webpageBaseURL, entry.Content, &sanitizer.SanitizerOptions{OpenLinksInNewTab: user.OpenExternalLinksInNewTab})
return nil
}
diff --git a/internal/reader/rewrite/rewriter.go b/internal/reader/rewrite/rewriter.go
index 35395ac9..50284bc9 100644
--- a/internal/reader/rewrite/rewriter.go
+++ b/internal/reader/rewrite/rewriter.go
@@ -97,9 +97,8 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) {
}
}
-// Rewriter modify item contents with a set of rewriting rules.
-func Rewriter(entryURL string, entry *model.Entry, customRewriteRules string) {
- rulesList := getPredefinedRewriteRules(entryURL)
+func ApplyContentRewriteRules(entry *model.Entry, customRewriteRules string) {
+ rulesList := getPredefinedRewriteRules(entry.URL)
if customRewriteRules != "" {
rulesList = customRewriteRules
}
@@ -109,11 +108,11 @@ func Rewriter(entryURL string, entry *model.Entry, customRewriteRules string) {
slog.Debug("Rewrite rules applied",
slog.Any("rules", rules),
- slog.String("entry_url", entryURL),
+ slog.String("entry_url", entry.URL),
)
for _, rule := range rules {
- rule.applyRule(entryURL, entry)
+ rule.applyRule(entry.URL, entry)
}
}
diff --git a/internal/reader/rewrite/rewriter_test.go b/internal/reader/rewrite/rewriter_test.go
index 52ea5c01..4d9ba582 100644
--- a/internal/reader/rewrite/rewriter_test.go
+++ b/internal/reader/rewrite/rewriter_test.go
@@ -50,14 +50,16 @@ func TestReplaceTextLinks(t *testing.T) {
func TestRewriteWithNoMatchingRule(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Some text.`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Some text.`,
}
- Rewriter("https://example.org/article", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -68,14 +70,16 @@ func TestRewriteWithYoutubeLink(t *testing.T) {
config.Opts = config.NewOptions()
controlEntry := &model.Entry{
+ URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `
Video Description`,
}
testEntry := &model.Entry{
+ URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `Video Description`,
}
- Rewriter("https://www.youtube.com/watch?v=1234", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -95,14 +99,16 @@ func TestRewriteWithYoutubeLinkAndCustomEmbedURL(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `
Video Description`,
}
testEntry := &model.Entry{
+ URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `Video Description`,
}
- Rewriter("https://www.youtube.com/watch?v=1234", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -111,14 +117,16 @@ func TestRewriteWithYoutubeLinkAndCustomEmbedURL(t *testing.T) {
func TestRewriteWithInexistingCustomRule(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `Video Description`,
}
testEntry := &model.Entry{
+ URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `Video Description`,
}
- Rewriter("https://www.youtube.com/watch?v=1234", testEntry, `some rule`)
+ ApplyContentRewriteRules(testEntry, `some rule`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -127,14 +135,16 @@ func TestRewriteWithInexistingCustomRule(t *testing.T) {
func TestRewriteWithXkcdLink(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: `
Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.
`,
}
testEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: ``,
}
- Rewriter("https://xkcd.com/1912/", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -143,14 +153,16 @@ func TestRewriteWithXkcdLink(t *testing.T) {
func TestRewriteWithXkcdLinkHtmlInjection(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: `<foo>
`,
}
testEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: ``,
}
- Rewriter("https://xkcd.com/1912/", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -159,14 +171,16 @@ func TestRewriteWithXkcdLinkHtmlInjection(t *testing.T) {
func TestRewriteWithXkcdLinkAndImageNoTitle(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: ``,
}
testEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: ``,
}
- Rewriter("https://xkcd.com/1912/", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -175,14 +189,16 @@ func TestRewriteWithXkcdLinkAndImageNoTitle(t *testing.T) {
func TestRewriteWithXkcdLinkAndNoImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: `test`,
}
testEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: `test`,
}
- Rewriter("https://xkcd.com/1912/", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -191,14 +207,16 @@ func TestRewriteWithXkcdLinkAndNoImage(t *testing.T) {
func TestRewriteWithXkcdAndNoImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: `test`,
}
testEntry := &model.Entry{
+ URL: "https://xkcd.com/1912/",
Title: `A title`,
Content: `test`,
}
- Rewriter("https://xkcd.com/1912/", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -207,14 +225,16 @@ func TestRewriteWithXkcdAndNoImage(t *testing.T) {
func TestRewriteMailtoLink(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://www.qwantz.com/",
Title: `A title`,
Content: `contact [blah blah]`,
}
testEntry := &model.Entry{
+ URL: "https://www.qwantz.com/",
Title: `A title`,
Content: `contact`,
}
- Rewriter("https://www.qwantz.com/", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -223,14 +243,16 @@ func TestRewriteMailtoLink(t *testing.T) {
func TestRewriteWithPDFLink(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/document.pdf",
Title: `A title`,
Content: `PDF
test`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/document.pdf",
Title: `A title`,
Content: `test`,
}
- Rewriter("https://example.org/document.pdf", testEntry, ``)
+ ApplyContentRewriteRules(testEntry, ``)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -239,14 +261,16 @@ func TestRewriteWithPDFLink(t *testing.T) {
func TestRewriteWithNoLazyImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_image")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_image")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -255,14 +279,16 @@ func TestRewriteWithNoLazyImage(t *testing.T) {
func TestRewriteWithLazyImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
`,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_image")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_image")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -271,14 +297,16 @@ func TestRewriteWithLazyImage(t *testing.T) {
func TestRewriteWithLazyDivImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_image")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_image")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -287,14 +315,16 @@ func TestRewriteWithLazyDivImage(t *testing.T) {
func TestRewriteWithUnknownLazyNoScriptImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `![Image]()
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_image")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_image")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -303,14 +333,16 @@ func TestRewriteWithUnknownLazyNoScriptImage(t *testing.T) {
func TestRewriteWithLazySrcset(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `![Image]()
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `![Image]()
`,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_image")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_image")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -319,14 +351,16 @@ func TestRewriteWithLazySrcset(t *testing.T) {
func TestRewriteWithImageAndLazySrcset(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_image")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_image")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -335,14 +369,16 @@ func TestRewriteWithImageAndLazySrcset(t *testing.T) {
func TestRewriteWithNoLazyIframe(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_iframe")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -351,14 +387,16 @@ func TestRewriteWithNoLazyIframe(t *testing.T) {
func TestRewriteWithLazyIframe(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_iframe")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -367,14 +405,16 @@ func TestRewriteWithLazyIframe(t *testing.T) {
func TestRewriteWithLazyIframeAndSrc(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe")
+ ApplyContentRewriteRules(testEntry, "add_dynamic_iframe")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -383,14 +423,16 @@ func TestRewriteWithLazyIframeAndSrc(t *testing.T) {
func TestNewLineRewriteRule(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `A
B
C`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: "A\nB\nC",
}
- Rewriter("https://example.org/article", testEntry, "nl2br")
+ ApplyContentRewriteRules(testEntry, "nl2br")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -399,14 +441,16 @@ func TestNewLineRewriteRule(t *testing.T) {
func TestConvertTextLinkRewriteRule(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Test: http://example.org/a/b`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Test: http://example.org/a/b`,
}
- Rewriter("https://example.org/article", testEntry, "convert_text_link")
+ ApplyContentRewriteRules(testEntry, "convert_text_link")
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -415,10 +459,12 @@ func TestConvertTextLinkRewriteRule(t *testing.T) {
func TestMediumImage(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -440,7 +486,7 @@ func TestMediumImage(t *testing.T) {
`,
}
- Rewriter("https://example.org/article", testEntry, "fix_medium_images")
+ ApplyContentRewriteRules(testEntry, "fix_medium_images")
testEntry.Content = strings.TrimSpace(testEntry.Content)
if !reflect.DeepEqual(testEntry, controlEntry) {
@@ -450,14 +496,16 @@ func TestMediumImage(t *testing.T) {
func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
MDN Logo`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `MDN Logo`,
}
- Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images")
+ ApplyContentRewriteRules(testEntry, "use_noscript_figure_images")
testEntry.Content = strings.TrimSpace(testEntry.Content)
if !reflect.DeepEqual(testEntry, controlEntry) {
@@ -467,14 +515,16 @@ func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) {
func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
MDN Logo`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `MDN Logo`,
}
- Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images")
+ ApplyContentRewriteRules(testEntry, "use_noscript_figure_images")
testEntry.Content = strings.TrimSpace(testEntry.Content)
if !reflect.DeepEqual(testEntry, controlEntry) {
@@ -484,14 +534,16 @@ func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) {
func TestRewriteReplaceCustom(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
`,
}
- Rewriter("https://example.org/article", testEntry, `replace("article/(.*).svg"|"article/$1.png")`)
+ ApplyContentRewriteRules(testEntry, `replace("article/(.*).svg"|"article/$1.png")`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -500,14 +552,16 @@ func TestRewriteReplaceCustom(t *testing.T) {
func TestRewriteReplaceTitleCustom(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `Ouch, a thistle`,
Content: `The replace_title rewrite rule should not modify the content.`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `The replace_title rewrite rule should not modify the content.`,
}
- Rewriter("https://example.org/article", testEntry, `replace_title("(?i)^a\\s*ti"|"Ouch, a this")`)
+ ApplyContentRewriteRules(testEntry, `replace_title("(?i)^a\\s*ti"|"Ouch, a this")`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -516,14 +570,16 @@ func TestRewriteReplaceTitleCustom(t *testing.T) {
func TestRewriteRemoveCustom(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Lorem Ipsum Super important info
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Lorem Ipsum I dont want to see thisSuper important info
`,
}
- Rewriter("https://example.org/article", testEntry, `remove(".spam, .ads:not(.keep)")`)
+ ApplyContentRewriteRules(testEntry, `remove(".spam, .ads:not(.keep)")`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -532,14 +588,16 @@ func TestRewriteRemoveCustom(t *testing.T) {
func TestRewriteAddCastopodEpisode(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://podcast.demo/@demo/episodes/test",
Title: `A title`,
Content: `
Episode Description`,
}
testEntry := &model.Entry{
+ URL: "https://podcast.demo/@demo/episodes/test",
Title: `A title`,
Content: `Episode Description`,
}
- Rewriter("https://podcast.demo/@demo/episodes/test", testEntry, `add_castopod_episode`)
+ ApplyContentRewriteRules(testEntry, `add_castopod_episode`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -548,14 +606,16 @@ func TestRewriteAddCastopodEpisode(t *testing.T) {
func TestRewriteBase64Decode(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `This is some base64 encoded content`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=`,
}
- Rewriter("https://example.org/article", testEntry, `base64_decode`)
+ ApplyContentRewriteRules(testEntry, `base64_decode`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -564,14 +624,16 @@ func TestRewriteBase64Decode(t *testing.T) {
func TestRewriteBase64DecodeInHTML(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Lorem Ipsum not valid base64This is some base64 encoded content
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Lorem Ipsum not valid base64VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`,
}
- Rewriter("https://example.org/article", testEntry, `base64_decode`)
+ ApplyContentRewriteRules(testEntry, `base64_decode`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -580,14 +642,16 @@ func TestRewriteBase64DecodeInHTML(t *testing.T) {
func TestRewriteBase64DecodeArgs(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Lorem IpsumThis is some base64 encoded content
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Lorem IpsumVGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`,
}
- Rewriter("https://example.org/article", testEntry, `base64_decode(".base64")`)
+ ApplyContentRewriteRules(testEntry, `base64_decode(".base64")`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -596,14 +660,16 @@ func TestRewriteBase64DecodeArgs(t *testing.T) {
func TestRewriteRemoveTables(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Test
Hello World!
Test
`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, `remove_tables`)
+ ApplyContentRewriteRules(testEntry, `remove_tables`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -612,14 +678,16 @@ func TestRewriteRemoveTables(t *testing.T) {
func TestRemoveClickbait(t *testing.T) {
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `This Is Amazing`,
Content: `Some description`,
}
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `THIS IS AMAZING`,
Content: `Some description`,
}
- Rewriter("https://example.org/article", testEntry, `remove_clickbait`)
+ ApplyContentRewriteRules(testEntry, `remove_clickbait`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -628,6 +696,7 @@ func TestRemoveClickbait(t *testing.T) {
func TestAddHackerNewsLinksUsingHack(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Article URL: https://example.org/article
Comments URL: https://news.ycombinator.com/item?id=37620043
@@ -636,13 +705,14 @@ func TestAddHackerNewsLinksUsingHack(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Article URL: https://example.org/article
Comments URL: https://news.ycombinator.com/item?id=37620043 Open with HACK
Points: 23
# Comments: 38
`,
}
- Rewriter("https://example.org/article", testEntry, `add_hn_links_using_hack`)
+ ApplyContentRewriteRules(testEntry, `add_hn_links_using_hack`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -651,6 +721,7 @@ func TestAddHackerNewsLinksUsingHack(t *testing.T) {
func TestAddHackerNewsLinksUsingOpener(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Article URL: https://example.org/article
Comments URL: https://news.ycombinator.com/item?id=37620043
@@ -659,13 +730,14 @@ func TestAddHackerNewsLinksUsingOpener(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Article URL: https://example.org/article
Comments URL: https://news.ycombinator.com/item?id=37620043 Open with Opener
Points: 23
# Comments: 38
`,
}
- Rewriter("https://example.org/article", testEntry, `add_hn_links_using_opener`)
+ ApplyContentRewriteRules(testEntry, `add_hn_links_using_opener`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -674,6 +746,7 @@ func TestAddHackerNewsLinksUsingOpener(t *testing.T) {
func TestAddImageTitle(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -687,6 +760,7 @@ func TestAddImageTitle(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `pouf
pouf
@@ -697,7 +771,7 @@ func TestAddImageTitle(t *testing.T) {
;"onerror=alert(1) a=;"
`,
}
- Rewriter("https://example.org/article", testEntry, `add_image_title`)
+ ApplyContentRewriteRules(testEntry, `add_image_title`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -706,6 +780,7 @@ func TestAddImageTitle(t *testing.T) {
func TestFixGhostCard(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -726,10 +801,11 @@ func TestFixGhostCard(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Example Article - Example`,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -738,15 +814,17 @@ func TestFixGhostCard(t *testing.T) {
func TestFixGhostCardNoCard(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Example Article - Example`,
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Example Article - Example`,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -755,6 +833,7 @@ func TestFixGhostCardNoCard(t *testing.T) {
func TestFixGhostCardInvalidCard(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
This card does not have the required fields
@@ -762,12 +841,13 @@ func TestFixGhostCardInvalidCard(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
This card does not have the required fields
`,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -776,6 +856,7 @@ func TestFixGhostCardInvalidCard(t *testing.T) {
func TestFixGhostCardMissingAuthor(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -791,10 +872,11 @@ func TestFixGhostCardMissingAuthor(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Example Article`,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -803,6 +885,7 @@ func TestFixGhostCardMissingAuthor(t *testing.T) {
func TestFixGhostCardDuplicatedAuthor(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -823,10 +906,11 @@ func TestFixGhostCardDuplicatedAuthor(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Example Article - Example`,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -835,6 +919,7 @@ func TestFixGhostCardDuplicatedAuthor(t *testing.T) {
func TestFixGhostCardMultiple(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -871,10 +956,11 @@ func TestFixGhostCardMultiple(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: ``,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -883,6 +969,7 @@ func TestFixGhostCardMultiple(t *testing.T) {
func TestFixGhostCardMultipleSplit(t *testing.T) {
testEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `
@@ -920,12 +1007,13 @@ func TestFixGhostCardMultipleSplit(t *testing.T) {
}
controlEntry := &model.Entry{
+ URL: "https://example.org/article",
Title: `A title`,
Content: `Example Article 1 - Example
This separates the two cards
Example Article 2 - Example`,
}
- Rewriter("https://example.org/article", testEntry, `fix_ghost_cards`)
+ ApplyContentRewriteRules(testEntry, `fix_ghost_cards`)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)