oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-07-02 16:38:37 +00:00
Code Wiki Activity

Add Prometheus exporter

Browse source
This commit is contained in:
Frédéric Guillot 2020-09-27 16:01:06 -07:00 • committed by Frédéric Guillot
parent 16b7b3bc3e
commit c394a61a4e
61 changed files with 809 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

2
reader/feed/handler.go
View file

@ -81,7 +81,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
return subscription, nil
}
// RefreshFeed fetch and update a feed if necessary.
// RefreshFeed refreshes a feed.
func (h *Handler) RefreshFeed(userID, feedID int64) error {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:RefreshFeed] feedID=%d", feedID))
userLanguage := h.store.UserLanguage(userID)

35
reader/processor/processor.go
View file

@ -5,7 +5,11 @@
package processor
import (
"time"
"miniflux.app/config"
"miniflux.app/logger"
"miniflux.app/metric"
"miniflux.app/model"
"miniflux.app/reader/rewrite"
"miniflux.app/reader/sanitizer"
@ -20,9 +24,19 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
if feed.Crawler {
if !store.EntryURLExists(feed.ID, entry.URL) {
content, err := scraper.Fetch(entry.URL, feed.ScraperRules, feed.UserAgent)
if err != nil {
logger.Error(`[Filter] Unable to crawl this entry: %q => %v`, entry.URL, err)
startTime := time.Now()
content, scraperErr := scraper.Fetch(entry.URL, feed.ScraperRules, feed.UserAgent)
if config.Opts.HasMetricsCollector() {
status := "success"
if scraperErr != nil {
status = "error"
}
metric.ScraperRequestDuration.WithLabelValues(status).Observe(time.Since(startTime).Seconds())
}
if scraperErr != nil {
logger.Error(`[Filter] Unable to crawl this entry: %q => %v`, entry.URL, scraperErr)
} else if content != "" {
// We replace the entry content only if the scraper doesn't return any error.
entry.Content = content
@ -39,9 +53,18 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(entry *model.Entry) error {
content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules, entry.Feed.UserAgent)
if err != nil {
return err
startTime := time.Now()
content, scraperErr := scraper.Fetch(entry.URL, entry.Feed.ScraperRules, entry.Feed.UserAgent)
if config.Opts.HasMetricsCollector() {
status := "success"
if scraperErr != nil {
status = "error"
}
metric.ScraperRequestDuration.WithLabelValues(status).Observe(time.Since(startTime).Seconds())
}
if scraperErr != nil {
return scraperErr
}
content = rewrite.Rewriter(entry.URL, content, entry.Feed.RewriteRules)

4
reader/scraper/scraper.go
View file

@ -35,7 +35,7 @@ func Fetch(websiteURL, rules, userAgent string) (string, error) {
return "", errors.New("scraper: unable to download web page")
}
if !isWhitelistedContentType(response.ContentType) {
if !isAllowedContentType(response.ContentType) {
return "", fmt.Errorf("scraper: this resource is not a HTML document (%s)", response.ContentType)
}
@ -95,7 +95,7 @@ func getPredefinedScraperRules(websiteURL string) string {
return ""
}
func isWhitelistedContentType(contentType string) bool {
func isAllowedContentType(contentType string) bool {
contentType = strings.ToLower(contentType)
return strings.HasPrefix(contentType, "text/html") ||
strings.HasPrefix(contentType, "application/xhtml+xml")

10
reader/scraper/scraper_test.go
View file

@ -39,7 +39,7 @@ func TestWhitelistedContentTypes(t *testing.T) {
}
for inputValue, expectedResult := range scenarios {
actualResult := isWhitelistedContentType(inputValue)
actualResult := isAllowedContentType(inputValue)
if actualResult != expectedResult {
t.Errorf(`Unexpected result for content type whitelist, got "%v" instead of "%v"`, actualResult, expectedResult)
}
@ -47,10 +47,10 @@ func TestWhitelistedContentTypes(t *testing.T) {
}
func TestSelectorRules(t *testing.T) {
var ruleTestCases = map[string]string {
"img.html": "article > img",
"iframe.html": "article > iframe",
"p.html": "article > p",
var ruleTestCases = map[string]string{
"img.html": "article > img",
"iframe.html": "article > iframe",
"p.html": "article > p",
}
for filename, rule := range ruleTestCases {