mirror of
https://github.com/miniflux/v2.git
synced 2025-06-27 16:36:00 +00:00
perf: cache the format of feeds
Detecting the format of a feed accounts for up to 30% of the time spent in `parser.ParseFeed`. Cache the value once detected, as it'll never change.
This commit is contained in:
parent
e342a4f143
commit
c4a2afb7b7
12 changed files with 58 additions and 11 deletions
|
@ -1015,4 +1015,12 @@ var migrations = []func(tx *sql.Tx, driver string) error{
|
||||||
_, err = tx.Exec(sql)
|
_, err = tx.Exec(sql)
|
||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
|
func(tx *sql.Tx, _ string) (err error) {
|
||||||
|
sql := `
|
||||||
|
ALTER TABLE feeds ADD COLUMN format text default '';
|
||||||
|
ALTER TABLE feeds ADD COLUMN format_version text default '';
|
||||||
|
`
|
||||||
|
_, err = tx.Exec(sql)
|
||||||
|
return err
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -59,6 +59,8 @@ type Feed struct {
|
||||||
NtfyTopic string `json:"ntfy_topic"`
|
NtfyTopic string `json:"ntfy_topic"`
|
||||||
PushoverEnabled bool `json:"pushover_enabled,omitempty"`
|
PushoverEnabled bool `json:"pushover_enabled,omitempty"`
|
||||||
PushoverPriority int `json:"pushover_priority,omitempty"`
|
PushoverPriority int `json:"pushover_priority,omitempty"`
|
||||||
|
Format string `json:"format"`
|
||||||
|
FormatVersion string `json:"format_version"`
|
||||||
|
|
||||||
// Non-persisted attributes
|
// Non-persisted attributes
|
||||||
Category *Category `json:"category,omitempty"`
|
Category *Category `json:"category,omitempty"`
|
||||||
|
|
|
@ -25,6 +25,9 @@ func NewAtom03Adapter(atomFeed *Atom03Feed) *Atom03Adapter {
|
||||||
func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed {
|
func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed {
|
||||||
feed := new(model.Feed)
|
feed := new(model.Feed)
|
||||||
|
|
||||||
|
feed.Format = "atom"
|
||||||
|
feed.FormatVersion = "0.3"
|
||||||
|
|
||||||
// Populate the feed URL.
|
// Populate the feed URL.
|
||||||
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
|
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
|
||||||
if feedURL != "" {
|
if feedURL != "" {
|
||||||
|
|
|
@ -29,6 +29,9 @@ func NewAtom10Adapter(atomFeed *Atom10Feed) *Atom10Adapter {
|
||||||
func (a *Atom10Adapter) BuildFeed(baseURL string) *model.Feed {
|
func (a *Atom10Adapter) BuildFeed(baseURL string) *model.Feed {
|
||||||
feed := new(model.Feed)
|
feed := new(model.Feed)
|
||||||
|
|
||||||
|
feed.Format = "atom"
|
||||||
|
feed.FormatVersion = "10"
|
||||||
|
|
||||||
// Populate the feed URL.
|
// Populate the feed URL.
|
||||||
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
|
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
|
||||||
if feedURL != "" {
|
if feedURL != "" {
|
||||||
|
|
|
@ -275,7 +275,18 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
return localizedError
|
return localizedError
|
||||||
}
|
}
|
||||||
|
|
||||||
updatedFeed, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
|
var updatedFeed *model.Feed
|
||||||
|
var parseErr error
|
||||||
|
if originalFeed.Format != "" {
|
||||||
|
format, version := originalFeed.Format, originalFeed.FormatVersion
|
||||||
|
updatedFeed, parseErr = parser.ParseFeedWithFormat(responseHandler.EffectiveURL(), bytes.NewReader(responseBody), format, version)
|
||||||
|
if parseErr != nil { // Maybe the feed changed its format.
|
||||||
|
slog.Warn("Unable to parse feed with the given format", slog.String("feed_url", originalFeed.FeedURL), slog.String("format", format), slog.Any("error", parseErr))
|
||||||
|
updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
|
||||||
|
}
|
||||||
if parseErr != nil {
|
if parseErr != nil {
|
||||||
localizedError := locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr)
|
localizedError := locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr)
|
||||||
|
|
||||||
|
|
|
@ -29,6 +29,7 @@ func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {
|
||||||
Title: strings.TrimSpace(j.jsonFeed.Title),
|
Title: strings.TrimSpace(j.jsonFeed.Title),
|
||||||
FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
|
FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
|
||||||
SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
|
SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
|
||||||
|
Format: "json",
|
||||||
}
|
}
|
||||||
|
|
||||||
if feed.FeedURL == "" {
|
if feed.FeedURL == "" {
|
||||||
|
|
|
@ -16,10 +16,8 @@ import (
|
||||||
|
|
||||||
var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format")
|
var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format")
|
||||||
|
|
||||||
// ParseFeed analyzes the input data and returns a normalized feed object.
|
// ParseFeedWithFormat returns a normalized feed object.
|
||||||
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
|
func ParseFeedWithFormat(baseURL string, r io.ReadSeeker, format, version string) (*model.Feed, error) {
|
||||||
r.Seek(0, io.SeekStart)
|
|
||||||
format, version := DetectFeedFormat(r)
|
|
||||||
switch format {
|
switch format {
|
||||||
case FormatAtom:
|
case FormatAtom:
|
||||||
r.Seek(0, io.SeekStart)
|
r.Seek(0, io.SeekStart)
|
||||||
|
@ -37,3 +35,10 @@ func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
|
||||||
return nil, ErrFeedFormatNotDetected
|
return nil, ErrFeedFormatNotDetected
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseFeed analyzes the input data and returns a normalized feed object.
|
||||||
|
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
|
||||||
|
r.Seek(0, io.SeekStart)
|
||||||
|
format, version := DetectFeedFormat(r)
|
||||||
|
return ParseFeedWithFormat(baseURL, r, format, version)
|
||||||
|
}
|
||||||
|
|
|
@ -29,6 +29,7 @@ func (r *RDFAdapter) BuildFeed(baseURL string) *model.Feed {
|
||||||
Title: stripTags(r.rdf.Channel.Title),
|
Title: stripTags(r.rdf.Channel.Title),
|
||||||
FeedURL: strings.TrimSpace(baseURL),
|
FeedURL: strings.TrimSpace(baseURL),
|
||||||
SiteURL: strings.TrimSpace(r.rdf.Channel.Link),
|
SiteURL: strings.TrimSpace(r.rdf.Channel.Link),
|
||||||
|
Format: "rdf",
|
||||||
}
|
}
|
||||||
|
|
||||||
if feed.Title == "" {
|
if feed.Title == "" {
|
||||||
|
|
|
@ -31,6 +31,7 @@ func (r *RSSAdapter) BuildFeed(baseURL string) *model.Feed {
|
||||||
Title: html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)),
|
Title: html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)),
|
||||||
FeedURL: strings.TrimSpace(baseURL),
|
FeedURL: strings.TrimSpace(baseURL),
|
||||||
SiteURL: strings.TrimSpace(r.rss.Channel.Link),
|
SiteURL: strings.TrimSpace(r.rss.Channel.Link),
|
||||||
|
Format: "rss",
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure the Site URL is absolute.
|
// Ensure the Site URL is absolute.
|
||||||
|
|
|
@ -248,10 +248,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
|
||||||
apprise_service_urls,
|
apprise_service_urls,
|
||||||
webhook_url,
|
webhook_url,
|
||||||
disable_http2,
|
disable_http2,
|
||||||
description
|
description,
|
||||||
|
format,
|
||||||
|
format_version
|
||||||
)
|
)
|
||||||
VALUES
|
VALUES
|
||||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
|
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)
|
||||||
RETURNING
|
RETURNING
|
||||||
id
|
id
|
||||||
`
|
`
|
||||||
|
@ -284,6 +286,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
|
||||||
feed.WebhookURL,
|
feed.WebhookURL,
|
||||||
feed.DisableHTTP2,
|
feed.DisableHTTP2,
|
||||||
feed.Description,
|
feed.Description,
|
||||||
|
feed.Format,
|
||||||
|
feed.FormatVersion,
|
||||||
).Scan(&feed.ID)
|
).Scan(&feed.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
|
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
|
||||||
|
@ -363,9 +367,11 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
|
||||||
ntfy_priority=$32,
|
ntfy_priority=$32,
|
||||||
ntfy_topic=$33,
|
ntfy_topic=$33,
|
||||||
pushover_enabled=$34,
|
pushover_enabled=$34,
|
||||||
pushover_priority=$35
|
pushover_priority=$35,
|
||||||
|
format=$36,
|
||||||
|
format_version=$37
|
||||||
WHERE
|
WHERE
|
||||||
id=$36 AND user_id=$37
|
id=$38 AND user_id=$39
|
||||||
`
|
`
|
||||||
_, err = s.db.Exec(query,
|
_, err = s.db.Exec(query,
|
||||||
feed.FeedURL,
|
feed.FeedURL,
|
||||||
|
@ -403,6 +409,8 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
|
||||||
feed.NtfyTopic,
|
feed.NtfyTopic,
|
||||||
feed.PushoverEnabled,
|
feed.PushoverEnabled,
|
||||||
feed.PushoverPriority,
|
feed.PushoverPriority,
|
||||||
|
feed.Format,
|
||||||
|
feed.FormatVersion,
|
||||||
feed.ID,
|
feed.ID,
|
||||||
feed.UserID,
|
feed.UserID,
|
||||||
)
|
)
|
||||||
|
|
|
@ -171,7 +171,9 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
|
||||||
f.ntfy_priority,
|
f.ntfy_priority,
|
||||||
f.ntfy_topic,
|
f.ntfy_topic,
|
||||||
f.pushover_enabled,
|
f.pushover_enabled,
|
||||||
f.pushover_priority
|
f.pushover_priority,
|
||||||
|
f.format,
|
||||||
|
f.format_version
|
||||||
FROM
|
FROM
|
||||||
feeds f
|
feeds f
|
||||||
LEFT JOIN
|
LEFT JOIN
|
||||||
|
@ -246,6 +248,8 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
|
||||||
&feed.NtfyTopic,
|
&feed.NtfyTopic,
|
||||||
&feed.PushoverEnabled,
|
&feed.PushoverEnabled,
|
||||||
&feed.PushoverPriority,
|
&feed.PushoverPriority,
|
||||||
|
&feed.Format,
|
||||||
|
&feed.FormatVersion,
|
||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -75,7 +75,7 @@ a:hover {
|
||||||
padding: var(--padding-size);
|
padding: var(--padding-size);
|
||||||
position: absolute;
|
position: absolute;
|
||||||
transition: translate 0.3s;
|
transition: translate 0.3s;
|
||||||
translate: -50% calc(-100% - calc(var(--padding-size) * 2) - calc(var(--border-size) * 2));
|
translate: -50% calc(-100% - var(--padding-size) * 2 - var(--border-size) * 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
.skip-to-content-link:focus {
|
.skip-to-content-link:focus {
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue