oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-06-27 16:36:00 +00:00
Code Wiki Activity

perf: cache the format of feeds

Browse source 
Detecting the format of a feed accounts for up to 30% of the time spent in
`parser.ParseFeed`. Cache the value once detected, as it'll never change.
This commit is contained in:
jvoisin 2025-02-04 16:12:31 +01:00
parent e342a4f143
commit c4a2afb7b7
12 changed files with 58 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

8
internal/database/migrations.go
View file

@ -1015,4 +1015,12 @@ var migrations = []func(tx *sql.Tx, driver string) error{
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx, _ string) (err error) {
sql := `
ALTER TABLE feeds ADD COLUMN format text default '';
ALTER TABLE feeds ADD COLUMN format_version text default '';
`
_, err = tx.Exec(sql)
return err
},
}

2
internal/model/feed.go
View file

@ -59,6 +59,8 @@ type Feed struct {
NtfyTopic string `json:"ntfy_topic"`
PushoverEnabled bool `json:"pushover_enabled,omitempty"`
PushoverPriority int `json:"pushover_priority,omitempty"`
Format string `json:"format"`
FormatVersion string `json:"format_version"`
// Non-persisted attributes
Category *Category `json:"category,omitempty"`

3
internal/reader/atom/atom_03_adapter.go
View file

@ -25,6 +25,9 @@ func NewAtom03Adapter(atomFeed *Atom03Feed) *Atom03Adapter {
func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed {
feed := new(model.Feed)
feed.Format = "atom"
feed.FormatVersion = "0.3"
// Populate the feed URL.
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
if feedURL != "" {

3
internal/reader/atom/atom_10_adapter.go
View file

@ -29,6 +29,9 @@ func NewAtom10Adapter(atomFeed *Atom10Feed) *Atom10Adapter {
func (a *Atom10Adapter) BuildFeed(baseURL string) *model.Feed {
feed := new(model.Feed)
feed.Format = "atom"
feed.FormatVersion = "10"
// Populate the feed URL.
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
if feedURL != "" {

13
internal/reader/handler/handler.go
View file

@ -275,7 +275,18 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
return localizedError
}
updatedFeed, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
var updatedFeed *model.Feed
var parseErr error
if originalFeed.Format != "" {
format, version := originalFeed.Format, originalFeed.FormatVersion
updatedFeed, parseErr = parser.ParseFeedWithFormat(responseHandler.EffectiveURL(), bytes.NewReader(responseBody), format, version)
if parseErr != nil { // Maybe the feed changed its format.
slog.Warn("Unable to parse feed with the given format", slog.String("feed_url", originalFeed.FeedURL), slog.String("format", format), slog.Any("error", parseErr))
updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
}
} else {
updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
}
if parseErr != nil {
localizedError := locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr)

1
internal/reader/json/adapter.go
View file

@ -29,6 +29,7 @@ func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {
Title: strings.TrimSpace(j.jsonFeed.Title),
FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
Format: "json",
}
if feed.FeedURL == "" {

13
internal/reader/parser/parser.go
View file

@ -16,10 +16,8 @@ import (
var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format")
// ParseFeed analyzes the input data and returns a normalized feed object.
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
r.Seek(0, io.SeekStart)
format, version := DetectFeedFormat(r)
// ParseFeedWithFormat returns a normalized feed object.
func ParseFeedWithFormat(baseURL string, r io.ReadSeeker, format, version string) (*model.Feed, error) {
switch format {
case FormatAtom:
r.Seek(0, io.SeekStart)
@ -37,3 +35,10 @@ func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
return nil, ErrFeedFormatNotDetected
}
}
// ParseFeed analyzes the input data and returns a normalized feed object.
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
r.Seek(0, io.SeekStart)
format, version := DetectFeedFormat(r)
return ParseFeedWithFormat(baseURL, r, format, version)
}

1
internal/reader/rdf/adapter.go
View file

@ -29,6 +29,7 @@ func (r *RDFAdapter) BuildFeed(baseURL string) *model.Feed {
Title: stripTags(r.rdf.Channel.Title),
FeedURL: strings.TrimSpace(baseURL),
SiteURL: strings.TrimSpace(r.rdf.Channel.Link),
Format: "rdf",
}
if feed.Title == "" {

1
internal/reader/rss/adapter.go
View file

@ -31,6 +31,7 @@ func (r *RSSAdapter) BuildFeed(baseURL string) *model.Feed {
Title: html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)),
FeedURL: strings.TrimSpace(baseURL),
SiteURL: strings.TrimSpace(r.rss.Channel.Link),
Format: "rss",
}
// Ensure the Site URL is absolute.

16
internal/storage/feed.go
View file

@ -248,10 +248,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
apprise_service_urls,
webhook_url,
disable_http2,
description
description,
format,
format_version
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)
RETURNING
id
`
@ -284,6 +286,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.WebhookURL,
feed.DisableHTTP2,
feed.Description,
feed.Format,
feed.FormatVersion,
).Scan(&feed.ID)
if err != nil {
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
@ -363,9 +367,11 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
ntfy_priority=$32,
ntfy_topic=$33,
pushover_enabled=$34,
pushover_priority=$35
pushover_priority=$35,
format=$36,
format_version=$37
WHERE
id=$36 AND user_id=$37
id=$38 AND user_id=$39
`
_, err = s.db.Exec(query,
feed.FeedURL,
@ -403,6 +409,8 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.NtfyTopic,
feed.PushoverEnabled,
feed.PushoverPriority,
feed.Format,
feed.FormatVersion,
feed.ID,
feed.UserID,
)

6
internal/storage/feed_query_builder.go
View file

@ -171,7 +171,9 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
f.ntfy_priority,
f.ntfy_topic,
f.pushover_enabled,
f.pushover_priority
f.pushover_priority,
f.format,
f.format_version
FROM
feeds f
LEFT JOIN
@ -246,6 +248,8 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
&feed.NtfyTopic,
&feed.PushoverEnabled,
&feed.PushoverPriority,
&feed.Format,
&feed.FormatVersion,
)
if err != nil {

2
internal/ui/static/css/common.css
View file

@ -75,7 +75,7 @@ a:hover {
padding: var(--padding-size);
position: absolute;
transition: translate 0.3s;
translate: -50% calc(-100% - calc(var(--padding-size) * 2) - calc(var(--border-size) * 2));
translate: -50% calc(-100% - var(--padding-size) * 2 - var(--border-size) * 2);
}
.skip-to-content-link:focus {