oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-09-15 18:57:04 +00:00
Code Wiki Activity

feat: multiple database support

Browse source
This commit is contained in:
haras 2025-09-14 10:05:57 +02:00
parent 2c3ed81797
commit fd9072d187
7 changed files with 1515 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

14
internal/cli/cli.go
View file

@ -168,7 +168,13 @@ func Parse() {
printErrorAndExit(fmt.Errorf("unable to generate javascript bundle: %v", err))
}
kind, err := database.DetectKind(config.Opts.DatabaseURL())
if err != nil {
printErrorAndExit(fmt.Errorf("unable to parse database kind: %v", err))
}
db, err := database.NewConnectionPool(
kind,
config.Opts.DatabaseURL(),
config.Opts.DatabaseMinConns(),
config.Opts.DatabaseMaxConns(),
@ -179,14 +185,14 @@ func Parse() {
}
defer db.Close()
store := storage.NewStorage(db)
store := storage.NewStorage(kind, db)
if err := store.Ping(); err != nil {
printErrorAndExit(err)
}
if flagMigrate {
if err := database.Migrate(db); err != nil {
if err := database.Migrate(kind, db); err != nil {
printErrorAndExit(err)
}
return
@ -228,12 +234,12 @@ func Parse() {
// Run migrations and start the daemon.
if config.Opts.RunMigrations() {
if err := database.Migrate(db); err != nil {
if err := database.Migrate(kind, db); err != nil {
printErrorAndExit(err)
}
}
if err := database.IsSchemaUpToDate(db); err != nil {
if err := database.IsSchemaUpToDate(kind, db); err != nil {
printErrorAndExit(err)
}

159
internal/database/migrations.go → internal/database/cockroach.go
View file

@ -9,10 +9,10 @@ import (
"miniflux.app/v2/internal/crypto"
)
var schemaVersion = len(migrations)
var cockroachSchemaVersion = len(cockroachMigrations)
// Order is important. Add new migrations at the end of the list.
var migrations = [...]func(tx *sql.Tx) error{
var cockroachMigrations = []Migration{
func(tx *sql.Tx) (err error) {
sql := `
CREATE TABLE schema_version (
@ -123,13 +123,7 @@ var migrations = [...]func(tx *sql.Tx) error{
return err
},
func(tx *sql.Tx) (err error) {
sql := `
CREATE EXTENSION IF NOT EXISTS hstore;
ALTER TABLE users ADD COLUMN extra hstore;
CREATE INDEX users_extra_idx ON users using gin(extra);
`
_, err = tx.Exec(sql)
return err
return nil
},
func(tx *sql.Tx) (err error) {
sql := `
@ -263,11 +257,7 @@ var migrations = [...]func(tx *sql.Tx) error{
return err
},
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE user_sessions ALTER COLUMN ip SET DATA TYPE inet using ip::inet;
`
_, err = tx.Exec(sql)
return err
return nil
},
func(tx *sql.Tx) (err error) {
sql := `
@ -281,7 +271,13 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE entries ADD COLUMN document_vectors tsvector;
UPDATE entries SET document_vectors = to_tsvector(substring(title || ' ' || coalesce(content, '') for 1000000));
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
UPDATE entries SET document_vectors = to_tsvector(substring(title || ' ' || title || ' ' || coalesce(content, '') for 1000000));
CREATE INDEX document_vectors_idx ON entries USING gin(document_vectors);
`
_, err = tx.Exec(sql)
@ -292,16 +288,6 @@ var migrations = [...]func(tx *sql.Tx) error{
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
UPDATE
entries
SET
document_vectors = setweight(to_tsvector(substring(coalesce(title, '') for 1000000)), 'A') || setweight(to_tsvector(substring(coalesce(content, '') for 1000000)), 'B')
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `ALTER TABLE users ADD COLUMN keyboard_shortcuts boolean default 't'`
_, err = tx.Exec(sql)
@ -325,6 +311,12 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE entries ADD COLUMN changed_at timestamp with time zone;
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
UPDATE entries SET changed_at = published_at;
ALTER TABLE entries ALTER COLUMN changed_at SET not null;
`
@ -350,19 +342,31 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE entries ADD COLUMN share_code text not null default '';
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
CREATE UNIQUE INDEX entries_share_code_idx ON entries USING btree(share_code) WHERE share_code <> '';
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `CREATE INDEX enclosures_user_entry_url_idx ON enclosures(user_id, entry_id, md5(url))`
sql := `CREATE INDEX enclosures_user_entry_url_idx ON enclosures(user_id, entry_id, url);`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE feeds ADD COLUMN next_check_at timestamp with time zone default now();
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
CREATE INDEX entries_user_feed_idx ON entries (user_id, feed_id);
`
_, err = tx.Exec(sql)
@ -430,6 +434,12 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE entries ADD COLUMN created_at timestamp with time zone not null default now();
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
UPDATE entries SET created_at = published_at;
`
_, err = tx.Exec(sql)
@ -442,62 +452,9 @@ var migrations = [...]func(tx *sql.Tx) error{
ADD column google_id text not null default '',
ADD column openid_connect_id text not null default ''
`)
if err != nil {
return err
}
_, err = tx.Exec(`
DECLARE my_cursor CURSOR FOR
SELECT
id,
COALESCE(extra->'custom_css', '') as custom_css,
COALESCE(extra->'google_id', '') as google_id,
COALESCE(extra->'oidc_id', '') as oidc_id
FROM users
FOR UPDATE
`)
if err != nil {
return err
}
defer tx.Exec("CLOSE my_cursor")
for {
var (
userID int64
customStylesheet string
googleID string
oidcID string
)
if err := tx.QueryRow(`FETCH NEXT FROM my_cursor`).Scan(&userID, &customStylesheet, &googleID, &oidcID); err != nil {
if err == sql.ErrNoRows {
break
}
return err
}
_, err := tx.Exec(
`UPDATE
users
SET
stylesheet=$2,
google_id=$3,
openid_connect_id=$4
WHERE
id=$1
`,
userID, customStylesheet, googleID, oidcID)
if err != nil {
return err
}
}
return err
},
func(tx *sql.Tx) (err error) {
if _, err = tx.Exec(`ALTER TABLE users DROP COLUMN extra;`); err != nil {
return err
}
_, err = tx.Exec(`
CREATE UNIQUE INDEX users_google_id_idx ON users(google_id) WHERE google_id <> '';
CREATE UNIQUE INDEX users_openid_connect_id_idx ON users(openid_connect_id) WHERE openid_connect_id <> '';
@ -506,7 +463,7 @@ var migrations = [...]func(tx *sql.Tx) error{
},
func(tx *sql.Tx) (err error) {
_, err = tx.Exec(`
CREATE INDEX entries_feed_url_idx ON entries(feed_id, url) WHERE length(url) < 2000;
CREATE INDEX entries_feed_url_idx ON entries(feed_id, url);
CREATE INDEX entries_user_status_feed_idx ON entries(user_id, status, feed_id);
CREATE INDEX entries_user_status_changed_idx ON entries(user_id, status, changed_at);
`)
@ -531,6 +488,12 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
CREATE TYPE webapp_display_mode AS enum('fullscreen', 'standalone', 'minimal-ui', 'browser');
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE users ADD COLUMN display_mode webapp_display_mode default 'standalone';
`
_, err = tx.Exec(sql)
@ -566,6 +529,12 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
CREATE TYPE entry_sorting_order AS enum('published_at', 'created_at');
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE users ADD COLUMN entry_order entry_sorting_order default 'published_at';
`
_, err = tx.Exec(sql)
@ -659,10 +628,21 @@ var migrations = [...]func(tx *sql.Tx) error{
},
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE users RENAME double_tap TO gesture_nav;
ALTER TABLE users
ALTER COLUMN gesture_nav SET DATA TYPE text using case when gesture_nav = true then 'tap' when gesture_nav = false then 'none' end,
ALTER COLUMN gesture_nav SET default 'tap';
ALTER TABLE users ADD COLUMN gesture_nav text DEFAULT 'tap' NOT NULL;
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
UPDATE users SET gesture_nav = CASE WHEN double_tap THEN 'tap' ELSE 'none' END;
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE users DROP COLUMN double_tap;
`
_, err = tx.Exec(sql)
return err
@ -710,7 +690,7 @@ var migrations = [...]func(tx *sql.Tx) error{
}
// Create unique index
_, err = tx.Exec(`CREATE UNIQUE INDEX enclosures_user_entry_url_unique_idx ON enclosures(user_id, entry_id, md5(url))`)
_, err = tx.Exec(`CREATE UNIQUE INDEX enclosures_user_entry_url_unique_idx ON enclosures(user_id, entry_id, url);`)
if err != nil {
return err
}
@ -1045,6 +1025,12 @@ var migrations = [...]func(tx *sql.Tx) error{
func(tx *sql.Tx) (err error) {
sql := `
ALTER TABLE icons ADD COLUMN external_id text default '';
`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `
CREATE UNIQUE INDEX icons_external_id_idx ON icons USING btree(external_id) WHERE external_id <> '';
`
_, err = tx.Exec(sql)
@ -1079,7 +1065,6 @@ var migrations = [...]func(tx *sql.Tx) error{
UPDATE icons SET external_id = $1 WHERE id = $2
`,
crypto.GenerateRandomStringHex(20), id)
if err != nil {
return err
}
@ -1160,7 +1145,7 @@ var migrations = [...]func(tx *sql.Tx) error{
},
// This migration replaces deprecated timezones by their equivalent on Debian Trixie.
func(tx *sql.Tx) (err error) {
var deprecatedTimeZoneMap = map[string]string{
deprecatedTimeZoneMap := map[string]string{
// Africa
"Africa/Asmera": "Africa/Asmara",

68
internal/database/database.go
View file

@ -7,13 +7,60 @@ import (
"database/sql"
"fmt"
"log/slog"
"strings"
"time"
)
type DBKind int
const (
DBKindPostgres DBKind = iota
DBKindCockroach
)
var dbKindProto = map[DBKind]string{
DBKindPostgres: "postgresql",
DBKindCockroach: "cockroachdb",
}
var dbKindDriver = map[DBKind]string{
DBKindPostgres: "postgres",
DBKindCockroach: "postgres",
}
func DetectKind(conn string) (DBKind, error) {
switch {
case strings.HasPrefix(conn, "postgres://"),
strings.HasPrefix(conn, "postgresql://"):
return DBKindPostgres, nil
case strings.HasPrefix(conn, "cockroach://"),
strings.HasPrefix(conn, "cockroachdb://"):
return DBKindCockroach, nil
default:
return 0, fmt.Errorf("unknown db kind in conn string: %q", conn)
}
}
type Migration func(*sql.Tx) error
var dbKindMigrations = map[DBKind][]Migration{
DBKindPostgres: postgresMigrations,
DBKindCockroach: cockroachMigrations,
}
var dbKindSchemaVersion = map[DBKind]int{
DBKindPostgres: postgresSchemaVersion,
DBKindCockroach: cockroachSchemaVersion,
}
// Migrate executes database migrations.
func Migrate(db *sql.DB) error {
func Migrate(kind DBKind, db *sql.DB) error {
var currentVersion int
db.QueryRow(`SELECT version FROM schema_version`).Scan(&currentVersion)
migrations := dbKindMigrations[kind]
schemaVersion := dbKindSchemaVersion[kind]
slog.Info("Running database migrations",
slog.Int("current_version", currentVersion),
slog.Int("latest_version", schemaVersion),
@ -51,7 +98,9 @@ func Migrate(db *sql.DB) error {
}
// IsSchemaUpToDate checks if the database schema is up to date.
func IsSchemaUpToDate(db *sql.DB) error {
func IsSchemaUpToDate(kind DBKind, db *sql.DB) error {
schemaVersion := dbKindSchemaVersion[kind]
var currentVersion int
db.QueryRow(`SELECT version FROM schema_version`).Scan(&currentVersion)
if currentVersion < schemaVersion {
@ -59,3 +108,18 @@ func IsSchemaUpToDate(db *sql.DB) error {
}
return nil
}
func NewConnectionPool(kind DBKind, dsn string, minConnections, maxConnections int, connectionLifetime time.Duration) (*sql.DB, error) {
driver := dbKindDriver[kind]
db, err := sql.Open(driver, dsn)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(maxConnections)
db.SetMaxIdleConns(minConnections)
db.SetConnMaxLifetime(connectionLifetime)
return db, nil
}

1341
internal/database/postgresql.go
View file

File diff suppressed because it is too large Load diff

14
internal/storage/enclosure.go
View file

@ -8,6 +8,7 @@ import (
"fmt"
"strings"
"miniflux.app/v2/internal/database"
"miniflux.app/v2/internal/model"
"github.com/lib/pq"
@ -49,7 +50,6 @@ func (s *Storage) GetEnclosures(entryID int64) (model.EnclosureList, error) {
&enclosure.MimeType,
&enclosure.MediaProgression,
)
if err != nil {
return nil, fmt.Errorf(`store: unable to fetch enclosure row: %v`, err)
}
@ -150,15 +150,20 @@ func (s *Storage) createEnclosure(tx *sql.Tx, enclosure *model.Enclosure) error
return nil
}
query := `
urlPart := "md5(url)"
if s.kind == database.DBKindCockroach {
urlPart = "url"
}
query := fmt.Sprintf(`
INSERT INTO enclosures
(url, size, mime_type, entry_id, user_id, media_progression)
VALUES
($1, $2, $3, $4, $5, $6)
ON CONFLICT (user_id, entry_id, md5(url)) DO NOTHING
ON CONFLICT (user_id, entry_id, %s) DO NOTHING
RETURNING
id
`
`, urlPart)
if err := tx.QueryRow(
query,
enclosureURL,
@ -226,7 +231,6 @@ func (s *Storage) UpdateEnclosure(enclosure *model.Enclosure) error {
enclosure.MediaProgression,
enclosure.ID,
)
if err != nil {
return fmt.Errorf(`store: unable to update enclosure #%d : %v`, enclosure.ID, err)
}

31
internal/storage/entry.go
View file

@ -11,6 +11,7 @@ import (
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/database"
"miniflux.app/v2/internal/model"
"github.com/lib/pq"
@ -70,17 +71,21 @@ func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder {
// UpdateEntryTitleAndContent updates entry title and content.
func (s *Storage) UpdateEntryTitleAndContent(entry *model.Entry) error {
truncatedTitle, truncatedContent := truncateTitleAndContentForTSVectorField(entry.Title, entry.Content)
query := `
setweightPart := "setweight(to_tsvector($4), 'A') || setweight(to_tsvector($5), 'B')"
if s.kind == database.DBKindCockroach {
setweightPart = "to_tsvector(substring($4 || ' ' || $4 || ' ' || coalesce($5, '') for 1000000))"
}
query := fmt.Sprintf(`
UPDATE
entries
SET
title=$1,
content=$2,
reading_time=$3,
document_vectors = setweight(to_tsvector($4), 'A') || setweight(to_tsvector($5), 'B')
document_vectors = %s
WHERE
id=$6 AND user_id=$7
`
`, setweightPart)
if _, err := s.db.Exec(
query,
@ -100,7 +105,11 @@ func (s *Storage) UpdateEntryTitleAndContent(entry *model.Entry) error {
// createEntry add a new entry.
func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
truncatedTitle, truncatedContent := truncateTitleAndContentForTSVectorField(entry.Title, entry.Content)
query := `
setweightPart := "setweight(to_tsvector($11), 'A') || setweight(to_tsvector($12), 'B')"
if s.kind == database.DBKindCockroach {
setweightPart = "to_tsvector(substring($11 || ' ' || $11 || ' ' || coalesce($12, '') for 1000000))"
}
query := fmt.Sprintf(`
INSERT INTO entries
(
title,
@ -130,12 +139,12 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
$9,
$10,
now(),
setweight(to_tsvector($11), 'A') || setweight(to_tsvector($12), 'B'),
%s,
$13
)
RETURNING
id, status, created_at, changed_at
`
`, setweightPart)
err := tx.QueryRow(
query,
entry.Title,
@ -178,7 +187,11 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
// it default to time.Now() which could change the order of items on the history page.
func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
truncatedTitle, truncatedContent := truncateTitleAndContentForTSVectorField(entry.Title, entry.Content)
query := `
setweightPart := "setweight(to_tsvector($7), 'A') || setweight(to_tsvector($8), 'B')"
if s.kind == database.DBKindCockroach {
setweightPart = "to_tsvector(substring($7 || ' ' || $7 || ' ' || coalesce($8, '') for 1000000))"
}
query := fmt.Sprintf(`
UPDATE
entries
SET
@ -188,13 +201,13 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
content=$4,
author=$5,
reading_time=$6,
document_vectors = setweight(to_tsvector($7), 'A') || setweight(to_tsvector($8), 'B'),
document_vectors = %s,
tags=$12
WHERE
user_id=$9 AND feed_id=$10 AND hash=$11
RETURNING
id
`
`, setweightPart)
err := tx.QueryRow(
query,
entry.Title,

9
internal/storage/storage.go
View file

@ -7,16 +7,19 @@ import (
"context"
"database/sql"
"time"
"miniflux.app/v2/internal/database"
)
// Storage handles all operations related to the database.
type Storage struct {
db *sql.DB
kind database.DBKind
db *sql.DB
}
// NewStorage returns a new Storage.
func NewStorage(db *sql.DB) *Storage {
return &Storage{db}
func NewStorage(kind database.DBKind, db *sql.DB) *Storage {
return &Storage{kind, db}
}
// DatabaseVersion returns the version of the database which is in use.