diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b2eb4d95..d6d46511 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -5,9 +5,17 @@ permissions: read-all on: push: branches: [ main ] + paths: + - '**.js' + - '**.go' + - '!**_test.go' pull_request: # The branches below must be a subset of the branches above branches: [ main ] + paths: + - '**.js' + - '**.go' + - '!**_test.go' schedule: - cron: '45 22 * * 3' diff --git a/.github/workflows/debian_packages.yml b/.github/workflows/debian_packages.yml index babdeb63..20edc9c7 100644 --- a/.github/workflows/debian_packages.yml +++ b/.github/workflows/debian_packages.yml @@ -5,11 +5,11 @@ on: push: tags: - '[0-9]+.[0-9]+.[0-9]+' - pull_request: - branches: [ main ] + schedule: + - cron: '0 0 * * 1,4' # Runs at 00:00 UTC on Monday and Thursday jobs: test-packages: - if: github.event.pull_request + if: github.event_name == 'schedule' name: Test Packages runs-on: ubuntu-latest steps: @@ -30,7 +30,7 @@ jobs: - name: List generated files run: ls -l *.deb build-packages-manually: - if: github.event_name != 'pull_request' && github.event_name != 'push' + if: github.event_name == 'workflow_dispatch' name: Build Packages Manually runs-on: ubuntu-latest steps: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6d272101..e2754467 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,8 @@ on: - '[0-9]+.[0-9]+.[0-9]+' pull_request: branches: [ main ] + paths: + - 'packaging/docker/**' jobs: docker-images: name: Docker Images diff --git a/.github/workflows/rpm_packages.yml b/.github/workflows/rpm_packages.yml index e7c813fb..caea482b 100644 --- a/.github/workflows/rpm_packages.yml +++ b/.github/workflows/rpm_packages.yml @@ -5,11 +5,11 @@ on: push: tags: - '[0-9]+.[0-9]+.[0-9]+' - pull_request: - branches: [ main ] + schedule: + - cron: '0 0 * * 1,4' # Runs at 00:00 UTC on Monday and Thursday jobs: test-package: - if: github.event.pull_request + if: github.event_name == 'schedule' name: Test Packages runs-on: ubuntu-latest steps: @@ -21,7 +21,7 @@ jobs: - name: List generated files run: ls -l *.rpm build-package-manually: - if: github.event_name != 'pull_request' && github.event_name != 'push' + if: github.event_name == 'workflow_dispatch' name: Build Packages Manually runs-on: ubuntu-latest steps: diff --git a/go.mod b/go.mod index 38341a40..50d4cfa6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ module miniflux.app/v2 require ( github.com/PuerkitoBio/goquery v1.10.0 - github.com/abadojack/whatlanggo v1.0.1 github.com/andybalholm/brotli v1.1.1 github.com/coreos/go-oidc/v3 v3.11.0 github.com/go-webauthn/webauthn v0.11.2 diff --git a/go.sum b/go.sum index 68a5ed1c..28407d82 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4= github.com/PuerkitoBio/goquery v1.10.0/go.mod h1:TjZZl68Q3eGHNBA8CWaxAN7rOU1EbDz3CWuolcO5Yu4= -github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= -github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= diff --git a/internal/database/database.go b/internal/database/database.go index 859aa917..851d601f 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -10,7 +10,7 @@ import ( "time" // Postgresql driver import - _ "github.com/lib/pq" + pq "github.com/lib/pq" ) // NewConnectionPool configures the database connection pool. @@ -32,6 +32,14 @@ func Migrate(db *sql.DB) error { var currentVersion int db.QueryRow(`SELECT version FROM schema_version`).Scan(¤tVersion) + driver := "" + switch db.Driver().(type) { + case *pq.Driver: + driver = "postgresql" + default: + panic(fmt.Sprintf("the driver %s isn't supported", db.Driver())) + } + slog.Info("Running database migrations", slog.Int("current_version", currentVersion), slog.Int("latest_version", schemaVersion), @@ -45,7 +53,7 @@ func Migrate(db *sql.DB) error { return fmt.Errorf("[Migration v%d] %v", newVersion, err) } - if err := migrations[version](tx); err != nil { + if err := migrations[version](tx, driver); err != nil { tx.Rollback() return fmt.Errorf("[Migration v%d] %v", newVersion, err) } diff --git a/internal/database/migrations.go b/internal/database/migrations.go index 8285038e..f9e4e8bd 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -10,8 +10,8 @@ import ( var schemaVersion = len(migrations) // Order is important. Add new migrations at the end of the list. -var migrations = []func(tx *sql.Tx) error{ - func(tx *sql.Tx) (err error) { +var migrations = []func(tx *sql.Tx, driver string) error{ + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TABLE schema_version ( version text not null @@ -120,16 +120,19 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { - sql := ` + func(tx *sql.Tx, driver string) (err error) { + if driver == "postgresql" { + sql := ` CREATE EXTENSION IF NOT EXISTS hstore; ALTER TABLE users ADD COLUMN extra hstore; CREATE INDEX users_extra_idx ON users using gin(extra); - ` - _, err = tx.Exec(sql) - return err + ` + _, err = tx.Exec(sql) + return err + } + return nil }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TABLE tokens ( id text not null, @@ -141,7 +144,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TYPE entry_sorting_direction AS enum('asc', 'desc'); ALTER TABLE users ADD COLUMN entry_direction entry_sorting_direction default 'asc'; @@ -149,7 +152,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TABLE integrations ( user_id int not null, @@ -170,27 +173,27 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN scraper_rules text default ''` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN rewrite_rules text default ''` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN crawler boolean default 'f'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE sessions rename to user_sessions` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` DROP TABLE tokens; @@ -204,7 +207,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN wallabag_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN wallabag_url text default ''; @@ -216,12 +219,12 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE entries ADD COLUMN starred bool default 'f'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE INDEX entries_user_status_idx ON entries(user_id, status); CREATE INDEX feeds_user_category_idx ON feeds(user_id, category_id); @@ -229,7 +232,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN nunux_keeper_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN nunux_keeper_url text default ''; @@ -238,17 +241,17 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE enclosures ALTER COLUMN size SET DATA TYPE bigint` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE entries ADD COLUMN comments_url text default ''` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN pocket_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN pocket_access_token text default ''; @@ -257,14 +260,14 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE user_sessions ALTER COLUMN ip SET DATA TYPE inet using ip::inet; ` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE feeds ADD COLUMN username text default ''; ALTER TABLE feeds ADD COLUMN password text default ''; @@ -272,7 +275,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE entries ADD COLUMN document_vectors tsvector; UPDATE entries SET document_vectors = to_tsvector(substring(title || ' ' || coalesce(content, '') for 1000000)); @@ -281,12 +284,12 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN user_agent text default ''` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` UPDATE entries @@ -296,17 +299,17 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN keyboard_shortcuts boolean default 't'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN disabled boolean default 'f';` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE users ALTER COLUMN theme SET DEFAULT 'light_serif'; UPDATE users SET theme='light_serif' WHERE theme='default'; @@ -316,7 +319,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE entries ADD COLUMN changed_at timestamp with time zone; UPDATE entries SET changed_at = published_at; @@ -325,7 +328,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TABLE api_keys ( id serial not null, @@ -341,7 +344,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE entries ADD COLUMN share_code text not null default ''; CREATE UNIQUE INDEX entries_share_code_idx ON entries USING btree(share_code) WHERE share_code <> ''; @@ -349,12 +352,12 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `CREATE INDEX enclosures_user_entry_url_idx ON enclosures(user_id, entry_id, md5(url))` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE feeds ADD COLUMN next_check_at timestamp with time zone default now(); CREATE INDEX entries_user_feed_idx ON entries (user_id, feed_id); @@ -362,52 +365,52 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN ignore_http_cache bool default false` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN entries_per_page int default 100` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN show_reading_time boolean default 't'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status)` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN fetch_via_proxy bool default false` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `CREATE INDEX entries_feed_id_status_hash_idx ON entries USING btree (feed_id, status, hash)` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `CREATE INDEX entries_user_id_status_starred_idx ON entries (user_id, status, starred)` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN entry_swipe boolean default 't'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE integrations DROP COLUMN fever_password` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE feeds ADD COLUMN blocklist_rules text not null default '', @@ -416,12 +419,12 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE entries ADD COLUMN reading_time int not null default 0` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE entries ADD COLUMN created_at timestamp with time zone not null default now(); UPDATE entries SET created_at = published_at; @@ -429,7 +432,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, driver string) (err error) { _, err = tx.Exec(` ALTER TABLE users ADD column stylesheet text not null default '', @@ -440,63 +443,69 @@ var migrations = []func(tx *sql.Tx) error{ return err } - _, err = tx.Exec(` - DECLARE my_cursor CURSOR FOR - SELECT - id, - COALESCE(extra->'custom_css', '') as custom_css, - COALESCE(extra->'google_id', '') as google_id, - COALESCE(extra->'oidc_id', '') as oidc_id - FROM users - FOR UPDATE - `) - if err != nil { - return err - } - defer tx.Exec("CLOSE my_cursor") - - for { - var ( - userID int64 - customStylesheet string - googleID string - oidcID string - ) - - if err := tx.QueryRow(`FETCH NEXT FROM my_cursor`).Scan(&userID, &customStylesheet, &googleID, &oidcID); err != nil { - if err == sql.ErrNoRows { - break - } - return err - } - - _, err := tx.Exec( - `UPDATE - users - SET - stylesheet=$2, - google_id=$3, - openid_connect_id=$4 - WHERE - id=$1 - `, - userID, customStylesheet, googleID, oidcID) + if driver == "postgresql" { + _, err = tx.Exec(` + DECLARE my_cursor CURSOR FOR + SELECT + id, + COALESCE(extra->'custom_css', '') as custom_css, + COALESCE(extra->'google_id', '') as google_id, + COALESCE(extra->'oidc_id', '') as oidc_id + FROM users + FOR UPDATE + `) if err != nil { return err } + defer tx.Exec("CLOSE my_cursor") + + for { + var ( + userID int64 + customStylesheet string + googleID string + oidcID string + ) + + if err := tx.QueryRow(`FETCH NEXT FROM my_cursor`).Scan(&userID, &customStylesheet, &googleID, &oidcID); err != nil { + if err == sql.ErrNoRows { + break + } + return err + } + + _, err := tx.Exec( + `UPDATE + users + SET + stylesheet=$2, + google_id=$3, + openid_connect_id=$4 + WHERE + id=$1 + `, + userID, customStylesheet, googleID, oidcID) + if err != nil { + return err + } + } } return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, driver string) (err error) { + if driver == "postgresql" { + if _, err = tx.Exec(`ALTER TABLE users DROP COLUMN extra;`); err != nil { + return nil + } + } _, err = tx.Exec(` - ALTER TABLE users DROP COLUMN extra; CREATE UNIQUE INDEX users_google_id_idx ON users(google_id) WHERE google_id <> ''; CREATE UNIQUE INDEX users_openid_connect_id_idx ON users(openid_connect_id) WHERE openid_connect_id <> ''; `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` CREATE INDEX entries_feed_url_idx ON entries(feed_id, url); CREATE INDEX entries_user_status_feed_idx ON entries(user_id, status, feed_id); @@ -504,7 +513,7 @@ var migrations = []func(tx *sql.Tx) error{ `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` CREATE TABLE acme_cache ( key varchar(400) not null primary key, @@ -514,13 +523,13 @@ var migrations = []func(tx *sql.Tx) error{ `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE feeds ADD COLUMN allow_self_signed_certificates boolean not null default false `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TYPE webapp_display_mode AS enum('fullscreen', 'standalone', 'minimal-ui', 'browser'); ALTER TABLE users ADD COLUMN display_mode webapp_display_mode default 'standalone'; @@ -528,24 +537,24 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN cookie text default ''` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE categories ADD COLUMN hide_globally boolean not null default false `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE feeds ADD COLUMN hide_globally boolean not null default false `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN telegram_bot_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN telegram_bot_token text default ''; @@ -554,7 +563,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` CREATE TYPE entry_sorting_order AS enum('published_at', 'created_at'); ALTER TABLE users ADD COLUMN entry_order entry_sorting_order default 'published_at'; @@ -562,7 +571,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN googlereader_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN googlereader_username text default ''; @@ -571,7 +580,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN espial_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN espial_url text default ''; @@ -581,7 +590,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN linkding_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN linkding_url text default ''; @@ -590,38 +599,38 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE feeds ADD COLUMN url_rewrite_rules text not null default '' `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE users ADD COLUMN default_reading_speed int default 265; ALTER TABLE users ADD COLUMN cjk_reading_speed int default 500; `) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE users ADD COLUMN default_home_page text default 'unread'; `) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE integrations ADD COLUMN wallabag_only_url bool default 'f'; `) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE users ADD COLUMN categories_sorting_order text not null default 'unread_count'; `) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN matrix_bot_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN matrix_bot_user text default ''; @@ -632,18 +641,18 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN double_tap boolean default 't'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE entries ADD COLUMN tags text[] default '{}'; `) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE users RENAME double_tap TO gesture_nav; ALTER TABLE users ALTER COLUMN gesture_nav SET DATA TYPE text using case when gesture_nav = true then 'tap' when gesture_nav = false then 'none' end; @@ -652,14 +661,14 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN linkding_tags text default ''; ` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE feeds ADD COLUMN no_media_player boolean default 'f'; ALTER TABLE enclosures ADD COLUMN media_progression int default 0; @@ -667,14 +676,14 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN linkding_mark_as_unread bool default 'f'; ` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { // Delete duplicated rows sql := ` DELETE FROM enclosures a USING enclosures b @@ -702,12 +711,12 @@ var migrations = []func(tx *sql.Tx) error{ return nil }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN mark_read_on_view boolean default 't'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN notion_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN notion_token text default ''; @@ -716,7 +725,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN readwise_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN readwise_api_key text default ''; @@ -724,7 +733,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN apprise_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN apprise_url text default ''; @@ -733,7 +742,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN shiori_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN shiori_url text default ''; @@ -743,7 +752,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN shaarli_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN shaarli_url text default ''; @@ -752,13 +761,13 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` ALTER TABLE feeds ADD COLUMN apprise_service_urls text default ''; `) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN webhook_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN webhook_url text default ''; @@ -767,7 +776,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN telegram_bot_topic_id int; ALTER TABLE integrations ADD COLUMN telegram_bot_disable_web_page_preview bool default 'f'; @@ -776,14 +785,14 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN telegram_bot_disable_buttons bool default 'f'; ` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` -- Speed up has_enclosure CREATE INDEX enclosures_entry_id_idx ON enclosures(entry_id); @@ -799,7 +808,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN rssbridge_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN rssbridge_url text default ''; @@ -807,7 +816,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { _, err = tx.Exec(` CREATE TABLE webauthn_credentials ( handle bytea primary key, @@ -825,7 +834,7 @@ var migrations = []func(tx *sql.Tx) error{ `) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN omnivore_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN omnivore_api_key text default ''; @@ -834,7 +843,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN linkace_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN linkace_url text default ''; @@ -846,7 +855,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN linkwarden_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN linkwarden_url text default ''; @@ -855,7 +864,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN readeck_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN readeck_only_url bool default 'f'; @@ -866,29 +875,29 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN disable_http2 bool default 'f'` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN media_playback_rate numeric default 1;` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { // the WHERE part speed-up the request a lot sql := `UPDATE entries SET tags = array_remove(tags, '') WHERE '' = ANY(tags);` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { // Entry URLs can exceeds btree maximum size // Checking entry existence is now using entries_feed_id_status_hash_idx index _, err = tx.Exec(`DROP INDEX entries_feed_url_idx`) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN raindrop_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN raindrop_token text default ''; @@ -898,12 +907,12 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE feeds ADD COLUMN description text default ''` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE users ADD COLUMN block_filter_entry_rules text not null default '', @@ -912,7 +921,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN betula_url text default ''; ALTER TABLE integrations ADD COLUMN betula_token text default ''; @@ -921,7 +930,7 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN ntfy_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN ntfy_url text default ''; @@ -937,22 +946,22 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN mark_read_on_media_player_completion bool default 'f';` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN custom_js text not null default '';` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := `ALTER TABLE users ADD COLUMN external_font_hosts text not null default '';` _, err = tx.Exec(sql) return err }, - func(tx *sql.Tx) (err error) { + func(tx *sql.Tx, _ string) (err error) { sql := ` ALTER TABLE integrations ADD COLUMN cubox_enabled bool default 'f'; ALTER TABLE integrations ADD COLUMN cubox_api_link text default ''; diff --git a/internal/oauth2/authorization.go b/internal/oauth2/authorization.go index 5854cb8c..b9a1d08a 100644 --- a/internal/oauth2/authorization.go +++ b/internal/oauth2/authorization.go @@ -6,7 +6,6 @@ package oauth2 // import "miniflux.app/v2/internal/oauth2" import ( "crypto/sha256" "encoding/base64" - "io" "golang.org/x/oauth2" @@ -33,17 +32,14 @@ func (u *Authorization) CodeVerifier() string { func GenerateAuthorization(config *oauth2.Config) *Authorization { codeVerifier := crypto.GenerateRandomStringHex(32) - - sha2 := sha256.New() - io.WriteString(sha2, codeVerifier) - codeChallenge := base64.RawURLEncoding.EncodeToString(sha2.Sum(nil)) + sum := sha256.Sum256([]byte(codeVerifier)) state := crypto.GenerateRandomStringHex(24) authUrl := config.AuthCodeURL( state, oauth2.SetAuthURLParam("code_challenge_method", "S256"), - oauth2.SetAuthURLParam("code_challenge", codeChallenge), + oauth2.SetAuthURLParam("code_challenge", base64.RawURLEncoding.EncodeToString(sum[:])), ) return &Authorization{ diff --git a/internal/reader/readingtime/readingtime.go b/internal/reader/readingtime/readingtime.go index 9159ee71..6175718c 100644 --- a/internal/reader/readingtime/readingtime.go +++ b/internal/reader/readingtime/readingtime.go @@ -7,33 +7,37 @@ package readingtime import ( "math" "strings" + "unicode" "unicode/utf8" "miniflux.app/v2/internal/reader/sanitizer" - - "github.com/abadojack/whatlanggo" ) // EstimateReadingTime returns the estimated reading time of an article in minute. func EstimateReadingTime(content string, defaultReadingSpeed, cjkReadingSpeed int) int { sanitizedContent := sanitizer.StripTags(content) + truncationPoint := min(len(sanitizedContent), 50) - // Litterature on language detection says that around 100 signes is enough, we're safe here. - truncationPoint := min(len(sanitizedContent), 250) - - // We're only interested in identifying Japanse/Chinese/Korean - options := whatlanggo.Options{ - Whitelist: map[whatlanggo.Lang]bool{ - whatlanggo.Jpn: true, - whatlanggo.Cmn: true, - whatlanggo.Kor: true, - }, - } - langInfo := whatlanggo.DetectWithOptions(sanitizedContent[:truncationPoint], options) - - if langInfo.IsReliable() { + if isCJK(sanitizedContent[:truncationPoint]) { return int(math.Ceil(float64(utf8.RuneCountInString(sanitizedContent)) / float64(cjkReadingSpeed))) } - nbOfWords := len(strings.Fields(sanitizedContent)) - return int(math.Ceil(float64(nbOfWords) / float64(defaultReadingSpeed))) + return int(math.Ceil(float64(len(strings.Fields(sanitizedContent))) / float64(defaultReadingSpeed))) +} + +func isCJK(text string) bool { + totalCJK := 0 + + for _, r := range text[:min(len(text), 50)] { + if unicode.Is(unicode.Han, r) || + unicode.Is(unicode.Hangul, r) || + unicode.Is(unicode.Hiragana, r) || + unicode.Is(unicode.Katakana, r) || + unicode.Is(unicode.Yi, r) || + unicode.Is(unicode.Bopomofo, r) { + totalCJK++ + } + } + + // if at least 50% of the text is CJK, odds are that the text is in CJK. + return totalCJK > len(text)/50 } diff --git a/internal/validator/user.go b/internal/validator/user.go index a7e05edb..53220148 100644 --- a/internal/validator/user.go +++ b/internal/validator/user.go @@ -6,6 +6,7 @@ package validator // import "miniflux.app/v2/internal/validator" import ( "slices" "strings" + "unicode" "miniflux.app/v2/internal/locale" "miniflux.app/v2/internal/model" @@ -22,6 +23,10 @@ func ValidateUserCreationWithPassword(store *storage.Storage, request *model.Use return locale.NewLocalizedError("error.user_already_exists") } + if err := validateUsername(request.Username); err != nil { + return err + } + if err := validatePassword(request.Password); err != nil { return err } @@ -146,6 +151,23 @@ func validatePassword(password string) *locale.LocalizedError { return nil } +// validateUsername return an error if the `username` argument contains +// a character that isn't alphanumerical nor `_` and `-`. +func validateUsername(username string) *locale.LocalizedError { + if strings.ContainsFunc(username, func(r rune) bool { + if unicode.IsLetter(r) || unicode.IsNumber(r) { + return false + } + if r == '_' || r == '-' || r == '@' || r == '.' { + return false + } + return true + }) { + return locale.NewLocalizedError("error.invalid_username") + } + return nil +} + func validateTheme(theme string) *locale.LocalizedError { themes := model.Themes() if _, found := themes[theme]; !found { diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index 7121a111..e9a42b2a 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -3,7 +3,11 @@ package validator // import "miniflux.app/v2/internal/validator" -import "testing" +import ( + "testing" + + "miniflux.app/v2/internal/locale" +) func TestIsValidURL(t *testing.T) { scenarios := map[string]bool{ @@ -77,3 +81,25 @@ func TestIsValidDomain(t *testing.T) { } } } + +func TestValidateUsername(t *testing.T) { + scenarios := map[string]*locale.LocalizedError{ + "jvoisin": nil, + "j.voisin": nil, + "j@vois.in": nil, + "invalid username": locale.NewLocalizedError("error.invalid_username"), + } + + for username, expected := range scenarios { + result := validateUsername(username) + if expected == nil { + if result != nil { + t.Errorf(`got an unexpected error for %q instead of nil: %v`, username, result) + } + } else { + if result == nil { + t.Errorf(`expected an error, got nil.`) + } + } + } +}