From e9b827924119351ce46cd4d6a2318d755f6bed26 Mon Sep 17 00:00:00 2001 From: haras Date: Sun, 14 Sep 2025 08:50:42 +0200 Subject: [PATCH] feat: multi db support --- .envrc | 2 + .gitignore | 1 + .helix/languages.toml | 44 + CONTRIBUTING.md | 28 +- Makefile | 71 +- README.md | 33 +- contrib/sysvinit/etc/init.d/miniflux | 4 +- docker-compose.yml | 37 + flake.lock | 61 + flake.nix | 59 + go.mod | 9 + go.sum | 45 + internal/cli/cli.go | 14 +- internal/config/options.go | 6 +- internal/config/options_parsing_test.go | 2 +- internal/database/cockroach.go | 396 ++++++ internal/database/database.go | 109 +- internal/database/migrations.go | 1344 ------------------ internal/database/postgresql.go | 1342 ++++++++++++++++- internal/database/sqlite.go | 378 +++++ internal/storage/enclosure.go | 14 +- internal/storage/entry.go | 102 +- internal/storage/entry_pagination_builder.go | 47 +- internal/storage/entry_query_builder.go | 71 +- internal/storage/storage.go | 9 +- miniflux.1 | 4 +- packaging/systemd/miniflux.service | 2 +- 27 files changed, 2770 insertions(+), 1464 deletions(-) create mode 100644 .envrc create mode 100644 .helix/languages.toml create mode 100644 docker-compose.yml create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 internal/database/cockroach.go delete mode 100644 internal/database/migrations.go create mode 100644 internal/database/sqlite.go diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..60af40a5 --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +use flake . --show-trace +dotenv_if_exists .env diff --git a/.gitignore b/.gitignore index 0a23210f..908ff842 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.deb *.rpm miniflux-* +.direnv/ diff --git a/.helix/languages.toml b/.helix/languages.toml new file mode 100644 index 00000000..61211558 --- /dev/null +++ b/.helix/languages.toml @@ -0,0 +1,44 @@ +[language-server] +nil = { command = "nil" } +taplo = { command = "taplo", args = ["lsp", "stdio"] } +yaml-language-server = { command = "yaml-language-server", args = ["--stdio"] } +marksman = { command = "marksman", args = ["server"] } +vscode-json-language-server = { command = "vscode-json-language-server", args = [ + "--stdio", +], config = { json = { validate = { enable = true } } } } +gopls = { command = "gopls", config = { staticcheck = true } } + +[[language]] +name = "nix" +auto-format = true +formatter = { command = "nixpkgs-fmt" } +language-servers = ["nil"] + +[[language]] +name = "toml" +auto-format = true +language-servers = ["taplo"] + +[[language]] +name = "yaml" +auto-format = true +formatter = { command = "prettier", args = ["--parser", "yaml"] } +language-servers = ["yaml-language-server"] + +[[language]] +name = "json" +auto-format = true +formatter = { command = "prettier", args = ["--parser", "json"] } +language-servers = ["vscode-json-language-server"] + +[[language]] +name = "markdown" +auto-format = true +formatter = { command = "prettier", args = ["--parser", "markdown"] } +language-servers = ["marksman"] + +[[language]] +name = "go" +auto-format = true +language-servers = ["gopls"] +formatter = { command = "gofumpt" } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73929cf5..ce7804da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,17 +37,21 @@ When reporting bugs: - **Git** - **Go >= 1.24** - **PostgreSQL** +- **CockroachDB** +- **SQLite** ### Getting Started 1. **Fork the repository** on GitHub 2. **Clone your fork locally:** + ```bash git clone https://github.com/YOUR_USERNAME/miniflux.git cd miniflux ``` 3. **Build the application binary:** + ```bash make miniflux ``` @@ -59,15 +63,11 @@ When reporting bugs: ### Database Setup -For development and testing, you can run a local PostgreSQL database with Docker: +For development and testing, you can run PostgreSQL and CockroachDB via docker compose: ```bash -# Start PostgreSQL container -docker run --rm --name miniflux2-db -p 5432:5432 \ - -e POSTGRES_DB=miniflux2 \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - postgres +# Start PostgreSQL and CockroachDB containers +docker compose -d ``` You can also use an existing PostgreSQL instance. Make sure to set the `DATABASE_URL` environment variable accordingly. @@ -77,20 +77,27 @@ You can also use an existing PostgreSQL instance. Make sure to set the `DATABASE ### Code Quality 1. **Run the linter:** + ```bash make lint ``` + Requires `staticcheck` and `golangci-lint` to be installed. 2. **Run unit tests:** + ```bash make test ``` 3. **Run integration tests:** ```bash - make integration-test - make clean-integration-test + make integration-test-postgresql + make clean-integration-test-postgresql + make integration-test-cockroachdb + make clean-integration-test-cockroachdb + make integration-test-sqlite + make clean-integration-test-sqlite ``` ### Building @@ -103,6 +110,7 @@ You can also use an existing PostgreSQL instance. Make sure to set the `DATABASE ### Cross-Platform Support Miniflux supports multiple architectures. When making changes, ensure compatibility across: + - Linux (amd64, arm64, armv7, armv6, armv5) - macOS (amd64, arm64) - FreeBSD, OpenBSD, Windows (amd64) @@ -155,11 +163,13 @@ When creating a pull request, please include: ## Testing ### Unit Tests + - Write unit tests for new functions and methods - Ensure tests are fast and don't require external dependencies - Aim for good test coverage ### Integration Tests + - Add integration tests for new API endpoints - Tests run against a real PostgreSQL database - Ensure tests clean up after themselves diff --git a/Makefile b/Makefile index e7a6c6e4..ab9c81ff 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ DOCKER_IMAGE := miniflux/miniflux VERSION := $(shell git describe --tags --exact-match 2>/dev/null) LD_FLAGS := "-s -w -X 'miniflux.app/v2/internal/version.Version=$(VERSION)'" PKG_LIST := $(shell go list ./... | grep -v /vendor/) -DB_URL := postgres://postgres:postgres@localhost/miniflux_test?sslmode=disable DOCKER_PLATFORM := amd64 export PGPASSWORD := postgres @@ -28,8 +27,12 @@ export PGPASSWORD := postgres add-string \ test \ lint \ - integration-test \ - clean-integration-test \ + integration-test-postgresql \ + clean-integration-test-postgresql \ + integration-test-cockroachdb \ + clean-integration-test-cockroachdb \ + integration-test-sqlite \ + clean-integration-test-sqlite \ docker-image \ docker-image-distroless \ docker-images \ @@ -103,17 +106,18 @@ lint: staticcheck ./... golangci-lint run --disable errcheck --enable sqlclosecheck --enable misspell --enable gofmt --enable goimports --enable whitespace -integration-test: - psql -U postgres -c 'drop database if exists miniflux_test;' - psql -U postgres -c 'create database miniflux_test;' +integration-test-postgresql: + psql -U postgres -p 5432 -d postgres -c 'drop database if exists miniflux2;' + psql -U postgres -p 5432 -d postgres -c 'create database miniflux2;' - DATABASE_URL=$(DB_URL) \ + go build -o miniflux-test main.go + DATABASE_URL='postgres://postgres:postgres@localhost:5432/miniflux2?sslmode=disable' \ ADMIN_USERNAME=admin \ ADMIN_PASSWORD=test123 \ CREATE_ADMIN=1 \ RUN_MIGRATIONS=1 \ LOG_LEVEL=debug \ - go run main.go >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid" + ./miniflux-test >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid" while ! nc -z localhost 8080; do sleep 1; done @@ -122,10 +126,57 @@ integration-test: TEST_MINIFLUX_ADMIN_PASSWORD=test123 \ go test -v -count=1 ./internal/api -clean-integration-test: +clean-integration-test-postgresql: @ kill -9 `cat /tmp/miniflux.pid` @ rm -f /tmp/miniflux.pid /tmp/miniflux.log - @ psql -U postgres -c 'drop database if exists miniflux_test;' + @ psql -U postgres -d postgres -c 'drop database if exists miniflux2;' + +integration-test-cockroachdb: + psql -U postgres -d postgres -p 26257 -c 'drop database if exists miniflux2;' + psql -U postgres -d postgres -p 26257 -c 'create database miniflux2;' + + go build -o miniflux-test main.go + DATABASE_URL='cockroach://postgres:postgres@localhost:26257/miniflux2?sslmode=disable' \ + ADMIN_USERNAME=admin \ + ADMIN_PASSWORD=test123 \ + CREATE_ADMIN=1 \ + RUN_MIGRATIONS=1 \ + LOG_LEVEL=debug \ + ./miniflux-test >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid" + + while ! nc -z localhost 8080; do sleep 1; done + + TEST_MINIFLUX_BASE_URL=http://127.0.0.1:8080 \ + TEST_MINIFLUX_ADMIN_USERNAME=admin \ + TEST_MINIFLUX_ADMIN_PASSWORD=test123 \ + go test -v -count=1 ./internal/api + +clean-integration-test-cockroachdb: + @ kill -9 `cat /tmp/miniflux.pid` + @ rm -f /tmp/miniflux.pid /tmp/miniflux.log + @ psql -U postgres -d postgres -c 'drop database if exists miniflux2;' + +integration-test-sqlite: + go build -o miniflux-test main.go + DATABASE_URL='file::memory:?cache=shared&_pragma=foreign_keys(ON)' \ + ADMIN_USERNAME=admin \ + ADMIN_PASSWORD=test123 \ + CREATE_ADMIN=1 \ + RUN_MIGRATIONS=1 \ + DEBUG=1 \ + ./miniflux-test >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid" + + while ! nc -z localhost 8080; do sleep 1; done + + TEST_MINIFLUX_BASE_URL=http://127.0.0.1:8080 \ + TEST_MINIFLUX_ADMIN_USERNAME=admin \ + TEST_MINIFLUX_ADMIN_PASSWORD=test123 \ + go test -v -count=1 ./internal/api + +clean-integration-test-sqlite: + @ kill -9 `cat /tmp/miniflux.pid` + @ rm -f /tmp/miniflux.pid /tmp/miniflux.log + @ rm miniflux-test docker-image: docker build --pull -t $(DOCKER_IMAGE):$(VERSION) -f packaging/docker/alpine/Dockerfile . diff --git a/README.md b/README.md index b70ee929..76430e4f 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,11 @@ -Miniflux 2 -========== +# Miniflux 2 Miniflux is a minimalist and opinionated feed reader. It's simple, fast, lightweight and super easy to install. Official website: -Features --------- +## Features ### Feed Reader @@ -19,7 +17,7 @@ Features - Share individual articles publicly. - Fetches website icons (favicons). - Saves articles to third-party services. -- Provides full-text search (powered by Postgres). +- Provides full-text search. - Available in 20 languages: Portuguese (Brazilian), Chinese (Simplified and Traditional), Dutch, English (US), Finnish, French, German, Greek, Hindi, Indonesian, Italian, Japanese, Polish, Romanian, Russian, Taiwanese POJ, Ukrainian, Spanish, and Turkish. ### Privacy and Security @@ -34,7 +32,7 @@ Features - Supports alternative YouTube video players such as [Invidious](https://invidio.us). - Blocks external JavaScript to prevent tracking and enhance security. - Sanitizes external content before rendering it. -- Enforces a [Content Security](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) and a [Trusted Types Policy](https://developer.mozilla.org/en-US/docs/Web/API/Trusted_Types_API) to only application JavaScript and blocks inline scripts and styles. +- Enforces a [Content Security](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) and a [Trusted Types Policy](https://developer.mozilla.org/en-US/docs/Web/API/Trusted_Types_API) to only application JavaScript and blocks inline scripts and styles. ### Bot Protection Bypass Mechanisms @@ -63,12 +61,12 @@ Features - Optional touch gesture support for navigation on mobile devices. - Custom stylesheets and JavaScript to personalize the user interface to your preferences. - Themes: - - Light (Sans-Serif) - - Light (Serif) - - Dark (Sans-Serif) - - Dark (Serif) - - System (Sans-Serif) – Automatically switches between Dark and Light themes based on system preferences. - - System (Serif) + - Light (Sans-Serif) + - Light (Serif) + - Dark (Sans-Serif) + - Dark (Serif) + - System (Sans-Serif) – Automatically switches between Dark and Light themes based on system preferences. + - System (Serif) ### Integrations @@ -90,7 +88,7 @@ Features - Written in [Go (Golang)](https://golang.org/). - Single binary compiled statically without dependency. -- Works only with [PostgreSQL](https://www.postgresql.org/). +- Works with [PostgreSQL](https://www.postgresql.org/), [CockroachDB](https://www.cockroachlabs.com/) and [SQLite](https://sqlite.org/). - Does not use any ORM or any complicated frameworks. - Uses modern vanilla JavaScript only when necessary. - All static files are bundled into the application binary using the Go `embed` package. @@ -109,8 +107,7 @@ Features - Only uses a couple of MB of memory and a negligible amount of CPU, even with several hundreds of feeds. - Respects/sends Last-Modified, If-Modified-Since, If-None-Match, Cache-Control, Expires and ETags headers, and has a default polling interval of 1h. -Documentation -------------- +## Documentation The Miniflux documentation is available here: ([Man page](https://miniflux.app/miniflux.1.html)) @@ -130,8 +127,7 @@ The Miniflux documentation is available here: ([Man - [Internationalization](https://miniflux.app/docs/i18n.html) - [Frequently Asked Questions](https://miniflux.app/faq.html) -Screenshots ------------ +## Screenshots Default theme: @@ -141,8 +137,7 @@ Dark theme when using keyboard navigation: ![Dark theme](https://miniflux.app/images/item-selection-black-theme.png) -Credits -------- +## Credits - Authors: Frédéric Guillot - [List of contributors](https://github.com/miniflux/v2/graphs/contributors) - Distributed under Apache 2.0 License diff --git a/contrib/sysvinit/etc/init.d/miniflux b/contrib/sysvinit/etc/init.d/miniflux index 936f5074..c82e90ce 100755 --- a/contrib/sysvinit/etc/init.d/miniflux +++ b/contrib/sysvinit/etc/init.d/miniflux @@ -4,8 +4,8 @@ # Provides: miniflux # Required-Start: $syslog $network # Required-Stop: $syslog -# Should-Start: postgresql -# Should-Stop: postgresql +# Should-Start: postgresql cockroachdb +# Should-Stop: postgresql cockroachdb # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: A rss reader diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..76627837 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +services: + postgresql: + image: postgres:17.6 + volumes: + - postgres-data:/var/lib/postgresql/data + ports: + - 5432:5432 + environment: + POSTGRES_DB: miniflux2 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD", "pg_isready -d miniflux2 -U postgres"] + timeout: 30s + interval: 5s + + cockroachdb: + image: cockroachdb/cockroach:v23.2.28 + command: start-single-node --insecure + volumes: + - cockroach-data:/cockroach/cockroach-data + environment: + COCKROACH_DATABASE: miniflux2 + COCKROACH_USER: postgres + ports: + - "26257:26257" + - "26258:8080" + healthcheck: + test: ["CMD", "cockroach", "sql", "--insecure", "--host=localhost:26257", "-e", "select 1"] + timeout: 30s + interval: 5s + +volumes: + postgres-data: + driver: local + cockroach-data: + driver: local diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..87d1f954 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757545623, + "narHash": "sha256-mCxPABZ6jRjUQx3bPP4vjA68ETbPLNz9V2pk9tO7pRQ=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "8cd5ce828d5d1d16feff37340171a98fc3bf6526", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..0ddf551b --- /dev/null +++ b/flake.nix @@ -0,0 +1,59 @@ +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-25.05"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = + { nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + in + { + devShells.default = pkgs.mkShell { + PGHOST = "localhost"; + PGPORT = 5432; + PGPASSWORD = "postgres"; + PGUSER = "postgres"; + PGDATABASE = "miniflux2"; + + COCKROACH_URL = "postgresql://postgres:postgres@localhost:26257/miniflux2"; + COCKROACH_INSECURE = true; + + packages = with pkgs; [ + git + + gnumake + foreman + + nil + nixfmt-rfc-style + + nodePackages.prettier + nodePackages.yaml-language-server + nodePackages.vscode-langservers-extracted + markdownlint-cli + nodePackages.markdown-link-check + marksman + taplo + + go + gopls + go-tools + gofumpt + golangci-lint + + postgresql + cockroachdb + sqlite + usql + ]; + }; + } + ); +} diff --git a/go.mod b/go.mod index 9ef5c56e..c8c8c99a 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( golang.org/x/net v0.44.0 golang.org/x/oauth2 v0.31.0 golang.org/x/term v0.35.0 + modernc.org/sqlite v1.39.0 ) require ( @@ -28,21 +29,29 @@ require ( github.com/andybalholm/cascadia v1.3.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/tdewolff/parse/v2 v2.8.3 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.29.0 // indirect google.golang.org/protobuf v1.36.8 // indirect + modernc.org/libc v1.66.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) go 1.24.0 diff --git a/go.sum b/go.sum index c913ee16..4c569bfb 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmr github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= @@ -28,6 +30,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -42,10 +46,14 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -56,6 +64,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -85,6 +95,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA= golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -92,6 +104,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -112,12 +126,15 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -153,6 +170,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= @@ -161,3 +180,29 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= +modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= +modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= +modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 062b85e9..23975ea3 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -168,7 +168,13 @@ func Parse() { printErrorAndExit(fmt.Errorf("unable to generate javascript bundle: %v", err)) } + kind, err := database.DetectKind(config.Opts.DatabaseURL()) + if err != nil { + printErrorAndExit(fmt.Errorf("unable to parse database kind: %v", err)) + } + db, err := database.NewConnectionPool( + kind, config.Opts.DatabaseURL(), config.Opts.DatabaseMinConns(), config.Opts.DatabaseMaxConns(), @@ -179,14 +185,14 @@ func Parse() { } defer db.Close() - store := storage.NewStorage(db) + store := storage.NewStorage(kind, db) if err := store.Ping(); err != nil { printErrorAndExit(err) } if flagMigrate { - if err := database.Migrate(db); err != nil { + if err := database.Migrate(kind, db); err != nil { printErrorAndExit(err) } return @@ -228,12 +234,12 @@ func Parse() { // Run migrations and start the daemon. if config.Opts.RunMigrations() { - if err := database.Migrate(db); err != nil { + if err := database.Migrate(kind, db); err != nil { printErrorAndExit(err) } } - if err := database.IsSchemaUpToDate(db); err != nil { + if err := database.IsSchemaUpToDate(kind, db); err != nil { printErrorAndExit(err) } diff --git a/internal/config/options.go b/internal/config/options.go index be8768a6..f2f4402e 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -183,8 +183,8 @@ func NewConfigOptions() *configOptions { }, }, "DATABASE_URL": { - ParsedStringValue: "user=postgres password=postgres dbname=miniflux2 sslmode=disable", - RawValue: "user=postgres password=postgres dbname=miniflux2 sslmode=disable", + ParsedStringValue: "postgres://postgres:postgres/postgres?sslmode=disable", + RawValue: "postgres://postgres:postgres/postgres?sslmode=disable", ValueType: stringType, Secret: true, }, @@ -779,7 +779,7 @@ func (c *configOptions) IsAuthProxyUserCreationAllowed() bool { } func (c *configOptions) IsDefaultDatabaseURL() bool { - return c.options["DATABASE_URL"].RawValue == "user=postgres password=postgres dbname=miniflux2 sslmode=disable" + return c.options["DATABASE_URL"].RawValue == "postgres://postgres:postgres/postgres?sslmode=disable" } func (c *configOptions) IsOAuth2UserCreationAllowed() bool { diff --git a/internal/config/options_parsing_test.go b/internal/config/options_parsing_test.go index cf965644..5c58ffbe 100644 --- a/internal/config/options_parsing_test.go +++ b/internal/config/options_parsing_test.go @@ -348,7 +348,7 @@ func TestDatabaseMinConnsOptionParsing(t *testing.T) { func TestDatabaseURLOptionParsing(t *testing.T) { configParser := NewConfigParser() - if configParser.options.DatabaseURL() != "user=postgres password=postgres dbname=miniflux2 sslmode=disable" { + if configParser.options.DatabaseURL() != "postgres://postgres:postgres/postgres?sslmode=disable" { t.Fatal("Expected DATABASE_URL to have default value") } diff --git a/internal/database/cockroach.go b/internal/database/cockroach.go new file mode 100644 index 00000000..41087d80 --- /dev/null +++ b/internal/database/cockroach.go @@ -0,0 +1,396 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package database // import "miniflux.app/v2/internal/database" + +import ( + "database/sql" + + _ "github.com/lib/pq" +) + +var cockroachSchemaVersion = len(cockroachMigrations) + +// Order is important. Add new migrations at the end of the list. +var cockroachMigrations = []Migration{ + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TABLE schema_version ( + version STRING NOT NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT schema_version_pkey PRIMARY KEY (rowid ASC) + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TYPE entry_status AS ENUM ('unread', 'read', 'removed'); + CREATE TYPE entry_sorting_direction AS ENUM ('asc', 'desc'); + CREATE TYPE webapp_display_mode AS ENUM ('fullscreen', 'standalone', 'minimal-ui', 'browser'); + CREATE TYPE entry_sorting_order AS ENUM ('published_at', 'created_at'); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TABLE users ( + id INT8 NOT NULL DEFAULT unique_rowid(), + username STRING NOT NULL, + password STRING NULL, + is_admin BOOL NULL DEFAULT false, + language STRING NULL DEFAULT 'en_US':::STRING, + timezone STRING NULL DEFAULT 'UTC':::STRING, + theme STRING NULL DEFAULT 'light_serif':::STRING, + last_login_at TIMESTAMPTZ NULL, + entry_direction entry_sorting_direction NULL DEFAULT 'asc':::entry_sorting_direction, + keyboard_shortcuts BOOL NULL DEFAULT true, + entries_per_page INT8 NULL DEFAULT 100:::INT8, + show_reading_time BOOL NULL DEFAULT true, + entry_swipe BOOL NULL DEFAULT true, + stylesheet STRING NOT NULL DEFAULT '':::STRING, + google_id STRING NOT NULL DEFAULT '':::STRING, + openid_connect_id STRING NOT NULL DEFAULT '':::STRING, + display_mode webapp_display_mode NULL DEFAULT 'standalone':::webapp_display_mode, + entry_order entry_sorting_order NULL DEFAULT 'published_at':::entry_sorting_order, + default_reading_speed INT8 NULL DEFAULT 265:::INT8, + cjk_reading_speed INT8 NULL DEFAULT 500:::INT8, + default_home_page STRING NULL DEFAULT 'unread':::STRING, + categories_sorting_order STRING NOT NULL DEFAULT 'unread_count':::STRING, + gesture_nav STRING NOT NULL DEFAULT 'tap':::STRING, + mark_read_on_view BOOL NULL DEFAULT true, + media_playback_rate DECIMAL NULL DEFAULT 1:::DECIMAL, + block_filter_entry_rules STRING NOT NULL DEFAULT '':::STRING, + keep_filter_entry_rules STRING NOT NULL DEFAULT '':::STRING, + mark_read_on_media_player_completion BOOL NULL DEFAULT false, + custom_js STRING NOT NULL DEFAULT '':::STRING, + external_font_hosts STRING NOT NULL DEFAULT '':::STRING, + always_open_external_links BOOL NULL DEFAULT false, + open_external_links_in_new_tab BOOL NULL DEFAULT true, + CONSTRAINT users_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX users_username_key (username ASC), + UNIQUE INDEX users_google_id_idx (google_id ASC) WHERE google_id != '':::STRING, + UNIQUE INDEX users_openid_connect_id_idx (openid_connect_id ASC) WHERE openid_connect_id != '':::STRING + ); + CREATE TABLE user_sessions ( + id INT8 NOT NULL DEFAULT unique_rowid(), + user_id INT8 NOT NULL, + token STRING NOT NULL, + created_at TIMESTAMPTZ NULL DEFAULT now():::TIMESTAMPTZ, + user_agent STRING NULL, + ip STRING NULL, + CONSTRAINT sessions_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX sessions_token_key (token ASC), + UNIQUE INDEX sessions_user_id_token_key (user_id ASC, token ASC) + ); + CREATE TABLE categories ( + id INT8 NOT NULL DEFAULT unique_rowid(), + user_id INT8 NOT NULL, + title STRING NOT NULL, + hide_globally BOOL NOT NULL DEFAULT false, + CONSTRAINT categories_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX categories_user_id_title_key (user_id ASC, title ASC) + ); + CREATE TABLE feeds ( + id INT8 NOT NULL DEFAULT unique_rowid(), + user_id INT8 NOT NULL, + category_id INT8 NOT NULL, + title STRING NOT NULL, + feed_url STRING NOT NULL, + site_url STRING NOT NULL, + checked_at TIMESTAMPTZ NULL DEFAULT now():::TIMESTAMPTZ, + etag_header STRING NULL DEFAULT '':::STRING, + last_modified_header STRING NULL DEFAULT '':::STRING, + parsing_error_msg STRING NULL DEFAULT '':::STRING, + parsing_error_count INT8 NULL DEFAULT 0:::INT8, + scraper_rules STRING NULL DEFAULT '':::STRING, + rewrite_rules STRING NULL DEFAULT '':::STRING, + crawler BOOL NULL DEFAULT false, + username STRING NULL DEFAULT '':::STRING, + password STRING NULL DEFAULT '':::STRING, + user_agent STRING NULL DEFAULT '':::STRING, + disabled BOOL NULL DEFAULT false, + next_check_at TIMESTAMPTZ NULL DEFAULT now():::TIMESTAMPTZ, + ignore_http_cache BOOL NULL DEFAULT false, + fetch_via_proxy BOOL NULL DEFAULT false, + blocklist_rules STRING NOT NULL DEFAULT '':::STRING, + keeplist_rules STRING NOT NULL DEFAULT '':::STRING, + allow_self_signed_certificates BOOL NOT NULL DEFAULT false, + cookie STRING NULL DEFAULT '':::STRING, + hide_globally BOOL NOT NULL DEFAULT false, + url_rewrite_rules STRING NOT NULL DEFAULT '':::STRING, + no_media_player BOOL NULL DEFAULT false, + apprise_service_urls STRING NULL DEFAULT '':::STRING, + disable_http2 BOOL NULL DEFAULT false, + description STRING NULL DEFAULT '':::STRING, + ntfy_enabled BOOL NULL DEFAULT false, + ntfy_priority INT8 NULL DEFAULT 3:::INT8, + webhook_url STRING NULL DEFAULT '':::STRING, + pushover_enabled BOOL NULL DEFAULT false, + pushover_priority INT8 NULL DEFAULT 0:::INT8, + ntfy_topic STRING NULL DEFAULT '':::STRING, + proxy_url STRING NULL DEFAULT '':::STRING, + block_filter_entry_rules STRING NOT NULL DEFAULT '':::STRING, + keep_filter_entry_rules STRING NOT NULL DEFAULT '':::STRING, + CONSTRAINT feeds_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX feeds_user_id_feed_url_key (user_id ASC, feed_url ASC), + INDEX feeds_user_category_idx (user_id ASC, category_id ASC), + INDEX feeds_feed_id_hide_globally_idx (id ASC, hide_globally ASC) + ); + CREATE TABLE entries ( + id INT8 NOT NULL DEFAULT unique_rowid(), + user_id INT8 NOT NULL, + feed_id INT8 NOT NULL, + hash STRING NOT NULL, + published_at TIMESTAMPTZ NOT NULL, + title STRING NOT NULL, + url STRING NOT NULL, + author STRING NULL, + content STRING NULL, + status entry_status NULL DEFAULT 'unread':::entry_status, + starred BOOL NULL DEFAULT false, + comments_url STRING NULL DEFAULT '':::STRING, + document_vectors TSVECTOR NULL, + changed_at TIMESTAMPTZ NOT NULL, + share_code STRING NOT NULL DEFAULT '':::STRING, + reading_time INT8 NOT NULL DEFAULT 0:::INT8, + created_at TIMESTAMPTZ NOT NULL DEFAULT now():::TIMESTAMPTZ, + tags STRING[] NULL DEFAULT ARRAY[]:::STRING[], + CONSTRAINT entries_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX entries_feed_id_hash_key (feed_id ASC, hash ASC), + INDEX entries_feed_idx (feed_id ASC), + INDEX entries_user_status_idx (user_id ASC, status ASC), + INVERTED INDEX document_vectors_idx (document_vectors), + UNIQUE INDEX entries_share_code_idx (share_code ASC) WHERE share_code != '':::STRING, + INDEX entries_user_feed_idx (user_id ASC, feed_id ASC), + INDEX entries_id_user_status_idx (id ASC, user_id ASC, status ASC), + INDEX entries_feed_id_status_hash_idx (feed_id ASC, status ASC, hash ASC), + INDEX entries_user_id_status_starred_idx (user_id ASC, status ASC, starred ASC), + INDEX entries_user_status_feed_idx (user_id ASC, status ASC, feed_id ASC), + INDEX entries_user_status_changed_idx (user_id ASC, status ASC, changed_at ASC), + INDEX entries_user_status_published_idx (user_id ASC, status ASC, published_at ASC), + INDEX entries_user_status_created_idx (user_id ASC, status ASC, created_at ASC), + INDEX entries_user_status_changed_published_idx (user_id ASC, status ASC, changed_at ASC, published_at ASC) + ); + CREATE TABLE enclosures ( + id INT8 NOT NULL DEFAULT unique_rowid(), + user_id INT8 NOT NULL, + entry_id INT8 NOT NULL, + url STRING NOT NULL, + size INT8 NULL DEFAULT 0:::INT8, + mime_type STRING NULL DEFAULT '':::STRING, + media_progression INT8 NULL DEFAULT 0:::INT8, + CONSTRAINT enclosures_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX enclosures_user_entry_url_unique_idx (user_id ASC, entry_id ASC, url ASC), + INDEX enclosures_entry_id_idx (entry_id ASC) + ); + CREATE TABLE icons ( + id INT8 NOT NULL DEFAULT unique_rowid(), + hash STRING NOT NULL, + mime_type STRING NOT NULL, + content BYTES NOT NULL, + external_id STRING NULL DEFAULT '':::STRING, + CONSTRAINT icons_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX icons_hash_key (hash ASC), + UNIQUE INDEX icons_external_id_idx (external_id ASC) WHERE external_id != '':::STRING + ); + CREATE TABLE feed_icons ( + feed_id INT8 NOT NULL, + icon_id INT8 NOT NULL, + CONSTRAINT feed_icons_pkey PRIMARY KEY (feed_id ASC, icon_id ASC) + ); + CREATE TABLE integrations ( + user_id INT8 NOT NULL, + pinboard_enabled BOOL NULL DEFAULT false, + pinboard_token STRING NULL DEFAULT '':::STRING, + pinboard_tags STRING NULL DEFAULT 'miniflux':::STRING, + pinboard_mark_as_unread BOOL NULL DEFAULT false, + instapaper_enabled BOOL NULL DEFAULT false, + instapaper_username STRING NULL DEFAULT '':::STRING, + instapaper_password STRING NULL DEFAULT '':::STRING, + fever_enabled BOOL NULL DEFAULT false, + fever_username STRING NULL DEFAULT '':::STRING, + fever_token STRING NULL DEFAULT '':::STRING, + wallabag_enabled BOOL NULL DEFAULT false, + wallabag_url STRING NULL DEFAULT '':::STRING, + wallabag_client_id STRING NULL DEFAULT '':::STRING, + wallabag_client_secret STRING NULL DEFAULT '':::STRING, + wallabag_username STRING NULL DEFAULT '':::STRING, + wallabag_password STRING NULL DEFAULT '':::STRING, + nunux_keeper_enabled BOOL NULL DEFAULT false, + nunux_keeper_url STRING NULL DEFAULT '':::STRING, + nunux_keeper_api_key STRING NULL DEFAULT '':::STRING, + telegram_bot_enabled BOOL NULL DEFAULT false, + telegram_bot_token STRING NULL DEFAULT '':::STRING, + telegram_bot_chat_id STRING NULL DEFAULT '':::STRING, + googlereader_enabled BOOL NULL DEFAULT false, + googlereader_username STRING NULL DEFAULT '':::STRING, + googlereader_password STRING NULL DEFAULT '':::STRING, + espial_enabled BOOL NULL DEFAULT false, + espial_url STRING NULL DEFAULT '':::STRING, + espial_api_key STRING NULL DEFAULT '':::STRING, + espial_tags STRING NULL DEFAULT 'miniflux':::STRING, + linkding_enabled BOOL NULL DEFAULT false, + linkding_url STRING NULL DEFAULT '':::STRING, + linkding_api_key STRING NULL DEFAULT '':::STRING, + wallabag_only_url BOOL NULL DEFAULT false, + matrix_bot_enabled BOOL NULL DEFAULT false, + matrix_bot_user STRING NULL DEFAULT '':::STRING, + matrix_bot_password STRING NULL DEFAULT '':::STRING, + matrix_bot_url STRING NULL DEFAULT '':::STRING, + matrix_bot_chat_id STRING NULL DEFAULT '':::STRING, + linkding_tags STRING NULL DEFAULT '':::STRING, + linkding_mark_as_unread BOOL NULL DEFAULT false, + notion_enabled BOOL NULL DEFAULT false, + notion_token STRING NULL DEFAULT '':::STRING, + notion_page_id STRING NULL DEFAULT '':::STRING, + readwise_enabled BOOL NULL DEFAULT false, + readwise_api_key STRING NULL DEFAULT '':::STRING, + apprise_enabled BOOL NULL DEFAULT false, + apprise_url STRING NULL DEFAULT '':::STRING, + apprise_services_url STRING NULL DEFAULT '':::STRING, + shiori_enabled BOOL NULL DEFAULT false, + shiori_url STRING NULL DEFAULT '':::STRING, + shiori_username STRING NULL DEFAULT '':::STRING, + shiori_password STRING NULL DEFAULT '':::STRING, + shaarli_enabled BOOL NULL DEFAULT false, + shaarli_url STRING NULL DEFAULT '':::STRING, + shaarli_api_secret STRING NULL DEFAULT '':::STRING, + webhook_enabled BOOL NULL DEFAULT false, + webhook_url STRING NULL DEFAULT '':::STRING, + webhook_secret STRING NULL DEFAULT '':::STRING, + telegram_bot_topic_id INT8 NULL, + telegram_bot_disable_web_page_preview BOOL NULL DEFAULT false, + telegram_bot_disable_notification BOOL NULL DEFAULT false, + telegram_bot_disable_buttons BOOL NULL DEFAULT false, + rssbridge_enabled BOOL NULL DEFAULT false, + rssbridge_url STRING NULL DEFAULT '':::STRING, + omnivore_enabled BOOL NULL DEFAULT false, + omnivore_api_key STRING NULL DEFAULT '':::STRING, + omnivore_url STRING NULL DEFAULT '':::STRING, + linkace_enabled BOOL NULL DEFAULT false, + linkace_url STRING NULL DEFAULT '':::STRING, + linkace_api_key STRING NULL DEFAULT '':::STRING, + linkace_tags STRING NULL DEFAULT '':::STRING, + linkace_is_private BOOL NULL DEFAULT true, + linkace_check_disabled BOOL NULL DEFAULT true, + linkwarden_enabled BOOL NULL DEFAULT false, + linkwarden_url STRING NULL DEFAULT '':::STRING, + linkwarden_api_key STRING NULL DEFAULT '':::STRING, + readeck_enabled BOOL NULL DEFAULT false, + readeck_only_url BOOL NULL DEFAULT false, + readeck_url STRING NULL DEFAULT '':::STRING, + readeck_api_key STRING NULL DEFAULT '':::STRING, + readeck_labels STRING NULL DEFAULT '':::STRING, + raindrop_enabled BOOL NULL DEFAULT false, + raindrop_token STRING NULL DEFAULT '':::STRING, + raindrop_collection_id STRING NULL DEFAULT '':::STRING, + raindrop_tags STRING NULL DEFAULT '':::STRING, + betula_url STRING NULL DEFAULT '':::STRING, + betula_token STRING NULL DEFAULT '':::STRING, + betula_enabled BOOL NULL DEFAULT false, + ntfy_enabled BOOL NULL DEFAULT false, + ntfy_url STRING NULL DEFAULT '':::STRING, + ntfy_topic STRING NULL DEFAULT '':::STRING, + ntfy_api_token STRING NULL DEFAULT '':::STRING, + ntfy_username STRING NULL DEFAULT '':::STRING, + ntfy_password STRING NULL DEFAULT '':::STRING, + ntfy_icon_url STRING NULL DEFAULT '':::STRING, + cubox_enabled BOOL NULL DEFAULT false, + cubox_api_link STRING NULL DEFAULT '':::STRING, + discord_enabled BOOL NULL DEFAULT false, + discord_webhook_link STRING NULL DEFAULT '':::STRING, + ntfy_internal_links BOOL NULL DEFAULT false, + slack_enabled BOOL NULL DEFAULT false, + slack_webhook_link STRING NULL DEFAULT '':::STRING, + pushover_enabled BOOL NULL DEFAULT false, + pushover_user STRING NULL DEFAULT '':::STRING, + pushover_token STRING NULL DEFAULT '':::STRING, + pushover_device STRING NULL DEFAULT '':::STRING, + pushover_prefix STRING NULL DEFAULT '':::STRING, + rssbridge_token STRING NULL DEFAULT '':::STRING, + karakeep_enabled BOOL NULL DEFAULT false, + karakeep_api_key STRING NULL DEFAULT '':::STRING, + karakeep_url STRING NULL DEFAULT '':::STRING, + CONSTRAINT integrations_pkey PRIMARY KEY (user_id ASC) + ); + CREATE TABLE sessions ( + id STRING NOT NULL, + data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now():::TIMESTAMPTZ, + CONSTRAINT sessions_pkey PRIMARY KEY (id ASC) + ); + CREATE TABLE api_keys ( + id INT8 NOT NULL DEFAULT unique_rowid(), + user_id INT8 NOT NULL, + token STRING NOT NULL, + description STRING NOT NULL, + last_used_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NULL DEFAULT now():::TIMESTAMPTZ, + CONSTRAINT api_keys_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX api_keys_token_key (token ASC), + UNIQUE INDEX api_keys_user_id_description_key (user_id ASC, description ASC) + ); + CREATE TABLE acme_cache ( + key VARCHAR(400) NOT NULL, + data BYTES NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CONSTRAINT acme_cache_pkey PRIMARY KEY (key ASC) + ); + CREATE TABLE webauthn_credentials ( + handle BYTES NOT NULL, + cred_id BYTES NOT NULL, + user_id INT8 NOT NULL, + key BYTES NOT NULL, + attestation_type VARCHAR(255) NOT NULL, + aaguid BYTES NULL, + sign_count INT8 NULL, + clone_warning BOOL NULL, + name STRING NULL, + added_on TIMESTAMPTZ NULL DEFAULT now():::TIMESTAMPTZ, + last_seen_on TIMESTAMPTZ NULL DEFAULT now():::TIMESTAMPTZ, + CONSTRAINT webauthn_credentials_pkey PRIMARY KEY (handle ASC), + UNIQUE INDEX webauthn_credentials_cred_id_key (cred_id ASC) + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE user_sessions ADD CONSTRAINT sessions_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE categories ADD CONSTRAINT categories_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE feeds ADD CONSTRAINT feeds_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE feeds ADD CONSTRAINT feeds_category_id_fkey FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE; + ALTER TABLE entries ADD CONSTRAINT entries_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE entries ADD CONSTRAINT entries_feed_id_fkey FOREIGN KEY (feed_id) REFERENCES feeds(id) ON DELETE CASCADE; + ALTER TABLE enclosures ADD CONSTRAINT enclosures_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE enclosures ADD CONSTRAINT enclosures_entry_id_fkey FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE; + ALTER TABLE feed_icons ADD CONSTRAINT feed_icons_feed_id_fkey FOREIGN KEY (feed_id) REFERENCES feeds(id) ON DELETE CASCADE; + ALTER TABLE feed_icons ADD CONSTRAINT feed_icons_icon_id_fkey FOREIGN KEY (icon_id) REFERENCES icons(id) ON DELETE CASCADE; + ALTER TABLE api_keys ADD CONSTRAINT api_keys_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE webauthn_credentials ADD CONSTRAINT webauthn_credentials_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE user_sessions VALIDATE CONSTRAINT sessions_user_id_fkey; + ALTER TABLE categories VALIDATE CONSTRAINT categories_user_id_fkey; + ALTER TABLE feeds VALIDATE CONSTRAINT feeds_user_id_fkey; + ALTER TABLE feeds VALIDATE CONSTRAINT feeds_category_id_fkey; + ALTER TABLE entries VALIDATE CONSTRAINT entries_user_id_fkey; + ALTER TABLE entries VALIDATE CONSTRAINT entries_feed_id_fkey; + ALTER TABLE enclosures VALIDATE CONSTRAINT enclosures_user_id_fkey; + ALTER TABLE enclosures VALIDATE CONSTRAINT enclosures_entry_id_fkey; + ALTER TABLE feed_icons VALIDATE CONSTRAINT feed_icons_feed_id_fkey; + ALTER TABLE feed_icons VALIDATE CONSTRAINT feed_icons_icon_id_fkey; + ALTER TABLE api_keys VALIDATE CONSTRAINT api_keys_user_id_fkey; + ALTER TABLE webauthn_credentials VALIDATE CONSTRAINT webauthn_credentials_user_id_fkey; + ` + _, err = tx.Exec(sql) + return err + }, +} diff --git a/internal/database/database.go b/internal/database/database.go index 1b3c83df..a861aae6 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -7,13 +7,68 @@ import ( "database/sql" "fmt" "log/slog" + "strings" + "time" ) +type DBKind int + +const ( + DBKindPostgres DBKind = iota + DBKindCockroach + DBKindSqlite +) + +var dbKindProto = map[DBKind]string{ + DBKindPostgres: "postgresql", + DBKindCockroach: "cockroachdb", + DBKindSqlite: "sqlite", +} + +var dbKindDriver = map[DBKind]string{ + DBKindPostgres: "postgres", + DBKindCockroach: "postgres", + DBKindSqlite: "sqlite", +} + +func DetectKind(conn string) (DBKind, error) { + switch { + case strings.HasPrefix(conn, "postgres"), + strings.HasPrefix(conn, "postgresql"): + return DBKindPostgres, nil + case strings.HasPrefix(conn, "cockroach"), + strings.HasPrefix(conn, "cockroachdb"): + return DBKindCockroach, nil + case strings.HasPrefix(conn, "file"), + strings.HasPrefix(conn, "sqlite"): + return DBKindSqlite, nil + default: + return 0, fmt.Errorf("unknown db kind in conn string: %q", conn) + } +} + +type Migration func(*sql.Tx) error + +var dbKindMigrations = map[DBKind][]Migration{ + DBKindPostgres: postgresMigrations, + DBKindCockroach: cockroachMigrations, + DBKindSqlite: sqliteMigrations, +} + +var dbKindSchemaVersion = map[DBKind]int{ + DBKindPostgres: postgresSchemaVersion, + DBKindCockroach: cockroachSchemaVersion, + DBKindSqlite: sqliteSchemaVersion, +} + // Migrate executes database migrations. -func Migrate(db *sql.DB) error { +func Migrate(kind DBKind, db *sql.DB) error { var currentVersion int db.QueryRow(`SELECT version FROM schema_version`).Scan(¤tVersion) + migrations := dbKindMigrations[kind] + schemaVersion := dbKindSchemaVersion[kind] + slog.Info("Running database migrations", slog.Int("current_version", currentVersion), slog.Int("latest_version", schemaVersion), @@ -32,14 +87,24 @@ func Migrate(db *sql.DB) error { return fmt.Errorf("[Migration v%d] %v", newVersion, err) } - if _, err := tx.Exec(`TRUNCATE schema_version`); err != nil { - tx.Rollback() - return fmt.Errorf("[Migration v%d] %v", newVersion, err) - } - - if _, err := tx.Exec(`INSERT INTO schema_version (version) VALUES ($1)`, newVersion); err != nil { - tx.Rollback() - return fmt.Errorf("[Migration v%d] %v", newVersion, err) + if kind == DBKindSqlite { + if _, err := tx.Exec(`DELETE FROM schema_version`); err != nil { + tx.Rollback() + return fmt.Errorf("[Migration v%d] %v", newVersion, err) + } + if _, err := tx.Exec(`INSERT INTO schema_version (version) VALUES (?)`, newVersion); err != nil { + tx.Rollback() + return fmt.Errorf("[Migration v%d] %v", newVersion, err) + } + } else { + if _, err := tx.Exec(`TRUNCATE schema_version`); err != nil { + tx.Rollback() + return fmt.Errorf("[Migration v%d] %v", newVersion, err) + } + if _, err := tx.Exec(`INSERT INTO schema_version (version) VALUES ($1)`, newVersion); err != nil { + tx.Rollback() + return fmt.Errorf("[Migration v%d] %v", newVersion, err) + } } if err := tx.Commit(); err != nil { @@ -51,7 +116,9 @@ func Migrate(db *sql.DB) error { } // IsSchemaUpToDate checks if the database schema is up to date. -func IsSchemaUpToDate(db *sql.DB) error { +func IsSchemaUpToDate(kind DBKind, db *sql.DB) error { + schemaVersion := dbKindSchemaVersion[kind] + var currentVersion int db.QueryRow(`SELECT version FROM schema_version`).Scan(¤tVersion) if currentVersion < schemaVersion { @@ -59,3 +126,25 @@ func IsSchemaUpToDate(db *sql.DB) error { } return nil } + +func NewConnectionPool(kind DBKind, dsn string, minConnections, maxConnections int, connectionLifetime time.Duration) (*sql.DB, error) { + driver := dbKindDriver[kind] + + // replace cockroachdb protocol with postgres + // we use cockroachdb protocol to detect cockroachdb but go wants postgres + if kind == DBKindCockroach { + split := strings.SplitN(dsn, ":", 2) + dsn = fmt.Sprintf("postgres:%s", split[1]) + } + + db, err := sql.Open(driver, dsn) + if err != nil { + return nil, err + } + + db.SetMaxOpenConns(maxConnections) + db.SetMaxIdleConns(minConnections) + db.SetConnMaxLifetime(connectionLifetime) + + return db, nil +} diff --git a/internal/database/migrations.go b/internal/database/migrations.go deleted file mode 100644 index 2f6eaa35..00000000 --- a/internal/database/migrations.go +++ /dev/null @@ -1,1344 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package database // import "miniflux.app/v2/internal/database" - -import ( - "database/sql" - - "miniflux.app/v2/internal/crypto" -) - -var schemaVersion = len(migrations) - -// Order is important. Add new migrations at the end of the list. -var migrations = [...]func(tx *sql.Tx) error{ - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TABLE schema_version ( - version text not null - ); - - CREATE TABLE users ( - id serial not null, - username text not null unique, - password text, - is_admin bool default 'f', - language text default 'en_US', - timezone text default 'UTC', - theme text default 'default', - last_login_at timestamp with time zone, - primary key (id) - ); - - CREATE TABLE sessions ( - id serial not null, - user_id int not null, - token text not null unique, - created_at timestamp with time zone default now(), - user_agent text, - ip text, - primary key (id), - unique (user_id, token), - foreign key (user_id) references users(id) on delete cascade - ); - - CREATE TABLE categories ( - id serial not null, - user_id int not null, - title text not null, - primary key (id), - unique (user_id, title), - foreign key (user_id) references users(id) on delete cascade - ); - - CREATE TABLE feeds ( - id bigserial not null, - user_id int not null, - category_id int not null, - title text not null, - feed_url text not null, - site_url text not null, - checked_at timestamp with time zone default now(), - etag_header text default '', - last_modified_header text default '', - parsing_error_msg text default '', - parsing_error_count int default 0, - primary key (id), - unique (user_id, feed_url), - foreign key (user_id) references users(id) on delete cascade, - foreign key (category_id) references categories(id) on delete cascade - ); - - CREATE TYPE entry_status as enum('unread', 'read', 'removed'); - - CREATE TABLE entries ( - id bigserial not null, - user_id int not null, - feed_id bigint not null, - hash text not null, - published_at timestamp with time zone not null, - title text not null, - url text not null, - author text, - content text, - status entry_status default 'unread', - primary key (id), - unique (feed_id, hash), - foreign key (user_id) references users(id) on delete cascade, - foreign key (feed_id) references feeds(id) on delete cascade - ); - - CREATE INDEX entries_feed_idx on entries using btree(feed_id); - - CREATE TABLE enclosures ( - id bigserial not null, - user_id int not null, - entry_id bigint not null, - url text not null, - size int default 0, - mime_type text default '', - primary key (id), - foreign key (user_id) references users(id) on delete cascade, - foreign key (entry_id) references entries(id) on delete cascade - ); - - CREATE TABLE icons ( - id bigserial not null, - hash text not null unique, - mime_type text not null, - content bytea not null, - primary key (id) - ); - - CREATE TABLE feed_icons ( - feed_id bigint not null, - icon_id bigint not null, - primary key(feed_id, icon_id), - foreign key (feed_id) references feeds(id) on delete cascade, - foreign key (icon_id) references icons(id) on delete cascade - ); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE EXTENSION IF NOT EXISTS hstore; - ALTER TABLE users ADD COLUMN extra hstore; - CREATE INDEX users_extra_idx ON users using gin(extra); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TABLE tokens ( - id text not null, - value text not null, - created_at timestamp with time zone not null default now(), - primary key(id, value) - ); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TYPE entry_sorting_direction AS enum('asc', 'desc'); - ALTER TABLE users ADD COLUMN entry_direction entry_sorting_direction default 'asc'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TABLE integrations ( - user_id int not null, - pinboard_enabled bool default 'f', - pinboard_token text default '', - pinboard_tags text default 'miniflux', - pinboard_mark_as_unread bool default 'f', - instapaper_enabled bool default 'f', - instapaper_username text default '', - instapaper_password text default '', - fever_enabled bool default 'f', - fever_username text default '', - fever_password text default '', - fever_token text default '', - primary key(user_id) - ); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN scraper_rules text default ''` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN rewrite_rules text default ''` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN crawler boolean default 'f'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE sessions rename to user_sessions` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - DROP TABLE tokens; - - CREATE TABLE sessions ( - id text not null, - data jsonb not null, - created_at timestamp with time zone not null default now(), - primary key(id) - ); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN wallabag_enabled bool default 'f', - ADD COLUMN wallabag_url text default '', - ADD COLUMN wallabag_client_id text default '', - ADD COLUMN wallabag_client_secret text default '', - ADD COLUMN wallabag_username text default '', - ADD COLUMN wallabag_password text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE entries ADD COLUMN starred bool default 'f'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE INDEX entries_user_status_idx ON entries(user_id, status); - CREATE INDEX feeds_user_category_idx ON feeds(user_id, category_id); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN nunux_keeper_enabled bool default 'f', - ADD COLUMN nunux_keeper_url text default '', - ADD COLUMN nunux_keeper_api_key text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE enclosures ALTER COLUMN size SET DATA TYPE bigint` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE entries ADD COLUMN comments_url text default ''` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN pocket_enabled bool default 'f', - ADD COLUMN pocket_access_token text default '', - ADD COLUMN pocket_consumer_key text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE user_sessions ALTER COLUMN ip SET DATA TYPE inet using ip::inet; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE feeds - ADD COLUMN username text default '', - ADD COLUMN password text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE entries ADD COLUMN document_vectors tsvector; - UPDATE entries SET document_vectors = to_tsvector(substring(title || ' ' || coalesce(content, '') for 1000000)); - CREATE INDEX document_vectors_idx ON entries USING gin(document_vectors); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN user_agent text default ''` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - UPDATE - entries - SET - document_vectors = setweight(to_tsvector(substring(coalesce(title, '') for 1000000)), 'A') || setweight(to_tsvector(substring(coalesce(content, '') for 1000000)), 'B') - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN keyboard_shortcuts boolean default 't'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN disabled boolean default 'f';` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE users ALTER COLUMN theme SET DEFAULT 'light_serif'; - UPDATE users SET theme='light_serif' WHERE theme='default'; - UPDATE users SET theme='light_sans_serif' WHERE theme='sansserif'; - UPDATE users SET theme='dark_serif' WHERE theme='black'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE entries ADD COLUMN changed_at timestamp with time zone; - UPDATE entries SET changed_at = published_at; - ALTER TABLE entries ALTER COLUMN changed_at SET not null; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TABLE api_keys ( - id serial not null, - user_id int not null references users(id) on delete cascade, - token text not null unique, - description text not null, - last_used_at timestamp with time zone, - created_at timestamp with time zone default now(), - primary key(id), - unique (user_id, description) - ); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE entries ADD COLUMN share_code text not null default ''; - CREATE UNIQUE INDEX entries_share_code_idx ON entries USING btree(share_code) WHERE share_code <> ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `CREATE INDEX enclosures_user_entry_url_idx ON enclosures(user_id, entry_id, md5(url))` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE feeds ADD COLUMN next_check_at timestamp with time zone default now(); - CREATE INDEX entries_user_feed_idx ON entries (user_id, feed_id); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN ignore_http_cache bool default false` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN entries_per_page int default 100` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN show_reading_time boolean default 't'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status)` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN fetch_via_proxy bool default false` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `CREATE INDEX entries_feed_id_status_hash_idx ON entries USING btree (feed_id, status, hash)` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `CREATE INDEX entries_user_id_status_starred_idx ON entries (user_id, status, starred)` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN entry_swipe boolean default 't'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE integrations DROP COLUMN fever_password` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE feeds - ADD COLUMN blocklist_rules text not null default '', - ADD COLUMN keeplist_rules text not null default '' - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE entries ADD COLUMN reading_time int not null default 0` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE entries ADD COLUMN created_at timestamp with time zone not null default now(); - UPDATE entries SET created_at = published_at; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE users - ADD column stylesheet text not null default '', - ADD column google_id text not null default '', - ADD column openid_connect_id text not null default '' - `) - if err != nil { - return err - } - - _, err = tx.Exec(` - DECLARE my_cursor CURSOR FOR - SELECT - id, - COALESCE(extra->'custom_css', '') as custom_css, - COALESCE(extra->'google_id', '') as google_id, - COALESCE(extra->'oidc_id', '') as oidc_id - FROM users - FOR UPDATE - `) - if err != nil { - return err - } - defer tx.Exec("CLOSE my_cursor") - - for { - var ( - userID int64 - customStylesheet string - googleID string - oidcID string - ) - - if err := tx.QueryRow(`FETCH NEXT FROM my_cursor`).Scan(&userID, &customStylesheet, &googleID, &oidcID); err != nil { - if err == sql.ErrNoRows { - break - } - return err - } - - _, err := tx.Exec( - `UPDATE - users - SET - stylesheet=$2, - google_id=$3, - openid_connect_id=$4 - WHERE - id=$1 - `, - userID, customStylesheet, googleID, oidcID) - if err != nil { - return err - } - } - - return err - }, - func(tx *sql.Tx) (err error) { - if _, err = tx.Exec(`ALTER TABLE users DROP COLUMN extra;`); err != nil { - return err - } - _, err = tx.Exec(` - CREATE UNIQUE INDEX users_google_id_idx ON users(google_id) WHERE google_id <> ''; - CREATE UNIQUE INDEX users_openid_connect_id_idx ON users(openid_connect_id) WHERE openid_connect_id <> ''; - `) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - CREATE INDEX entries_feed_url_idx ON entries(feed_id, url) WHERE length(url) < 2000; - CREATE INDEX entries_user_status_feed_idx ON entries(user_id, status, feed_id); - CREATE INDEX entries_user_status_changed_idx ON entries(user_id, status, changed_at); - `) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - CREATE TABLE acme_cache ( - key varchar(400) not null primary key, - data bytea not null, - updated_at timestamptz not null - ); - `) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE feeds ADD COLUMN allow_self_signed_certificates boolean not null default false - `) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TYPE webapp_display_mode AS enum('fullscreen', 'standalone', 'minimal-ui', 'browser'); - ALTER TABLE users ADD COLUMN display_mode webapp_display_mode default 'standalone'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN cookie text default ''` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE categories ADD COLUMN hide_globally boolean not null default false - `) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE feeds ADD COLUMN hide_globally boolean not null default false - `) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN telegram_bot_enabled bool default 'f', - ADD COLUMN telegram_bot_token text default '', - ADD COLUMN telegram_bot_chat_id text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TYPE entry_sorting_order AS enum('published_at', 'created_at'); - ALTER TABLE users ADD COLUMN entry_order entry_sorting_order default 'published_at'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN googlereader_enabled bool default 'f', - ADD COLUMN googlereader_username text default '', - ADD COLUMN googlereader_password text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN espial_enabled bool default 'f', - ADD COLUMN espial_url text default '', - ADD COLUMN espial_api_key text default '', - ADD COLUMN espial_tags text default 'miniflux'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN linkding_enabled bool default 'f', - ADD COLUMN linkding_url text default '', - ADD COLUMN linkding_api_key text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE feeds ADD COLUMN url_rewrite_rules text not null default '' - `) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE users - ADD COLUMN default_reading_speed int default 265, - ADD COLUMN cjk_reading_speed int default 500; - `) - return - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE users ADD COLUMN default_home_page text default 'unread'; - `) - return - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE integrations ADD COLUMN wallabag_only_url bool default 'f'; - `) - return - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE users ADD COLUMN categories_sorting_order text not null default 'unread_count'; - `) - return - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN matrix_bot_enabled bool default 'f', - ADD COLUMN matrix_bot_user text default '', - ADD COLUMN matrix_bot_password text default '', - ADD COLUMN matrix_bot_url text default '', - ADD COLUMN matrix_bot_chat_id text default ''; - ` - _, err = tx.Exec(sql) - return - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN double_tap boolean default 't'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE entries ADD COLUMN tags text[] default '{}'; - `) - return - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE users RENAME double_tap TO gesture_nav; - ALTER TABLE users - ALTER COLUMN gesture_nav SET DATA TYPE text using case when gesture_nav = true then 'tap' when gesture_nav = false then 'none' end, - ALTER COLUMN gesture_nav SET default 'tap'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations ADD COLUMN linkding_tags text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE feeds ADD COLUMN no_media_player boolean default 'f'; - ALTER TABLE enclosures ADD COLUMN media_progression int default 0; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations ADD COLUMN linkding_mark_as_unread bool default 'f'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - // Delete duplicated rows - sql := ` - DELETE FROM enclosures a USING enclosures b - WHERE a.id < b.id - AND a.user_id = b.user_id - AND a.entry_id = b.entry_id - AND a.url = b.url; - ` - _, err = tx.Exec(sql) - if err != nil { - return err - } - - // Remove previous index - _, err = tx.Exec(`DROP INDEX enclosures_user_entry_url_idx`) - if err != nil { - return err - } - - // Create unique index - _, err = tx.Exec(`CREATE UNIQUE INDEX enclosures_user_entry_url_unique_idx ON enclosures(user_id, entry_id, md5(url))`) - if err != nil { - return err - } - - return nil - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN mark_read_on_view boolean default 't'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN notion_enabled bool default 'f', - ADD COLUMN notion_token text default '', - ADD COLUMN notion_page_id text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN readwise_enabled bool default 'f', - ADD COLUMN readwise_api_key text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN apprise_enabled bool default 'f', - ADD COLUMN apprise_url text default '', - ADD COLUMN apprise_services_url text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN shiori_enabled bool default 'f', - ADD COLUMN shiori_url text default '', - ADD COLUMN shiori_username text default '', - ADD COLUMN shiori_password text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN shaarli_enabled bool default 'f', - ADD COLUMN shaarli_url text default '', - ADD COLUMN shaarli_api_secret text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - ALTER TABLE feeds ADD COLUMN apprise_service_urls text default ''; - `) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN webhook_enabled bool default 'f', - ADD COLUMN webhook_url text default '', - ADD COLUMN webhook_secret text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN telegram_bot_topic_id int, - ADD COLUMN telegram_bot_disable_web_page_preview bool default 'f', - ADD COLUMN telegram_bot_disable_notification bool default 'f'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations ADD COLUMN telegram_bot_disable_buttons bool default 'f'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - -- Speed up has_enclosure - CREATE INDEX enclosures_entry_id_idx ON enclosures(entry_id); - - -- Speed up unread page - CREATE INDEX entries_user_status_published_idx ON entries(user_id, status, published_at); - CREATE INDEX entries_user_status_created_idx ON entries(user_id, status, created_at); - CREATE INDEX feeds_feed_id_hide_globally_idx ON feeds(id, hide_globally); - - -- Speed up history page - CREATE INDEX entries_user_status_changed_published_idx ON entries(user_id, status, changed_at, published_at); - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN rssbridge_enabled bool default 'f', - ADD COLUMN rssbridge_url text default ''; - ` - _, err = tx.Exec(sql) - return - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - CREATE TABLE webauthn_credentials ( - handle bytea primary key, - cred_id bytea unique not null, - user_id int references users(id) on delete cascade not null, - public_key bytea not null, - attestation_type varchar(255) not null, - aaguid bytea, - sign_count bigint, - clone_warning bool, - name text, - added_on timestamp with time zone default now(), - last_seen_on timestamp with time zone default now() - ); - `) - return - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN omnivore_enabled bool default 'f', - ADD COLUMN omnivore_api_key text default '', - ADD COLUMN omnivore_url text default ''; - ` - _, err = tx.Exec(sql) - return - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN linkace_enabled bool default 'f', - ADD COLUMN linkace_url text default '', - ADD COLUMN linkace_api_key text default '', - ADD COLUMN linkace_tags text default '', - ADD COLUMN linkace_is_private bool default 't', - ADD COLUMN linkace_check_disabled bool default 't'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN linkwarden_enabled bool default 'f', - ADD COLUMN linkwarden_url text default '', - ADD COLUMN linkwarden_api_key text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN readeck_enabled bool default 'f', - ADD COLUMN readeck_only_url bool default 'f', - ADD COLUMN readeck_url text default '', - ADD COLUMN readeck_api_key text default '', - ADD COLUMN readeck_labels text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN disable_http2 bool default 'f'` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN media_playback_rate numeric default 1;` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - // the WHERE part speed-up the request a lot - sql := `UPDATE entries SET tags = array_remove(tags, '') WHERE '' = ANY(tags);` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - // Entry URLs can exceeds btree maximum size - // Checking entry existence is now using entries_feed_id_status_hash_idx index - _, err = tx.Exec(`DROP INDEX entries_feed_url_idx`) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN raindrop_enabled bool default 'f', - ADD COLUMN raindrop_token text default '', - ADD COLUMN raindrop_collection_id text default '', - ADD COLUMN raindrop_tags text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE feeds ADD COLUMN description text default ''` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE users - ADD COLUMN block_filter_entry_rules text not null default '', - ADD COLUMN keep_filter_entry_rules text not null default '' - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN betula_url text default '', - ADD COLUMN betula_token text default '', - ADD COLUMN betula_enabled bool default 'f'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN ntfy_enabled bool default 'f', - ADD COLUMN ntfy_url text default '', - ADD COLUMN ntfy_topic text default '', - ADD COLUMN ntfy_api_token text default '', - ADD COLUMN ntfy_username text default '', - ADD COLUMN ntfy_password text default '', - ADD COLUMN ntfy_icon_url text default ''; - - ALTER TABLE feeds - ADD COLUMN ntfy_enabled bool default 'f', - ADD COLUMN ntfy_priority int default '3'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN mark_read_on_media_player_completion bool default 'f';` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN custom_js text not null default '';` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE users ADD COLUMN external_font_hosts text not null default '';` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN cubox_enabled bool default 'f', - ADD COLUMN cubox_api_link text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN discord_enabled bool default 'f', - ADD COLUMN discord_webhook_link text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := `ALTER TABLE integrations ADD COLUMN ntfy_internal_links bool default 'f';` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN slack_enabled bool default 'f', - ADD COLUMN slack_webhook_link text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(`ALTER TABLE feeds ADD COLUMN webhook_url text default '';`) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN pushover_enabled bool default 'f', - ADD COLUMN pushover_user text default '', - ADD COLUMN pushover_token text default '', - ADD COLUMN pushover_device text default '', - ADD COLUMN pushover_prefix text default ''; - - ALTER TABLE feeds - ADD COLUMN pushover_enabled bool default 'f', - ADD COLUMN pushover_priority int default '0'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE feeds ADD COLUMN ntfy_topic text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE icons ADD COLUMN external_id text default ''; - CREATE UNIQUE INDEX icons_external_id_idx ON icons USING btree(external_id) WHERE external_id <> ''; - ` - _, err = tx.Exec(sql) - - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(` - DECLARE id_cursor CURSOR FOR - SELECT - id - FROM icons - WHERE external_id = '' - FOR UPDATE`) - if err != nil { - return err - } - defer tx.Exec("CLOSE id_cursor") - - for { - var id int64 - - if err := tx.QueryRow(`FETCH NEXT FROM id_cursor`).Scan(&id); err != nil { - if err == sql.ErrNoRows { - break - } - return err - } - - _, err = tx.Exec( - ` - UPDATE icons SET external_id = $1 WHERE id = $2 - `, - crypto.GenerateRandomStringHex(20), id) - - if err != nil { - return err - } - } - - return nil - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(`ALTER TABLE feeds ADD COLUMN proxy_url text default ''`) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations ADD COLUMN rssbridge_token text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(`ALTER TABLE users ADD COLUMN always_open_external_links bool default 'f'`) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - ADD COLUMN karakeep_enabled bool default 'f', - ADD COLUMN karakeep_api_key text default '', - ADD COLUMN karakeep_url text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - _, err = tx.Exec(`ALTER TABLE users ADD COLUMN open_external_links_in_new_tab bool default 't'`) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations - DROP COLUMN pocket_enabled, - DROP COLUMN pocket_access_token, - DROP COLUMN pocket_consumer_key; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE feeds - ADD COLUMN block_filter_entry_rules text not null default '', - ADD COLUMN keep_filter_entry_rules text not null default '' - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - CREATE TYPE linktaco_link_visibility AS ENUM ( - 'PUBLIC', - 'PRIVATE' - ); - ALTER TABLE integrations - ADD COLUMN linktaco_enabled bool default 'f', - ADD COLUMN linktaco_api_token text default '', - ADD COLUMN linktaco_org_slug text default '', - ADD COLUMN linktaco_tags text default '', - ADD COLUMN linktaco_visibility linktaco_link_visibility default 'PUBLIC'; - ` - _, err = tx.Exec(sql) - return err - }, - func(tx *sql.Tx) (err error) { - sql := ` - ALTER TABLE integrations ADD COLUMN wallabag_tags text default ''; - ` - _, err = tx.Exec(sql) - return err - }, - // This migration replaces deprecated timezones by their equivalent on Debian Trixie. - func(tx *sql.Tx) (err error) { - var deprecatedTimeZoneMap = map[string]string{ - // Africa - "Africa/Asmera": "Africa/Asmara", - - // America - Argentina - "America/Argentina/ComodRivadavia": "America/Argentina/Catamarca", - "America/Buenos_Aires": "America/Argentina/Buenos_Aires", - "America/Catamarca": "America/Argentina/Catamarca", - "America/Cordoba": "America/Argentina/Cordoba", - "America/Jujuy": "America/Argentina/Jujuy", - "America/Mendoza": "America/Argentina/Mendoza", - "America/Rosario": "America/Argentina/Cordoba", - - // America - US - "America/Fort_Wayne": "America/Indiana/Indianapolis", - "America/Indianapolis": "America/Indiana/Indianapolis", - "America/Knox_IN": "America/Indiana/Knox", - "America/Louisville": "America/Kentucky/Louisville", - - // America - Greenland - "America/Godthab": "America/Nuuk", - - // Antarctica - "Antarctica/South_Pole": "Pacific/Auckland", - - // Asia - "Asia/Ashkhabad": "Asia/Ashgabat", - "Asia/Calcutta": "Asia/Kolkata", - "Asia/Choibalsan": "Asia/Ulaanbaatar", - "Asia/Chungking": "Asia/Chongqing", - "Asia/Dacca": "Asia/Dhaka", - "Asia/Katmandu": "Asia/Kathmandu", - "Asia/Macao": "Asia/Macau", - "Asia/Rangoon": "Asia/Yangon", - "Asia/Saigon": "Asia/Ho_Chi_Minh", - "Asia/Thimbu": "Asia/Thimphu", - "Asia/Ujung_Pandang": "Asia/Makassar", - "Asia/Ulan_Bator": "Asia/Ulaanbaatar", - - // Atlantic - "Atlantic/Faeroe": "Atlantic/Faroe", - - // Australia - "Australia/ACT": "Australia/Sydney", - "Australia/LHI": "Australia/Lord_Howe", - "Australia/North": "Australia/Darwin", - "Australia/NSW": "Australia/Sydney", - "Australia/Queensland": "Australia/Brisbane", - "Australia/South": "Australia/Adelaide", - "Australia/Tasmania": "Australia/Hobart", - "Australia/Victoria": "Australia/Melbourne", - "Australia/West": "Australia/Perth", - - // Brazil - "Brazil/Acre": "America/Rio_Branco", - "Brazil/DeNoronha": "America/Noronha", - "Brazil/East": "America/Sao_Paulo", - "Brazil/West": "America/Manaus", - - // Canada - "Canada/Atlantic": "America/Halifax", - "Canada/Central": "America/Winnipeg", - "Canada/Eastern": "America/Toronto", - "Canada/Mountain": "America/Edmonton", - "Canada/Newfoundland": "America/St_Johns", - "Canada/Pacific": "America/Vancouver", - "Canada/Saskatchewan": "America/Regina", - "Canada/Yukon": "America/Whitehorse", - - // Europe - "CET": "Europe/Paris", - "EET": "Europe/Sofia", - "Europe/Kiev": "Europe/Kyiv", - "Europe/Uzhgorod": "Europe/Kyiv", - "Europe/Zaporozhye": "Europe/Kyiv", - "MET": "Europe/Paris", - "WET": "Europe/Lisbon", - - // Chile - "Chile/Continental": "America/Santiago", - "Chile/EasterIsland": "Pacific/Easter", - - // Fixed offset and generic zones - "CST6CDT": "America/Chicago", - "EST": "America/New_York", - "EST5EDT": "America/New_York", - "HST": "Pacific/Honolulu", - "MST": "America/Denver", - "MST7MDT": "America/Denver", - "PST8PDT": "America/Los_Angeles", - - // Countries/Regions - "Cuba": "America/Havana", - "Egypt": "Africa/Cairo", - "Eire": "Europe/Dublin", - "GB": "Europe/London", - "GB-Eire": "Europe/London", - "Hongkong": "Asia/Hong_Kong", - "Iceland": "Atlantic/Reykjavik", - "Iran": "Asia/Tehran", - "Israel": "Asia/Jerusalem", - "Jamaica": "America/Jamaica", - "Japan": "Asia/Tokyo", - "Libya": "Africa/Tripoli", - "Poland": "Europe/Warsaw", - "Portugal": "Europe/Lisbon", - "PRC": "Asia/Shanghai", - "ROC": "Asia/Taipei", - "ROK": "Asia/Seoul", - "Singapore": "Asia/Singapore", - "Turkey": "Europe/Istanbul", - - // GMT variations - "GMT+0": "GMT", - "GMT-0": "GMT", - "GMT0": "GMT", - "Greenwich": "GMT", - "UCT": "UTC", - "Universal": "UTC", - "Zulu": "UTC", - - // Mexico - "Mexico/BajaNorte": "America/Tijuana", - "Mexico/BajaSur": "America/Mazatlan", - "Mexico/General": "America/Mexico_City", - - // US zones - "Navajo": "America/Denver", - "US/Alaska": "America/Anchorage", - "US/Aleutian": "America/Adak", - "US/Arizona": "America/Phoenix", - "US/Central": "America/Chicago", - "US/Eastern": "America/New_York", - "US/East-Indiana": "America/Indiana/Indianapolis", - "US/Hawaii": "Pacific/Honolulu", - "US/Indiana-Starke": "America/Indiana/Knox", - "US/Michigan": "America/Detroit", - "US/Mountain": "America/Denver", - "US/Pacific": "America/Los_Angeles", - "US/Samoa": "Pacific/Pago_Pago", - - // Pacific - "Kwajalein": "Pacific/Kwajalein", - "NZ": "Pacific/Auckland", - "NZ-CHAT": "Pacific/Chatham", - "Pacific/Enderbury": "Pacific/Kanton", - "Pacific/Ponape": "Pacific/Pohnpei", - "Pacific/Truk": "Pacific/Chuuk", - - // Special cases - "Factory": "UTC", // Factory is used for unconfigured systems - "W-SU": "Europe/Moscow", - } - - // Loop through each user and correct the timezone - rows, err := tx.Query(`SELECT id, timezone FROM users`) - if err != nil { - return err - } - - userTimezoneMap := make(map[int64]string) - for rows.Next() { - var userID int64 - var userTimezone string - if err := rows.Scan(&userID, &userTimezone); err != nil { - return err - } - userTimezoneMap[userID] = userTimezone - } - rows.Close() - - for userID, userTimezone := range userTimezoneMap { - if newTimezone, found := deprecatedTimeZoneMap[userTimezone]; found { - if _, err := tx.Exec(`UPDATE users SET timezone = $1 WHERE id = $2`, newTimezone, userID); err != nil { - return err - } - } - } - - return nil - }, -} diff --git a/internal/database/postgresql.go b/internal/database/postgresql.go index df2ace59..c9b02ba0 100644 --- a/internal/database/postgresql.go +++ b/internal/database/postgresql.go @@ -5,21 +5,1341 @@ package database // import "miniflux.app/v2/internal/database" import ( "database/sql" - "time" + + "miniflux.app/v2/internal/crypto" _ "github.com/lib/pq" ) -// NewConnectionPool configures the database connection pool. -func NewConnectionPool(dsn string, minConnections, maxConnections int, connectionLifetime time.Duration) (*sql.DB, error) { - db, err := sql.Open("postgres", dsn) - if err != nil { - return nil, err - } +var postgresSchemaVersion = len(postgresMigrations) - db.SetMaxOpenConns(maxConnections) - db.SetMaxIdleConns(minConnections) - db.SetConnMaxLifetime(connectionLifetime) +// Order is important. Add new migrations at the end of the list. +var postgresMigrations = []Migration{ + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TABLE schema_version ( + version text not null + ); - return db, nil + CREATE TABLE users ( + id serial not null, + username text not null unique, + password text, + is_admin bool default 'f', + language text default 'en_US', + timezone text default 'UTC', + theme text default 'default', + last_login_at timestamp with time zone, + primary key (id) + ); + + CREATE TABLE sessions ( + id serial not null, + user_id int not null, + token text not null unique, + created_at timestamp with time zone default now(), + user_agent text, + ip text, + primary key (id), + unique (user_id, token), + foreign key (user_id) references users(id) on delete cascade + ); + + CREATE TABLE categories ( + id serial not null, + user_id int not null, + title text not null, + primary key (id), + unique (user_id, title), + foreign key (user_id) references users(id) on delete cascade + ); + + CREATE TABLE feeds ( + id bigserial not null, + user_id int not null, + category_id int not null, + title text not null, + feed_url text not null, + site_url text not null, + checked_at timestamp with time zone default now(), + etag_header text default '', + last_modified_header text default '', + parsing_error_msg text default '', + parsing_error_count int default 0, + primary key (id), + unique (user_id, feed_url), + foreign key (user_id) references users(id) on delete cascade, + foreign key (category_id) references categories(id) on delete cascade + ); + + CREATE TYPE entry_status as enum('unread', 'read', 'removed'); + + CREATE TABLE entries ( + id bigserial not null, + user_id int not null, + feed_id bigint not null, + hash text not null, + published_at timestamp with time zone not null, + title text not null, + url text not null, + author text, + content text, + status entry_status default 'unread', + primary key (id), + unique (feed_id, hash), + foreign key (user_id) references users(id) on delete cascade, + foreign key (feed_id) references feeds(id) on delete cascade + ); + + CREATE INDEX entries_feed_idx on entries using btree(feed_id); + + CREATE TABLE enclosures ( + id bigserial not null, + user_id int not null, + entry_id bigint not null, + url text not null, + size int default 0, + mime_type text default '', + primary key (id), + foreign key (user_id) references users(id) on delete cascade, + foreign key (entry_id) references entries(id) on delete cascade + ); + + CREATE TABLE icons ( + id bigserial not null, + hash text not null unique, + mime_type text not null, + content bytea not null, + primary key (id) + ); + + CREATE TABLE feed_icons ( + feed_id bigint not null, + icon_id bigint not null, + primary key(feed_id, icon_id), + foreign key (feed_id) references feeds(id) on delete cascade, + foreign key (icon_id) references icons(id) on delete cascade + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE EXTENSION IF NOT EXISTS hstore; + ALTER TABLE users ADD COLUMN extra hstore; + CREATE INDEX users_extra_idx ON users using gin(extra); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TABLE tokens ( + id text not null, + value text not null, + created_at timestamp with time zone not null default now(), + primary key(id, value) + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TYPE entry_sorting_direction AS enum('asc', 'desc'); + ALTER TABLE users ADD COLUMN entry_direction entry_sorting_direction default 'asc'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TABLE integrations ( + user_id int not null, + pinboard_enabled bool default 'f', + pinboard_token text default '', + pinboard_tags text default 'miniflux', + pinboard_mark_as_unread bool default 'f', + instapaper_enabled bool default 'f', + instapaper_username text default '', + instapaper_password text default '', + fever_enabled bool default 'f', + fever_username text default '', + fever_password text default '', + fever_token text default '', + primary key(user_id) + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN scraper_rules text default ''` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN rewrite_rules text default ''` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN crawler boolean default 'f'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE sessions rename to user_sessions` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + DROP TABLE tokens; + + CREATE TABLE sessions ( + id text not null, + data jsonb not null, + created_at timestamp with time zone not null default now(), + primary key(id) + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN wallabag_enabled bool default 'f', + ADD COLUMN wallabag_url text default '', + ADD COLUMN wallabag_client_id text default '', + ADD COLUMN wallabag_client_secret text default '', + ADD COLUMN wallabag_username text default '', + ADD COLUMN wallabag_password text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE entries ADD COLUMN starred bool default 'f'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE INDEX entries_user_status_idx ON entries(user_id, status); + CREATE INDEX feeds_user_category_idx ON feeds(user_id, category_id); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN nunux_keeper_enabled bool default 'f', + ADD COLUMN nunux_keeper_url text default '', + ADD COLUMN nunux_keeper_api_key text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE enclosures ALTER COLUMN size SET DATA TYPE bigint` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE entries ADD COLUMN comments_url text default ''` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN pocket_enabled bool default 'f', + ADD COLUMN pocket_access_token text default '', + ADD COLUMN pocket_consumer_key text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE user_sessions ALTER COLUMN ip SET DATA TYPE inet using ip::inet; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds + ADD COLUMN username text default '', + ADD COLUMN password text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE entries ADD COLUMN document_vectors tsvector; + UPDATE entries SET document_vectors = to_tsvector(substring(title || ' ' || coalesce(content, '') for 1000000)); + CREATE INDEX document_vectors_idx ON entries USING gin(document_vectors); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN user_agent text default ''` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + UPDATE + entries + SET + document_vectors = setweight(to_tsvector(substring(coalesce(title, '') for 1000000)), 'A') || setweight(to_tsvector(substring(coalesce(content, '') for 1000000)), 'B') + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN keyboard_shortcuts boolean default 't'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN disabled boolean default 'f';` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE users ALTER COLUMN theme SET DEFAULT 'light_serif'; + UPDATE users SET theme='light_serif' WHERE theme='default'; + UPDATE users SET theme='light_sans_serif' WHERE theme='sansserif'; + UPDATE users SET theme='dark_serif' WHERE theme='black'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE entries ADD COLUMN changed_at timestamp with time zone; + UPDATE entries SET changed_at = published_at; + ALTER TABLE entries ALTER COLUMN changed_at SET not null; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TABLE api_keys ( + id serial not null, + user_id int not null references users(id) on delete cascade, + token text not null unique, + description text not null, + last_used_at timestamp with time zone, + created_at timestamp with time zone default now(), + primary key(id), + unique (user_id, description) + ); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE entries ADD COLUMN share_code text not null default ''; + CREATE UNIQUE INDEX entries_share_code_idx ON entries USING btree(share_code) WHERE share_code <> ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `CREATE INDEX enclosures_user_entry_url_idx ON enclosures(user_id, entry_id, md5(url))` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds ADD COLUMN next_check_at timestamp with time zone default now(); + CREATE INDEX entries_user_feed_idx ON entries (user_id, feed_id); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN ignore_http_cache bool default false` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN entries_per_page int default 100` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN show_reading_time boolean default 't'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status)` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN fetch_via_proxy bool default false` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `CREATE INDEX entries_feed_id_status_hash_idx ON entries USING btree (feed_id, status, hash)` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `CREATE INDEX entries_user_id_status_starred_idx ON entries (user_id, status, starred)` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN entry_swipe boolean default 't'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE integrations DROP COLUMN fever_password` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds + ADD COLUMN blocklist_rules text not null default '', + ADD COLUMN keeplist_rules text not null default '' + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE entries ADD COLUMN reading_time int not null default 0` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE entries ADD COLUMN created_at timestamp with time zone not null default now(); + UPDATE entries SET created_at = published_at; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE users + ADD column stylesheet text not null default '', + ADD column google_id text not null default '', + ADD column openid_connect_id text not null default '' + `) + if err != nil { + return err + } + + _, err = tx.Exec(` + DECLARE my_cursor CURSOR FOR + SELECT + id, + COALESCE(extra->'custom_css', '') as custom_css, + COALESCE(extra->'google_id', '') as google_id, + COALESCE(extra->'oidc_id', '') as oidc_id + FROM users + FOR UPDATE + `) + if err != nil { + return err + } + defer tx.Exec("CLOSE my_cursor") + + for { + var ( + userID int64 + customStylesheet string + googleID string + oidcID string + ) + + if err := tx.QueryRow(`FETCH NEXT FROM my_cursor`).Scan(&userID, &customStylesheet, &googleID, &oidcID); err != nil { + if err == sql.ErrNoRows { + break + } + return err + } + + _, err := tx.Exec( + `UPDATE + users + SET + stylesheet=$2, + google_id=$3, + openid_connect_id=$4 + WHERE + id=$1 + `, + userID, customStylesheet, googleID, oidcID) + if err != nil { + return err + } + } + + return err + }, + func(tx *sql.Tx) (err error) { + if _, err = tx.Exec(`ALTER TABLE users DROP COLUMN extra;`); err != nil { + return err + } + _, err = tx.Exec(` + CREATE UNIQUE INDEX users_google_id_idx ON users(google_id) WHERE google_id <> ''; + CREATE UNIQUE INDEX users_openid_connect_id_idx ON users(openid_connect_id) WHERE openid_connect_id <> ''; + `) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + CREATE INDEX entries_feed_url_idx ON entries(feed_id, url) WHERE length(url) < 2000; + CREATE INDEX entries_user_status_feed_idx ON entries(user_id, status, feed_id); + CREATE INDEX entries_user_status_changed_idx ON entries(user_id, status, changed_at); + `) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + CREATE TABLE acme_cache ( + key varchar(400) not null primary key, + data bytea not null, + updated_at timestamptz not null + ); + `) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE feeds ADD COLUMN allow_self_signed_certificates boolean not null default false + `) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TYPE webapp_display_mode AS enum('fullscreen', 'standalone', 'minimal-ui', 'browser'); + ALTER TABLE users ADD COLUMN display_mode webapp_display_mode default 'standalone'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN cookie text default ''` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE categories ADD COLUMN hide_globally boolean not null default false + `) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE feeds ADD COLUMN hide_globally boolean not null default false + `) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN telegram_bot_enabled bool default 'f', + ADD COLUMN telegram_bot_token text default '', + ADD COLUMN telegram_bot_chat_id text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TYPE entry_sorting_order AS enum('published_at', 'created_at'); + ALTER TABLE users ADD COLUMN entry_order entry_sorting_order default 'published_at'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN googlereader_enabled bool default 'f', + ADD COLUMN googlereader_username text default '', + ADD COLUMN googlereader_password text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN espial_enabled bool default 'f', + ADD COLUMN espial_url text default '', + ADD COLUMN espial_api_key text default '', + ADD COLUMN espial_tags text default 'miniflux'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN linkding_enabled bool default 'f', + ADD COLUMN linkding_url text default '', + ADD COLUMN linkding_api_key text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE feeds ADD COLUMN url_rewrite_rules text not null default '' + `) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE users + ADD COLUMN default_reading_speed int default 265, + ADD COLUMN cjk_reading_speed int default 500; + `) + return + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE users ADD COLUMN default_home_page text default 'unread'; + `) + return + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE integrations ADD COLUMN wallabag_only_url bool default 'f'; + `) + return + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE users ADD COLUMN categories_sorting_order text not null default 'unread_count'; + `) + return + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN matrix_bot_enabled bool default 'f', + ADD COLUMN matrix_bot_user text default '', + ADD COLUMN matrix_bot_password text default '', + ADD COLUMN matrix_bot_url text default '', + ADD COLUMN matrix_bot_chat_id text default ''; + ` + _, err = tx.Exec(sql) + return + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN double_tap boolean default 't'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE entries ADD COLUMN tags text[] default '{}'; + `) + return + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE users RENAME double_tap TO gesture_nav; + ALTER TABLE users + ALTER COLUMN gesture_nav SET DATA TYPE text using case when gesture_nav = true then 'tap' when gesture_nav = false then 'none' end, + ALTER COLUMN gesture_nav SET default 'tap'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations ADD COLUMN linkding_tags text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds ADD COLUMN no_media_player boolean default 'f'; + ALTER TABLE enclosures ADD COLUMN media_progression int default 0; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations ADD COLUMN linkding_mark_as_unread bool default 'f'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + // Delete duplicated rows + sql := ` + DELETE FROM enclosures a USING enclosures b + WHERE a.id < b.id + AND a.user_id = b.user_id + AND a.entry_id = b.entry_id + AND a.url = b.url; + ` + _, err = tx.Exec(sql) + if err != nil { + return err + } + + // Remove previous index + _, err = tx.Exec(`DROP INDEX enclosures_user_entry_url_idx`) + if err != nil { + return err + } + + // Create unique index + _, err = tx.Exec(`CREATE UNIQUE INDEX enclosures_user_entry_url_unique_idx ON enclosures(user_id, entry_id, md5(url))`) + if err != nil { + return err + } + + return nil + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN mark_read_on_view boolean default 't'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN notion_enabled bool default 'f', + ADD COLUMN notion_token text default '', + ADD COLUMN notion_page_id text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN readwise_enabled bool default 'f', + ADD COLUMN readwise_api_key text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN apprise_enabled bool default 'f', + ADD COLUMN apprise_url text default '', + ADD COLUMN apprise_services_url text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN shiori_enabled bool default 'f', + ADD COLUMN shiori_url text default '', + ADD COLUMN shiori_username text default '', + ADD COLUMN shiori_password text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN shaarli_enabled bool default 'f', + ADD COLUMN shaarli_url text default '', + ADD COLUMN shaarli_api_secret text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE feeds ADD COLUMN apprise_service_urls text default ''; + `) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN webhook_enabled bool default 'f', + ADD COLUMN webhook_url text default '', + ADD COLUMN webhook_secret text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN telegram_bot_topic_id int, + ADD COLUMN telegram_bot_disable_web_page_preview bool default 'f', + ADD COLUMN telegram_bot_disable_notification bool default 'f'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations ADD COLUMN telegram_bot_disable_buttons bool default 'f'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + -- Speed up has_enclosure + CREATE INDEX enclosures_entry_id_idx ON enclosures(entry_id); + + -- Speed up unread page + CREATE INDEX entries_user_status_published_idx ON entries(user_id, status, published_at); + CREATE INDEX entries_user_status_created_idx ON entries(user_id, status, created_at); + CREATE INDEX feeds_feed_id_hide_globally_idx ON feeds(id, hide_globally); + + -- Speed up history page + CREATE INDEX entries_user_status_changed_published_idx ON entries(user_id, status, changed_at, published_at); + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN rssbridge_enabled bool default 'f', + ADD COLUMN rssbridge_url text default ''; + ` + _, err = tx.Exec(sql) + return + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + CREATE TABLE webauthn_credentials ( + handle bytea primary key, + cred_id bytea unique not null, + user_id int references users(id) on delete cascade not null, + public_key bytea not null, + attestation_type varchar(255) not null, + aaguid bytea, + sign_count bigint, + clone_warning bool, + name text, + added_on timestamp with time zone default now(), + last_seen_on timestamp with time zone default now() + ); + `) + return + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN omnivore_enabled bool default 'f', + ADD COLUMN omnivore_api_key text default '', + ADD COLUMN omnivore_url text default ''; + ` + _, err = tx.Exec(sql) + return + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN linkace_enabled bool default 'f', + ADD COLUMN linkace_url text default '', + ADD COLUMN linkace_api_key text default '', + ADD COLUMN linkace_tags text default '', + ADD COLUMN linkace_is_private bool default 't', + ADD COLUMN linkace_check_disabled bool default 't'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN linkwarden_enabled bool default 'f', + ADD COLUMN linkwarden_url text default '', + ADD COLUMN linkwarden_api_key text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN readeck_enabled bool default 'f', + ADD COLUMN readeck_only_url bool default 'f', + ADD COLUMN readeck_url text default '', + ADD COLUMN readeck_api_key text default '', + ADD COLUMN readeck_labels text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN disable_http2 bool default 'f'` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN media_playback_rate numeric default 1;` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + // the WHERE part speed-up the request a lot + sql := `UPDATE entries SET tags = array_remove(tags, '') WHERE '' = ANY(tags);` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + // Entry URLs can exceeds btree maximum size + // Checking entry existence is now using entries_feed_id_status_hash_idx index + _, err = tx.Exec(`DROP INDEX entries_feed_url_idx`) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN raindrop_enabled bool default 'f', + ADD COLUMN raindrop_token text default '', + ADD COLUMN raindrop_collection_id text default '', + ADD COLUMN raindrop_tags text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE feeds ADD COLUMN description text default ''` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE users + ADD COLUMN block_filter_entry_rules text not null default '', + ADD COLUMN keep_filter_entry_rules text not null default '' + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN betula_url text default '', + ADD COLUMN betula_token text default '', + ADD COLUMN betula_enabled bool default 'f'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN ntfy_enabled bool default 'f', + ADD COLUMN ntfy_url text default '', + ADD COLUMN ntfy_topic text default '', + ADD COLUMN ntfy_api_token text default '', + ADD COLUMN ntfy_username text default '', + ADD COLUMN ntfy_password text default '', + ADD COLUMN ntfy_icon_url text default ''; + + ALTER TABLE feeds + ADD COLUMN ntfy_enabled bool default 'f', + ADD COLUMN ntfy_priority int default '3'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN mark_read_on_media_player_completion bool default 'f';` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN custom_js text not null default '';` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN external_font_hosts text not null default '';` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN cubox_enabled bool default 'f', + ADD COLUMN cubox_api_link text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN discord_enabled bool default 'f', + ADD COLUMN discord_webhook_link text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE integrations ADD COLUMN ntfy_internal_links bool default 'f';` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN slack_enabled bool default 'f', + ADD COLUMN slack_webhook_link text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(`ALTER TABLE feeds ADD COLUMN webhook_url text default '';`) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN pushover_enabled bool default 'f', + ADD COLUMN pushover_user text default '', + ADD COLUMN pushover_token text default '', + ADD COLUMN pushover_device text default '', + ADD COLUMN pushover_prefix text default ''; + + ALTER TABLE feeds + ADD COLUMN pushover_enabled bool default 'f', + ADD COLUMN pushover_priority int default '0'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds ADD COLUMN ntfy_topic text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE icons ADD COLUMN external_id text default ''; + CREATE UNIQUE INDEX icons_external_id_idx ON icons USING btree(external_id) WHERE external_id <> ''; + ` + _, err = tx.Exec(sql) + + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + DECLARE id_cursor CURSOR FOR + SELECT + id + FROM icons + WHERE external_id = '' + FOR UPDATE`) + if err != nil { + return err + } + defer tx.Exec("CLOSE id_cursor") + + for { + var id int64 + + if err := tx.QueryRow(`FETCH NEXT FROM id_cursor`).Scan(&id); err != nil { + if err == sql.ErrNoRows { + break + } + return err + } + + _, err = tx.Exec( + ` + UPDATE icons SET external_id = $1 WHERE id = $2 + `, + crypto.GenerateRandomStringHex(20), id) + if err != nil { + return err + } + } + + return nil + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(`ALTER TABLE feeds ADD COLUMN proxy_url text default ''`) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations ADD COLUMN rssbridge_token text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(`ALTER TABLE users ADD COLUMN always_open_external_links bool default 'f'`) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + ADD COLUMN karakeep_enabled bool default 'f', + ADD COLUMN karakeep_api_key text default '', + ADD COLUMN karakeep_url text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(`ALTER TABLE users ADD COLUMN open_external_links_in_new_tab bool default 't'`) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations + DROP COLUMN pocket_enabled, + DROP COLUMN pocket_access_token, + DROP COLUMN pocket_consumer_key; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds + ADD COLUMN block_filter_entry_rules text not null default '', + ADD COLUMN keep_filter_entry_rules text not null default '' + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + CREATE TYPE linktaco_link_visibility AS ENUM ( + 'PUBLIC', + 'PRIVATE' + ); + ALTER TABLE integrations + ADD COLUMN linktaco_enabled bool default 'f', + ADD COLUMN linktaco_api_token text default '', + ADD COLUMN linktaco_org_slug text default '', + ADD COLUMN linktaco_tags text default '', + ADD COLUMN linktaco_visibility linktaco_link_visibility default 'PUBLIC'; + ` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE integrations ADD COLUMN wallabag_tags text default ''; + ` + _, err = tx.Exec(sql) + return err + }, + // This migration replaces deprecated timezones by their equivalent on Debian Trixie. + func(tx *sql.Tx) (err error) { + deprecatedTimeZoneMap := map[string]string{ + // Africa + "Africa/Asmera": "Africa/Asmara", + + // America - Argentina + "America/Argentina/ComodRivadavia": "America/Argentina/Catamarca", + "America/Buenos_Aires": "America/Argentina/Buenos_Aires", + "America/Catamarca": "America/Argentina/Catamarca", + "America/Cordoba": "America/Argentina/Cordoba", + "America/Jujuy": "America/Argentina/Jujuy", + "America/Mendoza": "America/Argentina/Mendoza", + "America/Rosario": "America/Argentina/Cordoba", + + // America - US + "America/Fort_Wayne": "America/Indiana/Indianapolis", + "America/Indianapolis": "America/Indiana/Indianapolis", + "America/Knox_IN": "America/Indiana/Knox", + "America/Louisville": "America/Kentucky/Louisville", + + // America - Greenland + "America/Godthab": "America/Nuuk", + + // Antarctica + "Antarctica/South_Pole": "Pacific/Auckland", + + // Asia + "Asia/Ashkhabad": "Asia/Ashgabat", + "Asia/Calcutta": "Asia/Kolkata", + "Asia/Choibalsan": "Asia/Ulaanbaatar", + "Asia/Chungking": "Asia/Chongqing", + "Asia/Dacca": "Asia/Dhaka", + "Asia/Katmandu": "Asia/Kathmandu", + "Asia/Macao": "Asia/Macau", + "Asia/Rangoon": "Asia/Yangon", + "Asia/Saigon": "Asia/Ho_Chi_Minh", + "Asia/Thimbu": "Asia/Thimphu", + "Asia/Ujung_Pandang": "Asia/Makassar", + "Asia/Ulan_Bator": "Asia/Ulaanbaatar", + + // Atlantic + "Atlantic/Faeroe": "Atlantic/Faroe", + + // Australia + "Australia/ACT": "Australia/Sydney", + "Australia/LHI": "Australia/Lord_Howe", + "Australia/North": "Australia/Darwin", + "Australia/NSW": "Australia/Sydney", + "Australia/Queensland": "Australia/Brisbane", + "Australia/South": "Australia/Adelaide", + "Australia/Tasmania": "Australia/Hobart", + "Australia/Victoria": "Australia/Melbourne", + "Australia/West": "Australia/Perth", + + // Brazil + "Brazil/Acre": "America/Rio_Branco", + "Brazil/DeNoronha": "America/Noronha", + "Brazil/East": "America/Sao_Paulo", + "Brazil/West": "America/Manaus", + + // Canada + "Canada/Atlantic": "America/Halifax", + "Canada/Central": "America/Winnipeg", + "Canada/Eastern": "America/Toronto", + "Canada/Mountain": "America/Edmonton", + "Canada/Newfoundland": "America/St_Johns", + "Canada/Pacific": "America/Vancouver", + "Canada/Saskatchewan": "America/Regina", + "Canada/Yukon": "America/Whitehorse", + + // Europe + "CET": "Europe/Paris", + "EET": "Europe/Sofia", + "Europe/Kiev": "Europe/Kyiv", + "Europe/Uzhgorod": "Europe/Kyiv", + "Europe/Zaporozhye": "Europe/Kyiv", + "MET": "Europe/Paris", + "WET": "Europe/Lisbon", + + // Chile + "Chile/Continental": "America/Santiago", + "Chile/EasterIsland": "Pacific/Easter", + + // Fixed offset and generic zones + "CST6CDT": "America/Chicago", + "EST": "America/New_York", + "EST5EDT": "America/New_York", + "HST": "Pacific/Honolulu", + "MST": "America/Denver", + "MST7MDT": "America/Denver", + "PST8PDT": "America/Los_Angeles", + + // Countries/Regions + "Cuba": "America/Havana", + "Egypt": "Africa/Cairo", + "Eire": "Europe/Dublin", + "GB": "Europe/London", + "GB-Eire": "Europe/London", + "Hongkong": "Asia/Hong_Kong", + "Iceland": "Atlantic/Reykjavik", + "Iran": "Asia/Tehran", + "Israel": "Asia/Jerusalem", + "Jamaica": "America/Jamaica", + "Japan": "Asia/Tokyo", + "Libya": "Africa/Tripoli", + "Poland": "Europe/Warsaw", + "Portugal": "Europe/Lisbon", + "PRC": "Asia/Shanghai", + "ROC": "Asia/Taipei", + "ROK": "Asia/Seoul", + "Singapore": "Asia/Singapore", + "Turkey": "Europe/Istanbul", + + // GMT variations + "GMT+0": "GMT", + "GMT-0": "GMT", + "GMT0": "GMT", + "Greenwich": "GMT", + "UCT": "UTC", + "Universal": "UTC", + "Zulu": "UTC", + + // Mexico + "Mexico/BajaNorte": "America/Tijuana", + "Mexico/BajaSur": "America/Mazatlan", + "Mexico/General": "America/Mexico_City", + + // US zones + "Navajo": "America/Denver", + "US/Alaska": "America/Anchorage", + "US/Aleutian": "America/Adak", + "US/Arizona": "America/Phoenix", + "US/Central": "America/Chicago", + "US/Eastern": "America/New_York", + "US/East-Indiana": "America/Indiana/Indianapolis", + "US/Hawaii": "Pacific/Honolulu", + "US/Indiana-Starke": "America/Indiana/Knox", + "US/Michigan": "America/Detroit", + "US/Mountain": "America/Denver", + "US/Pacific": "America/Los_Angeles", + "US/Samoa": "Pacific/Pago_Pago", + + // Pacific + "Kwajalein": "Pacific/Kwajalein", + "NZ": "Pacific/Auckland", + "NZ-CHAT": "Pacific/Chatham", + "Pacific/Enderbury": "Pacific/Kanton", + "Pacific/Ponape": "Pacific/Pohnpei", + "Pacific/Truk": "Pacific/Chuuk", + + // Special cases + "Factory": "UTC", // Factory is used for unconfigured systems + "W-SU": "Europe/Moscow", + } + + // Loop through each user and correct the timezone + rows, err := tx.Query(`SELECT id, timezone FROM users`) + if err != nil { + return err + } + + userTimezoneMap := make(map[int64]string) + for rows.Next() { + var userID int64 + var userTimezone string + if err := rows.Scan(&userID, &userTimezone); err != nil { + return err + } + userTimezoneMap[userID] = userTimezone + } + rows.Close() + + for userID, userTimezone := range userTimezoneMap { + if newTimezone, found := deprecatedTimeZoneMap[userTimezone]; found { + if _, err := tx.Exec(`UPDATE users SET timezone = $1 WHERE id = $2`, newTimezone, userID); err != nil { + return err + } + } + } + + return nil + }, } diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go new file mode 100644 index 00000000..326adcf3 --- /dev/null +++ b/internal/database/sqlite.go @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package database // import "miniflux.app/v2/internal/database" + +import ( + "database/sql" + + _ "modernc.org/sqlite" +) + +var sqliteSchemaVersion = len(sqliteMigrations) + +// Order is important. Add new migrations at the end of the list. +var sqliteMigrations = []Migration{ + func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TABLE schema_version ( + version INTEGER NOT NULL + ); + `) + return err + }, + func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL UNIQUE, + password TEXT, + is_admin INTEGER NOT NULL DEFAULT 0, + language TEXT NOT NULL DEFAULT 'en_US', + timezone TEXT NOT NULL DEFAULT 'UTC', + theme TEXT NOT NULL DEFAULT 'light_serif', + last_login_at DATETIME, + entry_direction TEXT NOT NULL DEFAULT 'asc' CHECK (entry_direction IN ('asc','desc')), + keyboard_shortcuts INTEGER NOT NULL DEFAULT 1, + entries_per_page INTEGER NOT NULL DEFAULT 100, + show_reading_time INTEGER NOT NULL DEFAULT 1, + entry_swipe INTEGER NOT NULL DEFAULT 1, + stylesheet TEXT NOT NULL DEFAULT '', + google_id TEXT NOT NULL DEFAULT '', + openid_connect_id TEXT NOT NULL DEFAULT '', + display_mode TEXT NOT NULL DEFAULT 'standalone' CHECK (display_mode IN ('fullscreen','standalone','minimal-ui','browser')), + entry_order TEXT NOT NULL DEFAULT 'published_at' CHECK (entry_order IN ('published_at','created_at')), + default_reading_speed INTEGER NOT NULL DEFAULT 265, + cjk_reading_speed INTEGER NOT NULL DEFAULT 500, + default_home_page TEXT NOT NULL DEFAULT 'unread', + categories_sorting_order TEXT NOT NULL DEFAULT 'unread_count', + gesture_nav TEXT NOT NULL DEFAULT 'tap', + mark_read_on_view INTEGER NOT NULL DEFAULT 1, + media_playback_rate REAL NOT NULL DEFAULT 1.0, + block_filter_entry_rules TEXT NOT NULL DEFAULT '', + keep_filter_entry_rules TEXT NOT NULL DEFAULT '', + mark_read_on_media_player_completion INTEGER NOT NULL DEFAULT 0, + custom_js TEXT NOT NULL DEFAULT '', + external_font_hosts TEXT NOT NULL DEFAULT '', + always_open_external_links INTEGER NOT NULL DEFAULT 0, + open_external_links_in_new_tab INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE user_sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + token TEXT NOT NULL UNIQUE, + created_at DATETIME NOT NULL DEFAULT (DATETIME('now')), + user_agent TEXT, + ip TEXT, + UNIQUE(user_id, token), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE categories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + title TEXT NOT NULL, + hide_globally INTEGER NOT NULL DEFAULT 0, + UNIQUE(user_id, title), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE feeds ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + category_id INTEGER NOT NULL, + title TEXT NOT NULL, + feed_url TEXT NOT NULL, + site_url TEXT NOT NULL, + checked_at DATETIME NOT NULL DEFAULT (DATETIME('now')), + etag_header TEXT NOT NULL DEFAULT '', + last_modified_header TEXT NOT NULL DEFAULT '', + parsing_error_msg TEXT NOT NULL DEFAULT '', + parsing_error_count INTEGER NOT NULL DEFAULT 0, + scraper_rules TEXT NOT NULL DEFAULT '', + rewrite_rules TEXT NOT NULL DEFAULT '', + crawler INTEGER NOT NULL DEFAULT 0, + username TEXT NOT NULL DEFAULT '', + password TEXT NOT NULL DEFAULT '', + user_agent TEXT NOT NULL DEFAULT '', + disabled INTEGER NOT NULL DEFAULT 0, + next_check_at DATETIME NOT NULL DEFAULT (DATETIME('now')), + ignore_http_cache INTEGER NOT NULL DEFAULT 0, + fetch_via_proxy INTEGER NOT NULL DEFAULT 0, + blocklist_rules TEXT NOT NULL DEFAULT '', + keeplist_rules TEXT NOT NULL DEFAULT '', + allow_self_signed_certificates INTEGER NOT NULL DEFAULT 0, + cookie TEXT NOT NULL DEFAULT '', + hide_globally INTEGER NOT NULL DEFAULT 0, + url_rewrite_rules TEXT NOT NULL DEFAULT '', + no_media_player INTEGER NOT NULL DEFAULT 0, + apprise_service_urls TEXT NOT NULL DEFAULT '', + disable_http2 INTEGER NOT NULL DEFAULT 0, + description TEXT NOT NULL DEFAULT '', + ntfy_enabled INTEGER NOT NULL DEFAULT 0, + ntfy_priority INTEGER NOT NULL DEFAULT 3, + webhook_url TEXT NOT NULL DEFAULT '', + pushover_enabled INTEGER NOT NULL DEFAULT 0, + pushover_priority INTEGER NOT NULL DEFAULT 0, + ntfy_topic TEXT NOT NULL DEFAULT '', + proxy_url TEXT NOT NULL DEFAULT '', + block_filter_entry_rules TEXT NOT NULL DEFAULT '', + keep_filter_entry_rules TEXT NOT NULL DEFAULT '', + UNIQUE(user_id, feed_url), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE + ); + CREATE TABLE entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + feed_id INTEGER NOT NULL, + hash TEXT NOT NULL, + published_at DATETIME NOT NULL, + title TEXT NOT NULL, + url TEXT NOT NULL, + author TEXT, + content TEXT, + status TEXT NOT NULL DEFAULT 'unread' CHECK (status IN ('unread','read','removed')), + starred INTEGER NOT NULL DEFAULT 0, + comments_url TEXT NOT NULL DEFAULT '', + changed_at DATETIME NOT NULL, + share_code TEXT NOT NULL DEFAULT '', + reading_time INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT (DATETIME('now')), + tags TEXT NOT NULL DEFAULT '', + UNIQUE(feed_id, hash), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (feed_id) REFERENCES feeds(id) ON DELETE CASCADE + ); + CREATE TABLE enclosures ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + url TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + mime_type TEXT NOT NULL DEFAULT '', + media_progression INTEGER NOT NULL DEFAULT 0, + UNIQUE(user_id, entry_id, url), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE + ); + CREATE TABLE icons ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT NOT NULL, + mime_type TEXT NOT NULL, + content BLOB NOT NULL, + external_id TEXT NOT NULL DEFAULT '', + UNIQUE(hash) + ); + CREATE TABLE feed_icons ( + feed_id INTEGER NOT NULL, + icon_id INTEGER NOT NULL, + PRIMARY KEY (feed_id, icon_id), + FOREIGN KEY (feed_id) REFERENCES feeds(id) ON DELETE CASCADE, + FOREIGN KEY (icon_id) REFERENCES icons(id) ON DELETE CASCADE + ); + CREATE TABLE integrations ( + user_id INTEGER PRIMARY KEY, + pinboard_enabled INTEGER NOT NULL DEFAULT 0, + pinboard_token TEXT NOT NULL DEFAULT '', + pinboard_tags TEXT NOT NULL DEFAULT 'miniflux', + pinboard_mark_as_unread INTEGER NOT NULL DEFAULT 0, + instapaper_enabled INTEGER NOT NULL DEFAULT 0, + instapaper_username TEXT NOT NULL DEFAULT '', + instapaper_password TEXT NOT NULL DEFAULT '', + fever_enabled INTEGER NOT NULL DEFAULT 0, + fever_username TEXT NOT NULL DEFAULT '', + fever_token TEXT NOT NULL DEFAULT '', + wallabag_enabled INTEGER NOT NULL DEFAULT 0, + wallabag_url TEXT NOT NULL DEFAULT '', + wallabag_client_id TEXT NOT NULL DEFAULT '', + wallabag_client_secret TEXT NOT NULL DEFAULT '', + wallabag_username TEXT NOT NULL DEFAULT '', + wallabag_password TEXT NOT NULL DEFAULT '', + nunux_keeper_enabled INTEGER NOT NULL DEFAULT 0, + nunux_keeper_url TEXT NOT NULL DEFAULT '', + nunux_keeper_api_key TEXT NOT NULL DEFAULT '', + telegram_bot_enabled INTEGER NOT NULL DEFAULT 0, + telegram_bot_token TEXT NOT NULL DEFAULT '', + telegram_bot_chat_id TEXT NOT NULL DEFAULT '', + googlereader_enabled INTEGER NOT NULL DEFAULT 0, + googlereader_username TEXT NOT NULL DEFAULT '', + googlereader_password TEXT NOT NULL DEFAULT '', + espial_enabled INTEGER NOT NULL DEFAULT 0, + espial_url TEXT NOT NULL DEFAULT '', + espial_api_key TEXT NOT NULL DEFAULT '', + espial_tags TEXT NOT NULL DEFAULT 'miniflux', + linkding_enabled INTEGER NOT NULL DEFAULT 0, + linkding_url TEXT NOT NULL DEFAULT '', + linkding_api_key TEXT NOT NULL DEFAULT '', + wallabag_only_url INTEGER NOT NULL DEFAULT 0, + matrix_bot_enabled INTEGER NOT NULL DEFAULT 0, + matrix_bot_user TEXT NOT NULL DEFAULT '', + matrix_bot_password TEXT NOT NULL DEFAULT '', + matrix_bot_url TEXT NOT NULL DEFAULT '', + matrix_bot_chat_id TEXT NOT NULL DEFAULT '', + linkding_tags TEXT NOT NULL DEFAULT '', + linkding_mark_as_unread INTEGER NOT NULL DEFAULT 0, + notion_enabled INTEGER NOT NULL DEFAULT 0, + notion_token TEXT NOT NULL DEFAULT '', + notion_page_id TEXT NOT NULL DEFAULT '', + readwise_enabled INTEGER NOT NULL DEFAULT 0, + readwise_api_key TEXT NOT NULL DEFAULT '', + apprise_enabled INTEGER NOT NULL DEFAULT 0, + apprise_url TEXT NOT NULL DEFAULT '', + apprise_services_url TEXT NOT NULL DEFAULT '', + shiori_enabled INTEGER NOT NULL DEFAULT 0, + shiori_url TEXT NOT NULL DEFAULT '', + shiori_username TEXT NOT NULL DEFAULT '', + shiori_password TEXT NOT NULL DEFAULT '', + shaarli_enabled INTEGER NOT NULL DEFAULT 0, + shaarli_url TEXT NOT NULL DEFAULT '', + shaarli_api_secret TEXT NOT NULL DEFAULT '', + webhook_enabled INTEGER NOT NULL DEFAULT 0, + webhook_url TEXT NOT NULL DEFAULT '', + webhook_secret TEXT NOT NULL DEFAULT '', + telegram_bot_topic_id INTEGER, + telegram_bot_disable_web_page_preview INTEGER NOT NULL DEFAULT 0, + telegram_bot_disable_notification INTEGER NOT NULL DEFAULT 0, + telegram_bot_disable_buttons INTEGER NOT NULL DEFAULT 0, + rssbridge_enabled INTEGER NOT NULL DEFAULT 0, + rssbridge_url TEXT NOT NULL DEFAULT '', + omnivore_enabled INTEGER NOT NULL DEFAULT 0, + omnivore_api_key TEXT NOT NULL DEFAULT '', + omnivore_url TEXT NOT NULL DEFAULT '', + linkace_enabled INTEGER NOT NULL DEFAULT 0, + linkace_url TEXT NOT NULL DEFAULT '', + linkace_api_key TEXT NOT NULL DEFAULT '', + linkace_tags TEXT NOT NULL DEFAULT '', + linkace_is_private INTEGER NOT NULL DEFAULT 1, + linkace_check_disabled INTEGER NOT NULL DEFAULT 1, + linkwarden_enabled INTEGER NOT NULL DEFAULT 0, + linkwarden_url TEXT NOT NULL DEFAULT '', + linkwarden_api_key TEXT NOT NULL DEFAULT '', + readeck_enabled INTEGER NOT NULL DEFAULT 0, + readeck_only_url INTEGER NOT NULL DEFAULT 0, + readeck_url TEXT NOT NULL DEFAULT '', + readeck_api_key TEXT NOT NULL DEFAULT '', + readeck_labels TEXT NOT NULL DEFAULT '', + raindrop_enabled INTEGER NOT NULL DEFAULT 0, + raindrop_token TEXT NOT NULL DEFAULT '', + raindrop_collection_id TEXT NOT NULL DEFAULT '', + raindrop_tags TEXT NOT NULL DEFAULT '', + betula_url TEXT NOT NULL DEFAULT '', + betula_token TEXT NOT NULL DEFAULT '', + betula_enabled INTEGER NOT NULL DEFAULT 0, + ntfy_enabled INTEGER NOT NULL DEFAULT 0, + ntfy_url TEXT NOT NULL DEFAULT '', + ntfy_topic TEXT NOT NULL DEFAULT '', + ntfy_api_token TEXT NOT NULL DEFAULT '', + ntfy_username TEXT NOT NULL DEFAULT '', + ntfy_password TEXT NOT NULL DEFAULT '', + ntfy_icon_url TEXT NOT NULL DEFAULT '', + cubox_enabled INTEGER NOT NULL DEFAULT 0, + cubox_api_link TEXT NOT NULL DEFAULT '', + discord_enabled INTEGER NOT NULL DEFAULT 0, + discord_webhook_link TEXT NOT NULL DEFAULT '', + ntfy_internal_links INTEGER NOT NULL DEFAULT 0, + slack_enabled INTEGER NOT NULL DEFAULT 0, + slack_webhook_link TEXT NOT NULL DEFAULT '', + pushover_enabled INTEGER NOT NULL DEFAULT 0, + pushover_user TEXT NOT NULL DEFAULT '', + pushover_token TEXT NOT NULL DEFAULT '', + pushover_device TEXT NOT NULL DEFAULT '', + pushover_prefix TEXT NOT NULL DEFAULT '', + rssbridge_token TEXT NOT NULL DEFAULT '', + karakeep_enabled INTEGER NOT NULL DEFAULT 0, + karakeep_api_key TEXT NOT NULL DEFAULT '', + karakeep_url TEXT NOT NULL DEFAULT '', + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + data TEXT NOT NULL, + created_at DATETIME NOT NULL DEFAULT (DATETIME('now')) + ); + CREATE TABLE api_keys ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + token TEXT NOT NULL UNIQUE, + description TEXT NOT NULL, + last_used_at DATETIME, + created_at DATETIME NOT NULL DEFAULT (DATETIME('now')), + UNIQUE(user_id, description), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE acme_cache ( + key TEXT PRIMARY KEY, + data BLOB NOT NULL, + updated_at DATETIME NOT NULL + ); + CREATE TABLE webauthn_credentials ( + handle BLOB PRIMARY KEY, + cred_id BLOB NOT NULL UNIQUE, + user_id INTEGER NOT NULL, + key BLOB NOT NULL, + attestation_type TEXT NOT NULL, + aaguid BLOB, + sign_count INTEGER, + clone_warning INTEGER, + name TEXT, + added_on DATETIME NOT NULL DEFAULT (DATETIME('now')), + last_seen_on DATETIME NOT NULL DEFAULT (DATETIME('now')), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + `) + return err + }, + func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE UNIQUE INDEX icons_external_id_uq ON icons(external_id) WHERE external_id != ''; + CREATE UNIQUE INDEX users_google_id_uq ON users(google_id) WHERE google_id != ''; + CREATE UNIQUE INDEX users_openid_connect_id_uq ON users(openid_connect_id) WHERE openid_connect_id != ''; + CREATE UNIQUE INDEX entries_share_code_uq ON entries(share_code) WHERE share_code != ''; + + CREATE INDEX feeds_user_category_idx ON feeds(user_id, category_id); + CREATE INDEX feeds_id_hide_globally_idx ON feeds(id, hide_globally); + CREATE INDEX entries_feed_idx ON entries(feed_id); + CREATE INDEX entries_user_status_idx ON entries(user_id, status); + CREATE INDEX entries_user_feed_idx ON entries(user_id, feed_id); + CREATE INDEX entries_user_status_changed_idx ON entries(user_id, status, changed_at); + CREATE INDEX entries_user_status_published_idx ON entries(user_id, status, published_at); + CREATE INDEX entries_user_status_created_idx ON entries(user_id, status, created_at); + CREATE INDEX entries_id_user_status_idx ON entries(id, user_id, status); + CREATE INDEX entries_feed_id_status_hash_idx ON entries(feed_id, status, hash); + CREATE INDEX entries_user_id_status_starred_idx ON entries(user_id, status, starred); + CREATE INDEX entries_user_status_feed_idx ON entries(user_id, status, feed_id); + CREATE INDEX entries_user_status_changed_published_idx ON entries(user_id, status, changed_at, published_at); + CREATE INDEX enclosures_entry_id_idx ON enclosures(entry_id); + CREATE INDEX feed_icons_icon_id_idx ON feed_icons(icon_id); + `) + return err + }, + func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE VIRTUAL TABLE entries_fts USING fts5( + title, + content, + tags, + entry_id UNINDEXED, + content='', + -- as close to PostgreSQL as possible while being multilingual + tokenize = "unicode61 remove_diacritics 2 tokenchars '-_'" + ); + + CREATE TRIGGER entries_ai AFTER INSERT ON entries BEGIN + INSERT INTO entries_fts(rowid,title,content,tags,entry_id) + VALUES (new.id,new.title,COALESCE(new.content,''),COALESCE(new.tags,''),new.id); + END; + + CREATE TRIGGER entries_au AFTER UPDATE ON entries BEGIN + INSERT INTO entries_fts(entries_fts,rowid) VALUES('delete', old.id); + INSERT INTO entries_fts(rowid,title,content,tags,entry_id) + VALUES (new.id,new.title,COALESCE(new.content,''),COALESCE(new.tags,''),new.id); + END; + + CREATE TRIGGER entries_ad AFTER DELETE ON entries BEGIN + INSERT INTO entries_fts(entries_fts,rowid) VALUES('delete', old.id); + END; + `) + return err + }, +} diff --git a/internal/storage/enclosure.go b/internal/storage/enclosure.go index ba0a69de..3c3e864f 100644 --- a/internal/storage/enclosure.go +++ b/internal/storage/enclosure.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" + "miniflux.app/v2/internal/database" "miniflux.app/v2/internal/model" "github.com/lib/pq" @@ -49,7 +50,6 @@ func (s *Storage) GetEnclosures(entryID int64) (model.EnclosureList, error) { &enclosure.MimeType, &enclosure.MediaProgression, ) - if err != nil { return nil, fmt.Errorf(`store: unable to fetch enclosure row: %v`, err) } @@ -150,15 +150,20 @@ func (s *Storage) createEnclosure(tx *sql.Tx, enclosure *model.Enclosure) error return nil } - query := ` + urlPart := "md5(url)" + if s.kind == database.DBKindCockroach { + urlPart = "url" + } + + query := fmt.Sprintf(` INSERT INTO enclosures (url, size, mime_type, entry_id, user_id, media_progression) VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (user_id, entry_id, md5(url)) DO NOTHING + ON CONFLICT (user_id, entry_id, %s) DO NOTHING RETURNING id - ` + `, urlPart) if err := tx.QueryRow( query, enclosureURL, @@ -226,7 +231,6 @@ func (s *Storage) UpdateEnclosure(enclosure *model.Enclosure) error { enclosure.MediaProgression, enclosure.ID, ) - if err != nil { return fmt.Errorf(`store: unable to update enclosure #%d : %v`, enclosure.ID, err) } diff --git a/internal/storage/entry.go b/internal/storage/entry.go index 272e11d2..c539763d 100644 --- a/internal/storage/entry.go +++ b/internal/storage/entry.go @@ -5,12 +5,14 @@ package storage // import "miniflux.app/v2/internal/storage" import ( "database/sql" + "encoding/json" "errors" "fmt" "log/slog" "time" "miniflux.app/v2/internal/crypto" + "miniflux.app/v2/internal/database" "miniflux.app/v2/internal/model" "github.com/lib/pq" @@ -70,17 +72,24 @@ func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder { // UpdateEntryTitleAndContent updates entry title and content. func (s *Storage) UpdateEntryTitleAndContent(entry *model.Entry) error { truncatedTitle, truncatedContent := truncateTitleAndContentForTSVectorField(entry.Title, entry.Content) - query := ` + vectorPart := "" + if s.kind != database.DBKindSqlite { + vectorPart = ", document_vectors = setweight(to_tsvector($4), 'A') || setweight(to_tsvector($5), 'B')" + if s.kind == database.DBKindCockroach { + vectorPart = ", document_vectors = to_tsvector(substring($4 || ' ' || $4 || ' ' || coalesce($5, '') for 1000000))" + } + } + query := fmt.Sprintf(` UPDATE entries SET title=$1, content=$2, - reading_time=$3, - document_vectors = setweight(to_tsvector($4), 'A') || setweight(to_tsvector($5), 'B') + reading_time=$3 + %s WHERE id=$6 AND user_id=$7 - ` + `, vectorPart) if _, err := s.db.Exec( query, @@ -100,7 +109,14 @@ func (s *Storage) UpdateEntryTitleAndContent(entry *model.Entry) error { // createEntry add a new entry. func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { truncatedTitle, truncatedContent := truncateTitleAndContentForTSVectorField(entry.Title, entry.Content) - query := ` + vectorParts := [2]string{"", ""} + if s.kind != database.DBKindSqlite { + vectorParts = [2]string{",document_vectors", ",setweight(to_tsvector($11), 'A') || setweight(to_tsvector($12), 'B')"} + if s.kind == database.DBKindCockroach { + vectorParts[1] = ",to_tsvector(substring($11 || ' ' || $11 || ' ' || coalesce($12, '') for 1000000))" + } + } + query := fmt.Sprintf(` INSERT INTO entries ( title, @@ -114,8 +130,8 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { feed_id, reading_time, changed_at, - document_vectors, tags + %s ) VALUES ( @@ -130,12 +146,24 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { $9, $10, now(), - setweight(to_tsvector($11), 'A') || setweight(to_tsvector($12), 'B'), $13 + %s ) RETURNING id, status, created_at, changed_at - ` + `, vectorParts[0], vectorParts[1]) + + var tagsParam any + if s.kind == database.DBKindSqlite { + sqliteTagsParam, err := encodeTagsJSON(entry.Tags) + if err != nil { + return fmt.Errorf("encode tags: %w", err) + } + tagsParam = sqliteTagsParam + } else { + tagsParam = pq.Array(entry.Tags) + } + err := tx.QueryRow( query, entry.Title, @@ -150,7 +178,7 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { entry.ReadingTime, truncatedTitle, truncatedContent, - pq.Array(entry.Tags), + tagsParam, ).Scan( &entry.ID, &entry.Status, @@ -178,7 +206,14 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { // it default to time.Now() which could change the order of items on the history page. func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error { truncatedTitle, truncatedContent := truncateTitleAndContentForTSVectorField(entry.Title, entry.Content) - query := ` + vectorPart := "" + if s.kind != database.DBKindSqlite { + vectorPart = ",document_vectors = setweight(to_tsvector($7), 'A') || setweight(to_tsvector($8), 'B')" + if s.kind == database.DBKindCockroach { + vectorPart = ",document_vectors = to_tsvector(substring($7 || ' ' || $7 || ' ' || coalesce($8, '') for 1000000))" + } + } + query := fmt.Sprintf(` UPDATE entries SET @@ -188,13 +223,25 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error { content=$4, author=$5, reading_time=$6, - document_vectors = setweight(to_tsvector($7), 'A') || setweight(to_tsvector($8), 'B'), tags=$12 + %s WHERE user_id=$9 AND feed_id=$10 AND hash=$11 RETURNING id - ` + `, vectorPart) + + var tagsParam any + if s.kind == database.DBKindSqlite { + sqliteTagsParam, err := encodeTagsJSON(entry.Tags) + if err != nil { + return fmt.Errorf("encode tags: %w", err) + } + tagsParam = sqliteTagsParam + } else { + tagsParam = pq.Array(entry.Tags) + } + err := tx.QueryRow( query, entry.Title, @@ -208,7 +255,7 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error { entry.UserID, entry.FeedID, entry.Hash, - pq.Array(entry.Tags), + tagsParam, ).Scan(&entry.ID) if err != nil { return fmt.Errorf(`store: unable to update entry %q: %v`, entry.URL, err) @@ -301,7 +348,12 @@ func (s *Storage) DeleteRemovedEntriesEnclosures() (int64, error) { // ClearRemovedEntriesContent clears the content fields of entries marked as "removed", keeping only their metadata. func (s *Storage) ClearRemovedEntriesContent(limit int) (int64, error) { - query := ` + vectorPart := "" + if s.kind != database.DBKindSqlite { + vectorPart = ",document_vectors=NULL" + } + + query := fmt.Sprintf(` UPDATE entries SET @@ -309,8 +361,8 @@ func (s *Storage) ClearRemovedEntriesContent(limit int) (int64, error) { content=NULL, url='', author=NULL, - comments_url=NULL, - document_vectors=NULL + comments_url=NULL + %s WHERE id IN ( SELECT id FROM entries @@ -318,7 +370,7 @@ func (s *Storage) ClearRemovedEntriesContent(limit int) (int64, error) { ORDER BY id ASC LIMIT $2 ) - ` + `, vectorPart) result, err := s.db.Exec(query, model.EntryStatusRemoved, limit) if err != nil { @@ -737,3 +789,19 @@ func truncateStringForTSVectorField(s string, maxSize int) string { // Fallback: return empty string if we can't find a valid UTF-8 boundary return "" } + +func encodeTagsJSON(tags []string) (string, error) { + b, err := json.Marshal(tags) + return string(b), err +} + +func decodeTagsJSON(s string) ([]string, error) { + if s == "" { + return nil, nil + } + var tags []string + if err := json.Unmarshal([]byte(s), &tags); err != nil { + return nil, err + } + return tags, nil +} diff --git a/internal/storage/entry_pagination_builder.go b/internal/storage/entry_pagination_builder.go index 0e1251ce..73afa7b9 100644 --- a/internal/storage/entry_pagination_builder.go +++ b/internal/storage/entry_pagination_builder.go @@ -9,24 +9,34 @@ import ( "strconv" "strings" + "miniflux.app/v2/internal/database" "miniflux.app/v2/internal/model" ) // EntryPaginationBuilder is a builder for entry prev/next queries. type EntryPaginationBuilder struct { - store *Storage - conditions []string - args []any - entryID int64 - order string - direction string + store *Storage + conditions []string + args []any + entryID int64 + order string + direction string + useSqliteFts bool } // WithSearchQuery adds full-text search query to the condition. func (e *EntryPaginationBuilder) WithSearchQuery(query string) { if query != "" { - e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", len(e.args)+1)) - e.args = append(e.args, query) + nArgs := len(e.args) + 1 + if e.store.kind == database.DBKindSqlite { + query = toSqliteFtsQuery(query) + e.useSqliteFts = true + e.conditions = append(e.conditions, fmt.Sprintf("fts MATCH $%d", nArgs)) + e.args = append(e.args, query) + } else { + e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", nArgs)) + e.args = append(e.args, query) + } } } @@ -61,9 +71,24 @@ func (e *EntryPaginationBuilder) WithStatus(status string) { func (e *EntryPaginationBuilder) WithTags(tags []string) { if len(tags) > 0 { - for _, tag := range tags { - e.conditions = append(e.conditions, fmt.Sprintf("LOWER($%d) = ANY(LOWER(e.tags::text)::text[])", len(e.args)+1)) - e.args = append(e.args, tag) + if e.store.kind == database.DBKindSqlite { + for _, tag := range tags { + n := len(e.args) + 1 + cond := fmt.Sprintf(` + EXISTS ( + SELECT 1 + FROM json_each(COALESCE(e.tags, '[]')) AS jt + WHERE lower(jt.value) = lower($%d) + ) + `, n) + e.conditions = append(e.conditions, cond) + e.args = append(e.args, tag) + } + } else { + for _, tag := range tags { + e.conditions = append(e.conditions, fmt.Sprintf("LOWER($%d) = ANY(LOWER(e.tags::text)::text[])", len(e.args)+1)) + e.args = append(e.args, tag) + } } } } diff --git a/internal/storage/entry_query_builder.go b/internal/storage/entry_query_builder.go index a3671d03..410eab6b 100644 --- a/internal/storage/entry_query_builder.go +++ b/internal/storage/entry_query_builder.go @@ -12,6 +12,7 @@ import ( "github.com/lib/pq" + "miniflux.app/v2/internal/database" "miniflux.app/v2/internal/model" "miniflux.app/v2/internal/timezone" ) @@ -25,6 +26,7 @@ type EntryQueryBuilder struct { limit int offset int fetchEnclosures bool + useSqliteFts bool } // WithEnclosures fetches enclosures for each entry. @@ -37,14 +39,23 @@ func (e *EntryQueryBuilder) WithEnclosures() *EntryQueryBuilder { func (e *EntryQueryBuilder) WithSearchQuery(query string) *EntryQueryBuilder { if query != "" { nArgs := len(e.args) + 1 - e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", nArgs)) - e.args = append(e.args, query) + if e.store.kind == database.DBKindSqlite { + query = toSqliteFtsQuery(query) + e.useSqliteFts = true + e.conditions = append(e.conditions, fmt.Sprintf("fts MATCH $%d", nArgs)) + e.args = append(e.args, query) - // 0.0000001 = 0.1 / (seconds_in_a_day) - e.WithSorting( - fmt.Sprintf("ts_rank(document_vectors, plainto_tsquery($%d)) - extract (epoch from now() - published_at)::float * 0.0000001", nArgs), - "DESC", - ) + e.WithSorting("bm25(fts)", "DESC") + } else { + e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", nArgs)) + e.args = append(e.args, query) + + // 0.0000001 = 0.1 / (seconds_in_a_day) + e.WithSorting( + fmt.Sprintf("ts_rank(document_vectors, plainto_tsquery($%d)) - extract (epoch from now() - published_at)::float * 0.0000001", nArgs), + "DESC", + ) + } } return e } @@ -160,9 +171,24 @@ func (e *EntryQueryBuilder) WithStatuses(statuses []string) *EntryQueryBuilder { // WithTags filter by a list of entry tags. func (e *EntryQueryBuilder) WithTags(tags []string) *EntryQueryBuilder { if len(tags) > 0 { - for _, cat := range tags { - e.conditions = append(e.conditions, fmt.Sprintf("LOWER($%d) = ANY(LOWER(e.tags::text)::text[])", len(e.args)+1)) - e.args = append(e.args, cat) + if e.store.kind == database.DBKindSqlite { + for _, tag := range tags { + n := len(e.args) + 1 + cond := fmt.Sprintf(` + EXISTS ( + SELECT 1 + FROM json_each(COALESCE(e.tags, '[]')) AS jt + WHERE lower(jt.value) = lower($%d) + ) + `, n) + e.conditions = append(e.conditions, cond) + e.args = append(e.args, tag) + } + } else { + for _, cat := range tags { + e.conditions = append(e.conditions, fmt.Sprintf("LOWER($%d) = ANY(LOWER(e.tags::text)::text[])", len(e.args)+1)) + e.args = append(e.args, cat) + } } } return e @@ -307,7 +333,16 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { icons i ON i.id=fi.icon_id LEFT JOIN users u ON u.id=e.user_id - WHERE ` + e.buildCondition() + " " + e.buildSorting() + ` + + if e.useSqliteFts { + query += ` + LEFT JOIN + entries_fts fts ON fts.rowid = e.id + ` + } + + query += " WHERE " + e.buildCondition() + " " + e.buildSorting() rows, err := e.store.db.Query(query, e.args...) if err != nil { @@ -364,7 +399,6 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { &externalIconID, &tz, ) - if err != nil { return nil, fmt.Errorf("store: unable to fetch entry row: %v", err) } @@ -480,3 +514,16 @@ func NewAnonymousQueryBuilder(store *Storage) *EntryQueryBuilder { store: store, } } + +func toSqliteFtsQuery(q string) string { + q = strings.TrimSpace(q) + if q == "" { + return "" + } + terms := strings.Fields(q) + for i, t := range terms { + t = strings.ReplaceAll(t, `"`, `""`) + terms[i] = `"` + t + `"` + } + return strings.Join(terms, " AND ") +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index ffae9692..aa218a6f 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -7,16 +7,19 @@ import ( "context" "database/sql" "time" + + "miniflux.app/v2/internal/database" ) // Storage handles all operations related to the database. type Storage struct { - db *sql.DB + kind database.DBKind + db *sql.DB } // NewStorage returns a new Storage. -func NewStorage(db *sql.DB) *Storage { - return &Storage{db} +func NewStorage(kind database.DBKind, db *sql.DB) *Storage { + return &Storage{kind, db} } // DatabaseVersion returns the version of the database which is in use. diff --git a/miniflux.1 b/miniflux.1 index fe059cf7..c5cfee81 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -226,9 +226,9 @@ Minimum number of database connections\&. Default is 1\&. .TP .B DATABASE_URL -Postgresql connection parameters\&. +Database connection parameters\&. .br -Default is "user=postgres password=postgres dbname=miniflux2 sslmode=disable"\&. +Default is "postgres://postgres:postgres/postgres?sslmode=disable"\&. .TP .B DATABASE_URL_FILE Path to a secret key exposed as a file, it should contain $DATABASE_URL value\&. diff --git a/packaging/systemd/miniflux.service b/packaging/systemd/miniflux.service index 6132ee53..62bcd11b 100644 --- a/packaging/systemd/miniflux.service +++ b/packaging/systemd/miniflux.service @@ -9,7 +9,7 @@ [Unit] Description=Miniflux Documentation=man:miniflux(1) https://miniflux.app/docs/index.html -After=network.target postgresql.service +After=network.target postgresql.service cockroachdb.service [Service] ExecStart=/usr/bin/miniflux