mirror of
https://github.com/miniflux/v2.git
synced 2025-07-27 17:28:38 +00:00
Merge branch 'main' into sitemap
This commit is contained in:
commit
893ae2f822
45 changed files with 676 additions and 258 deletions
4
.github/workflows/build_binaries.yml
vendored
4
.github/workflows/build_binaries.yml
vendored
|
@ -9,13 +9,13 @@ jobs:
|
||||||
name: Build
|
name: Build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
- name: Set up Golang
|
- name: Set up Golang
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: "1.23.x"
|
go-version: "1.23.x"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Compile binaries
|
- name: Compile binaries
|
||||||
env:
|
env:
|
||||||
CGO_ENABLED: 0
|
CGO_ENABLED: 0
|
||||||
|
|
2
.github/workflows/codeql-analysis.yml
vendored
2
.github/workflows/codeql-analysis.yml
vendored
|
@ -22,8 +22,6 @@ jobs:
|
||||||
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
|
||||||
language: [ 'go', 'javascript' ]
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
|
|
1
.github/workflows/linters.yml
vendored
1
.github/workflows/linters.yml
vendored
|
@ -29,7 +29,6 @@ jobs:
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: "1.23.x"
|
go-version: "1.23.x"
|
||||||
- run: "go vet ./..."
|
|
||||||
- uses: golangci/golangci-lint-action@v6
|
- uses: golangci/golangci-lint-action@v6
|
||||||
with:
|
with:
|
||||||
args: >
|
args: >
|
||||||
|
|
14
.github/workflows/tests.yml
vendored
14
.github/workflows/tests.yml
vendored
|
@ -17,14 +17,18 @@ jobs:
|
||||||
os: [ubuntu-latest, windows-latest, macOS-latest]
|
os: [ubuntu-latest, windows-latest, macOS-latest]
|
||||||
go-version: ["1.23.x"]
|
go-version: ["1.23.x"]
|
||||||
steps:
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
- name: Checkout
|
- name: Run unit tests with coverage and race conditions checking
|
||||||
uses: actions/checkout@v4
|
if: matrix.os == 'ubuntu-latest'
|
||||||
- name: Run unit tests
|
|
||||||
run: make test
|
run: make test
|
||||||
|
- name: Run unit tests without coverage and race conditions checking
|
||||||
|
if: matrix.os != 'ubuntu-latest'
|
||||||
|
run: go test ./...
|
||||||
|
|
||||||
integration-tests:
|
integration-tests:
|
||||||
name: Integration Tests
|
name: Integration Tests
|
||||||
|
@ -40,12 +44,12 @@ jobs:
|
||||||
- 5432:5432
|
- 5432:5432
|
||||||
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||||||
steps:
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: "1.23.x"
|
go-version: "1.23.x"
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Install Postgres client
|
- name: Install Postgres client
|
||||||
run: sudo apt update && sudo apt install -y postgresql-client
|
run: sudo apt update && sudo apt install -y postgresql-client
|
||||||
- name: Run integration tests
|
- name: Run integration tests
|
||||||
|
|
61
ChangeLog
61
ChangeLog
|
@ -1,3 +1,64 @@
|
||||||
|
Version 2.2.4 (December 20, 2024)
|
||||||
|
---------------------------------
|
||||||
|
|
||||||
|
* test(rewrite): add unit test for referer rewrite function
|
||||||
|
* refactor(subscription): use `strings.HasSuffix` instead of a regex in `FindSubscriptionsFromYouTubePlaylistPage`
|
||||||
|
* refactor(sanitizer): use `token.String()` instead of `html.EscapeString(token.Data)`
|
||||||
|
* refactor(sanitizer): simplify `isValidTag`
|
||||||
|
* refactor(sanitizer): simplify `hasRequiredAttributes`
|
||||||
|
* refactor(sanitizer): remove condition because `config.Opts` is guaranteed to never be nil
|
||||||
|
* refactor(sanitizer): remove a now-useless function after refactoring
|
||||||
|
* refactor(sanitizer): refactor conditions to highlight their similitude, enabling further refactoring
|
||||||
|
* refactor(sanitizer): optimize `strip_tags.go`
|
||||||
|
* refactor(sanitizer): micro-optimizations of `srcset.go`
|
||||||
|
* refactor(sanitizer): merge two conditions
|
||||||
|
* refactor(sanitizer): inline a function in `sanitizeAttributes` and fix a bug in it
|
||||||
|
* refactor(sanitizer): inline a condition in `sanitizeSrcsetAttr`
|
||||||
|
* refactor(sanitizer): improve `rewriteIframeURL()`
|
||||||
|
* refactor(sanitizer): Google+ isn't a thing anymore
|
||||||
|
* refactor(sanitizer): change the scope of a variable
|
||||||
|
* refactor(rewriter): replace regex with URL parsing for referrer override
|
||||||
|
* refactor(rewriter): avoid the use of regex in `addDynamicImage`
|
||||||
|
* refactor(rewrite): remove unused function arguments
|
||||||
|
* refactor(readability): various improvements and optimizations
|
||||||
|
* refactor(readability): simplify the regexes in `readability.go`
|
||||||
|
* refactor(processor): use URL parsing instead of a regex
|
||||||
|
* refactor(processor): improve the `rewrite` URL rule regex
|
||||||
|
* refactor(locale): delay parsing of translations until they're used
|
||||||
|
* refactor(js): factorise a line in `app.js`
|
||||||
|
* refactor(handler): delay `store.UserByID()` as much as possible
|
||||||
|
* refactor(css): replace `-ms-text-size-adjust` with `text-size-adjust`
|
||||||
|
* refactor(css): remove `-webkit-clip-path`
|
||||||
|
* refactor(css): factorise `.pagination-next` and `.pagination-last` together
|
||||||
|
* refactor: use a better construct than `doc.Find(…).First()`
|
||||||
|
* refactor: use `min/max` instead of `math.Min/math.Max`
|
||||||
|
* refactor: refactor `internal/reader/readability/testdata`
|
||||||
|
* refactor: optimize `sanitizeAttributes`
|
||||||
|
* refactor: get rid of `numberOfPluralFormsPerLanguage` test-only variable
|
||||||
|
* fix(storage): replace timezone function call with view
|
||||||
|
* fix(consistency): align feed modification behavior between API and UI
|
||||||
|
* fix(ci): fix grammar in pull-request template
|
||||||
|
* fix: load icon from site URL instead of feed URL
|
||||||
|
* fix: feed icon from xml ignored during force refresh
|
||||||
|
* feat(rewrite)!: remove `parse_markdown` rewrite rule
|
||||||
|
* feat(mediaproxy): update predefined referer spoofing rules for restricted media resources
|
||||||
|
* feat(locale): update translations to clarify readeck URL instead of readeck API endpoint
|
||||||
|
* feat(locale): update German translations
|
||||||
|
* feat(locale): update Chinese translations
|
||||||
|
* feat(apprise): update `SendNotification` to handle multiple entries and add logging
|
||||||
|
* feat(apprise): add title in notification request body
|
||||||
|
* feat: resize favicons before storing them in the database
|
||||||
|
* feat: optionally fetch watch time from YouTube API instead of website
|
||||||
|
* feat: only show the commit URL if it's not empty on `/about`
|
||||||
|
* feat: add predefined scraper rules for `arstechnica.com`
|
||||||
|
* feat: add date-based entry filtering rules
|
||||||
|
* chore: remove `blog.laravel.com` rewrite rule
|
||||||
|
* build(deps): bump `library/alpine` in `/packaging/docker/alpine` to `3.21`
|
||||||
|
* build(deps): bump `golang.org/x/term` from `0.26.0` to `0.27.0`
|
||||||
|
* build(deps): bump `golang.org/x/net` from `0.31.0` to `0.33.0`
|
||||||
|
* build(deps): bump `golang.org/x/crypto` from `0.30.0` to `0.31.0`
|
||||||
|
* build(deps): bump `github.com/tdewolff/minify/v2` from `2.21.1` to `2.21.2`
|
||||||
|
|
||||||
Version 2.2.3 (November 10, 2024)
|
Version 2.2.3 (November 10, 2024)
|
||||||
---------------------------------
|
---------------------------------
|
||||||
|
|
||||||
|
|
7
go.mod
7
go.mod
|
@ -12,11 +12,11 @@ require (
|
||||||
github.com/lib/pq v1.10.9
|
github.com/lib/pq v1.10.9
|
||||||
github.com/prometheus/client_golang v1.20.5
|
github.com/prometheus/client_golang v1.20.5
|
||||||
github.com/tdewolff/minify/v2 v2.21.2
|
github.com/tdewolff/minify/v2 v2.21.2
|
||||||
golang.org/x/crypto v0.30.0
|
golang.org/x/crypto v0.31.0
|
||||||
golang.org/x/net v0.32.0
|
golang.org/x/image v0.23.0
|
||||||
|
golang.org/x/net v0.33.0
|
||||||
golang.org/x/oauth2 v0.24.0
|
golang.org/x/oauth2 v0.24.0
|
||||||
golang.org/x/term v0.27.0
|
golang.org/x/term v0.27.0
|
||||||
golang.org/x/text v0.21.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
@ -41,6 +41,7 @@ require (
|
||||||
github.com/tdewolff/parse/v2 v2.7.19 // indirect
|
github.com/tdewolff/parse/v2 v2.7.19 // indirect
|
||||||
github.com/x448/float16 v0.8.4 // indirect
|
github.com/x448/float16 v0.8.4 // indirect
|
||||||
golang.org/x/sys v0.28.0 // indirect
|
golang.org/x/sys v0.28.0 // indirect
|
||||||
|
golang.org/x/text v0.21.0 // indirect
|
||||||
google.golang.org/protobuf v1.34.2 // indirect
|
google.golang.org/protobuf v1.34.2 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
10
go.sum
10
go.sum
|
@ -68,8 +68,10 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i
|
||||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY=
|
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||||
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
|
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
|
||||||
|
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
@ -77,8 +79,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
|
||||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||||
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
|
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
||||||
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
|
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||||
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
|
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
|
||||||
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
|
|
@ -13,7 +13,6 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/database"
|
"miniflux.app/v2/internal/database"
|
||||||
"miniflux.app/v2/internal/locale"
|
|
||||||
"miniflux.app/v2/internal/storage"
|
"miniflux.app/v2/internal/storage"
|
||||||
"miniflux.app/v2/internal/ui/static"
|
"miniflux.app/v2/internal/ui/static"
|
||||||
"miniflux.app/v2/internal/version"
|
"miniflux.app/v2/internal/version"
|
||||||
|
@ -153,10 +152,6 @@ func Parse() {
|
||||||
slog.Info("The default value for DATABASE_URL is used")
|
slog.Info("The default value for DATABASE_URL is used")
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := locale.LoadCatalogMessages(); err != nil {
|
|
||||||
printErrorAndExit(fmt.Errorf("unable to load translations: %v", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := static.CalculateBinaryFileChecksums(); err != nil {
|
if err := static.CalculateBinaryFileChecksums(); err != nil {
|
||||||
printErrorAndExit(fmt.Errorf("unable to calculate binary file checksums: %v", err))
|
printErrorAndExit(fmt.Errorf("unable to calculate binary file checksums: %v", err))
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,17 +12,26 @@ import (
|
||||||
type translationDict map[string]interface{}
|
type translationDict map[string]interface{}
|
||||||
type catalog map[string]translationDict
|
type catalog map[string]translationDict
|
||||||
|
|
||||||
var defaultCatalog catalog
|
var defaultCatalog = make(catalog, len(AvailableLanguages))
|
||||||
|
|
||||||
//go:embed translations/*.json
|
//go:embed translations/*.json
|
||||||
var translationFiles embed.FS
|
var translationFiles embed.FS
|
||||||
|
|
||||||
|
func GetTranslationDict(language string) (translationDict, error) {
|
||||||
|
if _, ok := defaultCatalog[language]; !ok {
|
||||||
|
var err error
|
||||||
|
if defaultCatalog[language], err = loadTranslationFile(language); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultCatalog[language], nil
|
||||||
|
}
|
||||||
|
|
||||||
// LoadCatalogMessages loads and parses all translations encoded in JSON.
|
// LoadCatalogMessages loads and parses all translations encoded in JSON.
|
||||||
func LoadCatalogMessages() error {
|
func LoadCatalogMessages() error {
|
||||||
var err error
|
var err error
|
||||||
defaultCatalog = make(catalog, len(AvailableLanguages()))
|
|
||||||
|
|
||||||
for language := range AvailableLanguages() {
|
for language := range AvailableLanguages {
|
||||||
defaultCatalog[language], err = loadTranslationFile(language)
|
defaultCatalog[language], err = loadTranslationFile(language)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -39,7 +39,7 @@ func TestLoadCatalog(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAllKeysHaveValue(t *testing.T) {
|
func TestAllKeysHaveValue(t *testing.T) {
|
||||||
for language := range AvailableLanguages() {
|
for language := range AvailableLanguages {
|
||||||
messages, err := loadTranslationFile(language)
|
messages, err := loadTranslationFile(language)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf(`Unable to load translation messages for language %q`, language)
|
t.Fatalf(`Unable to load translation messages for language %q`, language)
|
||||||
|
@ -71,7 +71,7 @@ func TestMissingTranslations(t *testing.T) {
|
||||||
t.Fatal(`Unable to parse reference language`)
|
t.Fatal(`Unable to parse reference language`)
|
||||||
}
|
}
|
||||||
|
|
||||||
for language := range AvailableLanguages() {
|
for language := range AvailableLanguages {
|
||||||
if language == refLang {
|
if language == refLang {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -110,7 +110,7 @@ func TestTranslationFilePluralForms(t *testing.T) {
|
||||||
"uk_UA": 3,
|
"uk_UA": 3,
|
||||||
"id_ID": 1,
|
"id_ID": 1,
|
||||||
}
|
}
|
||||||
for language := range AvailableLanguages() {
|
for language := range AvailableLanguages {
|
||||||
messages, err := loadTranslationFile(language)
|
messages, err := loadTranslationFile(language)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf(`Unable to load translation messages for language %q`, language)
|
t.Fatalf(`Unable to load translation messages for language %q`, language)
|
||||||
|
|
|
@ -3,9 +3,8 @@
|
||||||
|
|
||||||
package locale // import "miniflux.app/v2/internal/locale"
|
package locale // import "miniflux.app/v2/internal/locale"
|
||||||
|
|
||||||
// AvailableLanguages returns the list of available languages.
|
// AvailableLanguages is the list of available languages.
|
||||||
func AvailableLanguages() map[string]string {
|
var AvailableLanguages = map[string]string{
|
||||||
return map[string]string{
|
|
||||||
"en_US": "English",
|
"en_US": "English",
|
||||||
"es_ES": "Español",
|
"es_ES": "Español",
|
||||||
"fr_FR": "Français",
|
"fr_FR": "Français",
|
||||||
|
@ -24,5 +23,4 @@ func AvailableLanguages() map[string]string {
|
||||||
"hi_IN": "हिन्दी",
|
"hi_IN": "हिन्दी",
|
||||||
"uk_UA": "Українська",
|
"uk_UA": "Українська",
|
||||||
"id_ID": "Bahasa Indonesia",
|
"id_ID": "Bahasa Indonesia",
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,7 @@ package locale // import "miniflux.app/v2/internal/locale"
|
||||||
import "testing"
|
import "testing"
|
||||||
|
|
||||||
func TestAvailableLanguages(t *testing.T) {
|
func TestAvailableLanguages(t *testing.T) {
|
||||||
results := AvailableLanguages()
|
results := AvailableLanguages
|
||||||
for k, v := range results {
|
for k, v := range results {
|
||||||
if k == "" {
|
if k == "" {
|
||||||
t.Errorf(`Empty language key detected`)
|
t.Errorf(`Empty language key detected`)
|
||||||
|
|
|
@ -11,37 +11,42 @@ type Printer struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Printer) Print(key string) string {
|
func (p *Printer) Print(key string) string {
|
||||||
if str, ok := defaultCatalog[p.language][key]; ok {
|
if dict, err := GetTranslationDict(p.language); err == nil {
|
||||||
|
if str, ok := dict[key]; ok {
|
||||||
if translation, ok := str.(string); ok {
|
if translation, ok := str.(string); ok {
|
||||||
return translation
|
return translation
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return key
|
return key
|
||||||
}
|
}
|
||||||
|
|
||||||
// Printf is like fmt.Printf, but using language-specific formatting.
|
// Printf is like fmt.Printf, but using language-specific formatting.
|
||||||
func (p *Printer) Printf(key string, args ...interface{}) string {
|
func (p *Printer) Printf(key string, args ...interface{}) string {
|
||||||
var translation string
|
translation := key
|
||||||
|
|
||||||
str, found := defaultCatalog[p.language][key]
|
if dict, err := GetTranslationDict(p.language); err == nil {
|
||||||
if !found {
|
str, found := dict[key]
|
||||||
translation = key
|
if found {
|
||||||
} else {
|
|
||||||
var valid bool
|
var valid bool
|
||||||
translation, valid = str.(string)
|
translation, valid = str.(string)
|
||||||
if !valid {
|
if !valid {
|
||||||
translation = key
|
translation = key
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return fmt.Sprintf(translation, args...)
|
return fmt.Sprintf(translation, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Plural returns the translation of the given key by using the language plural form.
|
// Plural returns the translation of the given key by using the language plural form.
|
||||||
func (p *Printer) Plural(key string, n int, args ...interface{}) string {
|
func (p *Printer) Plural(key string, n int, args ...interface{}) string {
|
||||||
choices, found := defaultCatalog[p.language][key]
|
dict, err := GetTranslationDict(p.language)
|
||||||
|
if err != nil {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
if found {
|
if choices, found := dict[key]; found {
|
||||||
var plurals []string
|
var plurals []string
|
||||||
|
|
||||||
switch v := choices.(type) {
|
switch v := choices.(type) {
|
||||||
|
|
|
@ -87,7 +87,7 @@ func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output, err := doc.Find("body").First().Html()
|
output, err := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return htmlDocument
|
return htmlDocument
|
||||||
}
|
}
|
||||||
|
|
|
@ -123,8 +123,8 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelayInMinutes int) {
|
||||||
intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
|
intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
|
||||||
} else {
|
} else {
|
||||||
intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount*config.Opts.SchedulerEntryFrequencyFactor())))
|
intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount*config.Opts.SchedulerEntryFrequencyFactor())))
|
||||||
intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
|
intervalMinutes = min(intervalMinutes, config.Opts.SchedulerEntryFrequencyMaxInterval())
|
||||||
intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
|
intervalMinutes = max(intervalMinutes, config.Opts.SchedulerEntryFrequencyMinInterval())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -31,11 +31,6 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
|
||||||
slog.String("feed_url", feedCreationRequest.FeedURL),
|
slog.String("feed_url", feedCreationRequest.FeedURL),
|
||||||
)
|
)
|
||||||
|
|
||||||
user, storeErr := store.UserByID(userID)
|
|
||||||
if storeErr != nil {
|
|
||||||
return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) {
|
if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) {
|
||||||
return nil, locale.NewLocalizedErrorWrapper(ErrCategoryNotFound, "error.category_not_found")
|
return nil, locale.NewLocalizedErrorWrapper(ErrCategoryNotFound, "error.category_not_found")
|
||||||
}
|
}
|
||||||
|
@ -71,7 +66,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
|
||||||
subscription.WithCategoryID(feedCreationRequest.CategoryID)
|
subscription.WithCategoryID(feedCreationRequest.CategoryID)
|
||||||
subscription.CheckedNow()
|
subscription.CheckedNow()
|
||||||
|
|
||||||
processor.ProcessFeedEntries(store, subscription, user, true)
|
processor.ProcessFeedEntries(store, subscription, userID, true)
|
||||||
|
|
||||||
if storeErr := store.CreateFeed(subscription); storeErr != nil {
|
if storeErr := store.CreateFeed(subscription); storeErr != nil {
|
||||||
return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
@ -105,11 +100,6 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
|
||||||
slog.String("feed_url", feedCreationRequest.FeedURL),
|
slog.String("feed_url", feedCreationRequest.FeedURL),
|
||||||
)
|
)
|
||||||
|
|
||||||
user, storeErr := store.UserByID(userID)
|
|
||||||
if storeErr != nil {
|
|
||||||
return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) {
|
if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) {
|
||||||
return nil, locale.NewLocalizedErrorWrapper(ErrCategoryNotFound, "error.category_not_found")
|
return nil, locale.NewLocalizedErrorWrapper(ErrCategoryNotFound, "error.category_not_found")
|
||||||
}
|
}
|
||||||
|
@ -170,7 +160,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
|
||||||
subscription.WithCategoryID(feedCreationRequest.CategoryID)
|
subscription.WithCategoryID(feedCreationRequest.CategoryID)
|
||||||
subscription.CheckedNow()
|
subscription.CheckedNow()
|
||||||
|
|
||||||
processor.ProcessFeedEntries(store, subscription, user, true)
|
processor.ProcessFeedEntries(store, subscription, userID, true)
|
||||||
|
|
||||||
if storeErr := store.CreateFeed(subscription); storeErr != nil {
|
if storeErr := store.CreateFeed(subscription); storeErr != nil {
|
||||||
return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
@ -195,11 +185,6 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
slog.Bool("force_refresh", forceRefresh),
|
slog.Bool("force_refresh", forceRefresh),
|
||||||
)
|
)
|
||||||
|
|
||||||
user, storeErr := store.UserByID(userID)
|
|
||||||
if storeErr != nil {
|
|
||||||
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
originalFeed, storeErr := store.FeedByID(userID, feedID)
|
originalFeed, storeErr := store.FeedByID(userID, feedID)
|
||||||
if storeErr != nil {
|
if storeErr != nil {
|
||||||
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
@ -256,6 +241,10 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
|
|
||||||
if localizedError := responseHandler.LocalizedError(); localizedError != nil {
|
if localizedError := responseHandler.LocalizedError(); localizedError != nil {
|
||||||
slog.Warn("Unable to fetch feed", slog.String("feed_url", originalFeed.FeedURL), slog.Any("error", localizedError.Error()))
|
slog.Warn("Unable to fetch feed", slog.String("feed_url", originalFeed.FeedURL), slog.Any("error", localizedError.Error()))
|
||||||
|
user, storeErr := store.UserByID(userID)
|
||||||
|
if storeErr != nil {
|
||||||
|
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
}
|
||||||
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
||||||
store.UpdateFeedError(originalFeed)
|
store.UpdateFeedError(originalFeed)
|
||||||
return localizedError
|
return localizedError
|
||||||
|
@ -263,6 +252,10 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
|
|
||||||
if store.AnotherFeedURLExists(userID, originalFeed.ID, responseHandler.EffectiveURL()) {
|
if store.AnotherFeedURLExists(userID, originalFeed.ID, responseHandler.EffectiveURL()) {
|
||||||
localizedError := locale.NewLocalizedErrorWrapper(ErrDuplicatedFeed, "error.duplicated_feed")
|
localizedError := locale.NewLocalizedErrorWrapper(ErrDuplicatedFeed, "error.duplicated_feed")
|
||||||
|
user, storeErr := store.UserByID(userID)
|
||||||
|
if storeErr != nil {
|
||||||
|
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
}
|
||||||
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
||||||
store.UpdateFeedError(originalFeed)
|
store.UpdateFeedError(originalFeed)
|
||||||
return localizedError
|
return localizedError
|
||||||
|
@ -289,6 +282,10 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
if errors.Is(parseErr, parser.ErrFeedFormatNotDetected) {
|
if errors.Is(parseErr, parser.ErrFeedFormatNotDetected) {
|
||||||
localizedError = locale.NewLocalizedErrorWrapper(parseErr, "error.feed_format_not_detected", parseErr)
|
localizedError = locale.NewLocalizedErrorWrapper(parseErr, "error.feed_format_not_detected", parseErr)
|
||||||
}
|
}
|
||||||
|
user, storeErr := store.UserByID(userID)
|
||||||
|
if storeErr != nil {
|
||||||
|
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
}
|
||||||
|
|
||||||
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
||||||
store.UpdateFeedError(originalFeed)
|
store.UpdateFeedError(originalFeed)
|
||||||
|
@ -309,13 +306,17 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
)
|
)
|
||||||
|
|
||||||
originalFeed.Entries = updatedFeed.Entries
|
originalFeed.Entries = updatedFeed.Entries
|
||||||
processor.ProcessFeedEntries(store, originalFeed, user, forceRefresh)
|
processor.ProcessFeedEntries(store, originalFeed, userID, forceRefresh)
|
||||||
|
|
||||||
// We don't update existing entries when the crawler is enabled (we crawl only inexisting entries). Unless it is forced to refresh
|
// We don't update existing entries when the crawler is enabled (we crawl only inexisting entries). Unless it is forced to refresh
|
||||||
updateExistingEntries := forceRefresh || !originalFeed.Crawler
|
updateExistingEntries := forceRefresh || !originalFeed.Crawler
|
||||||
newEntries, storeErr := store.RefreshFeedEntries(originalFeed.UserID, originalFeed.ID, originalFeed.Entries, updateExistingEntries)
|
newEntries, storeErr := store.RefreshFeedEntries(originalFeed.UserID, originalFeed.ID, originalFeed.Entries, updateExistingEntries)
|
||||||
if storeErr != nil {
|
if storeErr != nil {
|
||||||
localizedError := locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
localizedError := locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
user, storeErr := store.UserByID(userID)
|
||||||
|
if storeErr != nil {
|
||||||
|
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
}
|
||||||
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
||||||
store.UpdateFeedError(originalFeed)
|
store.UpdateFeedError(originalFeed)
|
||||||
return localizedError
|
return localizedError
|
||||||
|
@ -359,6 +360,10 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
|
|
||||||
if storeErr := store.UpdateFeed(originalFeed); storeErr != nil {
|
if storeErr := store.UpdateFeed(originalFeed); storeErr != nil {
|
||||||
localizedError := locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
localizedError := locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
user, storeErr := store.UserByID(userID)
|
||||||
|
if storeErr != nil {
|
||||||
|
return locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
|
||||||
|
}
|
||||||
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
originalFeed.WithTranslatedErrorMessage(localizedError.Translate(user.Language))
|
||||||
store.UpdateFeedError(originalFeed)
|
store.UpdateFeedError(originalFeed)
|
||||||
return localizedError
|
return localizedError
|
||||||
|
|
|
@ -4,12 +4,18 @@
|
||||||
package icon // import "miniflux.app/v2/internal/reader/icon"
|
package icon // import "miniflux.app/v2/internal/reader/icon"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"image"
|
||||||
|
"image/gif"
|
||||||
|
"image/jpeg"
|
||||||
|
"image/png"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
|
@ -19,6 +25,7 @@ import (
|
||||||
"miniflux.app/v2/internal/urllib"
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"golang.org/x/image/draw"
|
||||||
"golang.org/x/net/html/charset"
|
"golang.org/x/net/html/charset"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -180,9 +187,59 @@ func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) {
|
||||||
Content: responseBody,
|
Content: responseBody,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
icon = resizeIcon(icon)
|
||||||
|
|
||||||
return icon, nil
|
return icon, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resizeIcon(icon *model.Icon) *model.Icon {
|
||||||
|
r := bytes.NewReader(icon.Content)
|
||||||
|
|
||||||
|
if !slices.Contains([]string{"image/jpeg", "image/png", "image/gif"}, icon.MimeType) {
|
||||||
|
slog.Info("icon isn't a png/gif/jpeg/ico, can't resize", slog.String("mimetype", icon.MimeType))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
|
// Don't resize icons that we can't decode, or that already have the right size.
|
||||||
|
config, _, err := image.DecodeConfig(r)
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("unable to decode the metadata of the icon", slog.Any("error", err))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
if config.Height <= 32 && config.Width <= 32 {
|
||||||
|
slog.Debug("icon don't need to be rescaled", slog.Int("height", config.Height), slog.Int("width", config.Width))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
|
r.Seek(0, io.SeekStart)
|
||||||
|
|
||||||
|
var src image.Image
|
||||||
|
switch icon.MimeType {
|
||||||
|
case "image/jpeg":
|
||||||
|
src, err = jpeg.Decode(r)
|
||||||
|
case "image/png":
|
||||||
|
src, err = png.Decode(r)
|
||||||
|
case "image/gif":
|
||||||
|
src, err = gif.Decode(r)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("unable to decode the icon", slog.Any("error", err))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
|
dst := image.NewRGBA(image.Rect(0, 0, 32, 32))
|
||||||
|
draw.BiLinear.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
|
||||||
|
|
||||||
|
var b bytes.Buffer
|
||||||
|
if err = png.Encode(io.Writer(&b), dst); err != nil {
|
||||||
|
slog.Warn("unable to encode the new icon", slog.Any("error", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
icon.Content = b.Bytes()
|
||||||
|
icon.MimeType = "image/png"
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string, error) {
|
func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string, error) {
|
||||||
queries := []string{
|
queries := []string{
|
||||||
"link[rel='icon' i]",
|
"link[rel='icon' i]",
|
||||||
|
|
|
@ -4,8 +4,13 @@
|
||||||
package icon // import "miniflux.app/v2/internal/reader/icon"
|
package icon // import "miniflux.app/v2/internal/reader/icon"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"image"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseImageDataURL(t *testing.T) {
|
func TestParseImageDataURL(t *testing.T) {
|
||||||
|
@ -125,3 +130,52 @@ func TestParseDocumentWithWhitespaceIconURL(t *testing.T) {
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURLs[0])
|
t.Errorf(`Invalid icon URL, got %q`, iconURLs[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResizeIconSmallGif(t *testing.T) {
|
||||||
|
data, err := base64.StdEncoding.DecodeString("R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
icon := model.Icon{
|
||||||
|
Content: data,
|
||||||
|
MimeType: "image/gif",
|
||||||
|
}
|
||||||
|
if !bytes.Equal(icon.Content, resizeIcon(&icon).Content) {
|
||||||
|
t.Fatalf("Converted gif smaller than 16x16")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResizeIconPng(t *testing.T) {
|
||||||
|
data, err := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAACEAAAAhCAYAAABX5MJvAAAALUlEQVR42u3OMQEAAAgDoJnc6BpjDyRgcrcpGwkJCQkJCQkJCQkJCQkJCYmyB7NfUj/Kk4FkAAAAAElFTkSuQmCC")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
icon := model.Icon{
|
||||||
|
Content: data,
|
||||||
|
MimeType: "image/png",
|
||||||
|
}
|
||||||
|
resizedIcon := resizeIcon(&icon)
|
||||||
|
|
||||||
|
if bytes.Equal(data, resizedIcon.Content) {
|
||||||
|
t.Fatalf("Didn't convert png of 33x33")
|
||||||
|
}
|
||||||
|
|
||||||
|
config, _, err := image.DecodeConfig(bytes.NewReader(resizedIcon.Content))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Couln't decode resulting png: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.Height != 32 || config.Width != 32 {
|
||||||
|
t.Fatalf("Was expecting an image of 16x16, got %dx%d", config.Width, config.Height)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResizeInvalidImage(t *testing.T) {
|
||||||
|
icon := model.Icon{
|
||||||
|
Content: []byte("invalid data"),
|
||||||
|
MimeType: "image/gif",
|
||||||
|
}
|
||||||
|
if !bytes.Equal(icon.Content, resizeIcon(&icon).Content) {
|
||||||
|
t.Fatalf("Tried to convert an invalid image")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"regexp"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
@ -17,14 +17,17 @@ import (
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
)
|
)
|
||||||
|
|
||||||
var nebulaRegex = regexp.MustCompile(`^https://nebula\.tv`)
|
|
||||||
|
|
||||||
func shouldFetchNebulaWatchTime(entry *model.Entry) bool {
|
func shouldFetchNebulaWatchTime(entry *model.Entry) bool {
|
||||||
if !config.Opts.FetchNebulaWatchTime() {
|
if !config.Opts.FetchNebulaWatchTime() {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
matches := nebulaRegex.FindStringSubmatch(entry.URL)
|
|
||||||
return matches != nil
|
u, err := url.Parse(entry.URL)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return u.Hostname() == "nebula.tv"
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchNebulaWatchTime(websiteURL string) (int, error) {
|
func fetchNebulaWatchTime(websiteURL string) (int, error) {
|
||||||
|
@ -45,7 +48,7 @@ func fetchNebulaWatchTime(websiteURL string) (int, error) {
|
||||||
return 0, docErr
|
return 0, docErr
|
||||||
}
|
}
|
||||||
|
|
||||||
durs, exists := doc.Find(`meta[property="video:duration"]`).First().Attr("content")
|
durs, exists := doc.FindMatcher(goquery.Single(`meta[property="video:duration"]`)).Attr("content")
|
||||||
// durs contains video watch time in seconds
|
// durs contains video watch time in seconds
|
||||||
if !exists {
|
if !exists {
|
||||||
return 0, errors.New("duration has not found")
|
return 0, errors.New("duration has not found")
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"regexp"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
@ -17,14 +17,17 @@ import (
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
)
|
)
|
||||||
|
|
||||||
var odyseeRegex = regexp.MustCompile(`^https://odysee\.com`)
|
|
||||||
|
|
||||||
func shouldFetchOdyseeWatchTime(entry *model.Entry) bool {
|
func shouldFetchOdyseeWatchTime(entry *model.Entry) bool {
|
||||||
if !config.Opts.FetchOdyseeWatchTime() {
|
if !config.Opts.FetchOdyseeWatchTime() {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
matches := odyseeRegex.FindStringSubmatch(entry.URL)
|
|
||||||
return matches != nil
|
u, err := url.Parse(entry.URL)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return u.Hostname() == "odysee.com"
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchOdyseeWatchTime(websiteURL string) (int, error) {
|
func fetchOdyseeWatchTime(websiteURL string) (int, error) {
|
||||||
|
@ -45,7 +48,7 @@ func fetchOdyseeWatchTime(websiteURL string) (int, error) {
|
||||||
return 0, docErr
|
return 0, docErr
|
||||||
}
|
}
|
||||||
|
|
||||||
durs, exists := doc.Find(`meta[property="og:video:duration"]`).First().Attr("content")
|
durs, exists := doc.FindMatcher(goquery.Single(`meta[property="og:video:duration"]`)).Attr("content")
|
||||||
// durs contains video watch time in seconds
|
// durs contains video watch time in seconds
|
||||||
if !exists {
|
if !exists {
|
||||||
return 0, errors.New("duration has not found")
|
return 0, errors.New("duration has not found")
|
||||||
|
|
|
@ -10,6 +10,9 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/tdewolff/minify/v2"
|
||||||
|
"github.com/tdewolff/minify/v2/html"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/metric"
|
"miniflux.app/v2/internal/metric"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
@ -20,17 +23,20 @@ import (
|
||||||
"miniflux.app/v2/internal/reader/scraper"
|
"miniflux.app/v2/internal/reader/scraper"
|
||||||
"miniflux.app/v2/internal/reader/urlcleaner"
|
"miniflux.app/v2/internal/reader/urlcleaner"
|
||||||
"miniflux.app/v2/internal/storage"
|
"miniflux.app/v2/internal/storage"
|
||||||
|
|
||||||
"github.com/tdewolff/minify/v2"
|
|
||||||
"github.com/tdewolff/minify/v2/html"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var customReplaceRuleRegex = regexp.MustCompile(`rewrite\("([^"]+)"\|"([^"]+)"\)`)
|
var customReplaceRuleRegex = regexp.MustCompile(`rewrite\("([^"]+)"\|"([^"]+)"\)`)
|
||||||
|
|
||||||
// ProcessFeedEntries downloads original web page for entries and apply filters.
|
// ProcessFeedEntries downloads original web page for entries and apply filters.
|
||||||
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.User, forceRefresh bool) {
|
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64, forceRefresh bool) {
|
||||||
var filteredEntries model.Entries
|
var filteredEntries model.Entries
|
||||||
|
|
||||||
|
user, storeErr := store.UserByID(userID)
|
||||||
|
if storeErr != nil {
|
||||||
|
slog.Error("Database error", slog.Any("error", storeErr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Process older entries first
|
// Process older entries first
|
||||||
for i := len(feed.Entries) - 1; i >= 0; i-- {
|
for i := len(feed.Entries) - 1; i >= 0; i-- {
|
||||||
entry := feed.Entries[i]
|
entry := feed.Entries[i]
|
||||||
|
@ -135,6 +141,9 @@ func isBlockedEntry(feed *model.Feed, entry *model.Entry, user *model.User) bool
|
||||||
|
|
||||||
var match bool
|
var match bool
|
||||||
switch parts[0] {
|
switch parts[0] {
|
||||||
|
case "EntryDate":
|
||||||
|
datePattern := parts[1]
|
||||||
|
match = isDateMatchingPattern(entry.Date, datePattern)
|
||||||
case "EntryTitle":
|
case "EntryTitle":
|
||||||
match, _ = regexp.MatchString(parts[1], entry.Title)
|
match, _ = regexp.MatchString(parts[1], entry.Title)
|
||||||
case "EntryURL":
|
case "EntryURL":
|
||||||
|
@ -205,6 +214,9 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry, user *model.User) bool
|
||||||
|
|
||||||
var match bool
|
var match bool
|
||||||
switch parts[0] {
|
switch parts[0] {
|
||||||
|
case "EntryDate":
|
||||||
|
datePattern := parts[1]
|
||||||
|
match = isDateMatchingPattern(entry.Date, datePattern)
|
||||||
case "EntryTitle":
|
case "EntryTitle":
|
||||||
match, _ = regexp.MatchString(parts[1], entry.Title)
|
match, _ = regexp.MatchString(parts[1], entry.Title)
|
||||||
case "EntryURL":
|
case "EntryURL":
|
||||||
|
@ -456,3 +468,44 @@ func minifyEntryContent(entryContent string) string {
|
||||||
|
|
||||||
return entryContent
|
return entryContent
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isDateMatchingPattern(entryDate time.Time, pattern string) bool {
|
||||||
|
if pattern == "future" {
|
||||||
|
return entryDate.After(time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := strings.SplitN(pattern, ":", 2)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
operator := parts[0]
|
||||||
|
dateStr := parts[1]
|
||||||
|
|
||||||
|
switch operator {
|
||||||
|
case "before":
|
||||||
|
targetDate, err := time.Parse("2006-01-02", dateStr)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return entryDate.Before(targetDate)
|
||||||
|
case "after":
|
||||||
|
targetDate, err := time.Parse("2006-01-02", dateStr)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return entryDate.After(targetDate)
|
||||||
|
case "between":
|
||||||
|
dates := strings.Split(dateStr, ",")
|
||||||
|
if len(dates) != 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
startDate, err1 := time.Parse("2006-01-02", dates[0])
|
||||||
|
endDate, err2 := time.Parse("2006-01-02", dates[1])
|
||||||
|
if err1 != nil || err2 != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return entryDate.After(startDate) && entryDate.Before(endDate)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
|
@ -75,6 +75,12 @@ func TestAllowEntries(t *testing.T) {
|
||||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Example", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, true},
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Example", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, true},
|
||||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)example"}, true},
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)example"}, true},
|
||||||
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, false},
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, false},
|
||||||
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Now().Add(24 * time.Hour)}, &model.User{KeepFilterEntryRules: "EntryDate=future"}, true},
|
||||||
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Now().Add(-24 * time.Hour)}, &model.User{KeepFilterEntryRules: "EntryDate=future"}, false},
|
||||||
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 14, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=before:2024-03-15"}, true},
|
||||||
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 16, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=after:2024-03-15"}, true},
|
||||||
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 10, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=between:2024-03-01,2024-03-15"}, true},
|
||||||
|
{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 2, 28, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=between:2024-03-01,2024-03-15"}, false},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range scenarios {
|
for _, tc := range scenarios {
|
||||||
|
|
|
@ -60,7 +60,7 @@ func fetchYouTubeWatchTimeFromWebsite(websiteURL string) (int, error) {
|
||||||
return 0, docErr
|
return 0, docErr
|
||||||
}
|
}
|
||||||
|
|
||||||
durs, exists := doc.Find(`meta[itemprop="duration"]`).First().Attr("content")
|
durs, exists := doc.FindMatcher(goquery.Single(`meta[itemprop="duration"]`)).Attr("content")
|
||||||
if !exists {
|
if !exists {
|
||||||
return 0, errors.New("duration has not found")
|
return 0, errors.New("duration has not found")
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,11 +4,9 @@
|
||||||
package readability // import "miniflux.app/v2/internal/reader/readability"
|
package readability // import "miniflux.app/v2/internal/reader/readability"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
@ -24,9 +22,7 @@ const (
|
||||||
|
|
||||||
var (
|
var (
|
||||||
divToPElementsRegexp = regexp.MustCompile(`(?i)<(a|blockquote|dl|div|img|ol|p|pre|table|ul)`)
|
divToPElementsRegexp = regexp.MustCompile(`(?i)<(a|blockquote|dl|div|img|ol|p|pre|table|ul)`)
|
||||||
sentenceRegexp = regexp.MustCompile(`\.( |$)`)
|
|
||||||
|
|
||||||
blacklistCandidatesRegexp = regexp.MustCompile(`popupbody|-ad|g-plus`)
|
|
||||||
okMaybeItsACandidateRegexp = regexp.MustCompile(`and|article|body|column|main|shadow`)
|
okMaybeItsACandidateRegexp = regexp.MustCompile(`and|article|body|column|main|shadow`)
|
||||||
unlikelyCandidatesRegexp = regexp.MustCompile(`banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
|
unlikelyCandidatesRegexp = regexp.MustCompile(`banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
|
||||||
|
|
||||||
|
@ -77,16 +73,14 @@ func ExtractContent(page io.Reader) (baseURL string, extractedContent string, er
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if hrefValue, exists := document.Find("head base").First().Attr("href"); exists {
|
if hrefValue, exists := document.FindMatcher(goquery.Single("head base")).Attr("href"); exists {
|
||||||
hrefValue = strings.TrimSpace(hrefValue)
|
hrefValue = strings.TrimSpace(hrefValue)
|
||||||
if urllib.IsAbsoluteURL(hrefValue) {
|
if urllib.IsAbsoluteURL(hrefValue) {
|
||||||
baseURL = hrefValue
|
baseURL = hrefValue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
document.Find("script,style").Each(func(i int, s *goquery.Selection) {
|
document.Find("script,style").Remove()
|
||||||
removeNodes(s)
|
|
||||||
})
|
|
||||||
|
|
||||||
transformMisusedDivsIntoParagraphs(document)
|
transformMisusedDivsIntoParagraphs(document)
|
||||||
removeUnlikelyCandidates(document)
|
removeUnlikelyCandidates(document)
|
||||||
|
@ -107,8 +101,9 @@ func ExtractContent(page io.Reader) (baseURL string, extractedContent string, er
|
||||||
// Now that we have the top candidate, look through its siblings for content that might also be related.
|
// Now that we have the top candidate, look through its siblings for content that might also be related.
|
||||||
// Things like preambles, content split by ads that we removed, etc.
|
// Things like preambles, content split by ads that we removed, etc.
|
||||||
func getArticle(topCandidate *candidate, candidates candidateList) string {
|
func getArticle(topCandidate *candidate, candidates candidateList) string {
|
||||||
output := bytes.NewBufferString("<div>")
|
var output strings.Builder
|
||||||
siblingScoreThreshold := float32(math.Max(10, float64(topCandidate.score*.2)))
|
output.WriteString("<div>")
|
||||||
|
siblingScoreThreshold := max(10, topCandidate.score*.2)
|
||||||
|
|
||||||
topCandidate.selection.Siblings().Union(topCandidate.selection).Each(func(i int, s *goquery.Selection) {
|
topCandidate.selection.Siblings().Union(topCandidate.selection).Each(func(i int, s *goquery.Selection) {
|
||||||
append := false
|
append := false
|
||||||
|
@ -125,12 +120,16 @@ func getArticle(topCandidate *candidate, candidates candidateList) string {
|
||||||
content := s.Text()
|
content := s.Text()
|
||||||
contentLength := len(content)
|
contentLength := len(content)
|
||||||
|
|
||||||
if contentLength >= 80 && linkDensity < .25 {
|
if contentLength >= 80 {
|
||||||
|
if linkDensity < .25 {
|
||||||
append = true
|
append = true
|
||||||
} else if contentLength < 80 && linkDensity == 0 && sentenceRegexp.MatchString(content) {
|
}
|
||||||
|
} else {
|
||||||
|
if linkDensity == 0 && containsSentence(content) {
|
||||||
append = true
|
append = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if append {
|
if append {
|
||||||
tag := "div"
|
tag := "div"
|
||||||
|
@ -139,7 +138,7 @@ func getArticle(topCandidate *candidate, candidates candidateList) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
html, _ := s.Html()
|
html, _ := s.Html()
|
||||||
fmt.Fprintf(output, "<%s>%s</%s>", tag, html, tag)
|
output.WriteString("<" + tag + ">" + html + "</" + tag + ">")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -148,18 +147,29 @@ func getArticle(topCandidate *candidate, candidates candidateList) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeUnlikelyCandidates(document *goquery.Document) {
|
func removeUnlikelyCandidates(document *goquery.Document) {
|
||||||
|
var shouldRemove = func(str string) bool {
|
||||||
|
str = strings.ToLower(str)
|
||||||
|
if strings.Contains(str, "popupbody") || strings.Contains(str, "-ad") || strings.Contains(str, "g-plus") {
|
||||||
|
return true
|
||||||
|
} else if unlikelyCandidatesRegexp.MatchString(str) && !okMaybeItsACandidateRegexp.MatchString(str) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
document.Find("*").Each(func(i int, s *goquery.Selection) {
|
document.Find("*").Each(func(i int, s *goquery.Selection) {
|
||||||
if s.Length() == 0 || s.Get(0).Data == "html" || s.Get(0).Data == "body" {
|
if s.Length() == 0 || s.Get(0).Data == "html" || s.Get(0).Data == "body" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
class, _ := s.Attr("class")
|
|
||||||
id, _ := s.Attr("id")
|
|
||||||
str := strings.ToLower(class + id)
|
|
||||||
|
|
||||||
if blacklistCandidatesRegexp.MatchString(str) {
|
if class, ok := s.Attr("class"); ok {
|
||||||
removeNodes(s)
|
if shouldRemove(class) {
|
||||||
} else if unlikelyCandidatesRegexp.MatchString(str) && !okMaybeItsACandidateRegexp.MatchString(str) {
|
s.Remove()
|
||||||
removeNodes(s)
|
}
|
||||||
|
} else if id, ok := s.Attr("id"); ok {
|
||||||
|
if shouldRemove(id) {
|
||||||
|
s.Remove()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -223,7 +233,7 @@ func getCandidates(document *goquery.Document) candidateList {
|
||||||
contentScore += float32(strings.Count(text, ",") + 1)
|
contentScore += float32(strings.Count(text, ",") + 1)
|
||||||
|
|
||||||
// For every 100 characters in this paragraph, add another point. Up to 3 points.
|
// For every 100 characters in this paragraph, add another point. Up to 3 points.
|
||||||
contentScore += float32(math.Min(float64(int(len(text)/100.0)), 3))
|
contentScore += float32(min(len(text)/100.0, 3))
|
||||||
|
|
||||||
candidates[parentNode].score += contentScore
|
candidates[parentNode].score += contentScore
|
||||||
if grandParentNode != nil {
|
if grandParentNode != nil {
|
||||||
|
@ -262,13 +272,14 @@ func scoreNode(s *goquery.Selection) *candidate {
|
||||||
// Get the density of links as a percentage of the content
|
// Get the density of links as a percentage of the content
|
||||||
// This is the amount of text that is inside a link divided by the total text in the node.
|
// This is the amount of text that is inside a link divided by the total text in the node.
|
||||||
func getLinkDensity(s *goquery.Selection) float32 {
|
func getLinkDensity(s *goquery.Selection) float32 {
|
||||||
linkLength := len(s.Find("a").Text())
|
|
||||||
textLength := len(s.Text())
|
textLength := len(s.Text())
|
||||||
|
|
||||||
if textLength == 0 {
|
if textLength == 0 {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
linkLength := len(s.Find("a").Text())
|
||||||
|
|
||||||
return float32(linkLength) / float32(textLength)
|
return float32(linkLength) / float32(textLength)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -276,28 +287,21 @@ func getLinkDensity(s *goquery.Selection) float32 {
|
||||||
// element looks good or bad.
|
// element looks good or bad.
|
||||||
func getClassWeight(s *goquery.Selection) float32 {
|
func getClassWeight(s *goquery.Selection) float32 {
|
||||||
weight := 0
|
weight := 0
|
||||||
class, _ := s.Attr("class")
|
|
||||||
id, _ := s.Attr("id")
|
|
||||||
|
|
||||||
|
if class, ok := s.Attr("class"); ok {
|
||||||
class = strings.ToLower(class)
|
class = strings.ToLower(class)
|
||||||
id = strings.ToLower(id)
|
|
||||||
|
|
||||||
if class != "" {
|
|
||||||
if negativeRegexp.MatchString(class) {
|
if negativeRegexp.MatchString(class) {
|
||||||
weight -= 25
|
weight -= 25
|
||||||
}
|
} else if positiveRegexp.MatchString(class) {
|
||||||
|
|
||||||
if positiveRegexp.MatchString(class) {
|
|
||||||
weight += 25
|
weight += 25
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if id != "" {
|
if id, ok := s.Attr("id"); ok {
|
||||||
|
id = strings.ToLower(id)
|
||||||
if negativeRegexp.MatchString(id) {
|
if negativeRegexp.MatchString(id) {
|
||||||
weight -= 25
|
weight -= 25
|
||||||
}
|
} else if positiveRegexp.MatchString(id) {
|
||||||
|
|
||||||
if positiveRegexp.MatchString(id) {
|
|
||||||
weight += 25
|
weight += 25
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -315,11 +319,6 @@ func transformMisusedDivsIntoParagraphs(document *goquery.Document) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeNodes(s *goquery.Selection) {
|
func containsSentence(content string) bool {
|
||||||
s.Each(func(i int, s *goquery.Selection) {
|
return strings.HasSuffix(content, ".") || strings.Contains(content, ". ")
|
||||||
parent := s.Parent()
|
|
||||||
if parent.Length() > 0 {
|
|
||||||
parent.Get(0).RemoveChild(s.Get(0))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,8 @@
|
||||||
package readability // import "miniflux.app/v2/internal/reader/readability"
|
package readability // import "miniflux.app/v2/internal/reader/readability"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
@ -100,3 +102,83 @@ func TestWithoutBaseURL(t *testing.T) {
|
||||||
t.Errorf(`Unexpected base URL, got %q instead of ""`, baseURL)
|
t.Errorf(`Unexpected base URL, got %q instead of ""`, baseURL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRemoveStyleScript(t *testing.T) {
|
||||||
|
html := `
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Test</title>
|
||||||
|
<script src="tololo.js"></script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<script src="tololo.js"></script>
|
||||||
|
<style>
|
||||||
|
h1 {color:red;}
|
||||||
|
p {color:blue;}
|
||||||
|
</style>
|
||||||
|
<article>Some content</article>
|
||||||
|
</body>
|
||||||
|
</html>`
|
||||||
|
want := `<div><div><article>Somecontent</article></div></div>`
|
||||||
|
|
||||||
|
_, content, err := ExtractContent(strings.NewReader(html))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
content = strings.ReplaceAll(content, "\n", "")
|
||||||
|
content = strings.ReplaceAll(content, " ", "")
|
||||||
|
content = strings.ReplaceAll(content, "\t", "")
|
||||||
|
|
||||||
|
if content != want {
|
||||||
|
t.Errorf(`Invalid content, got %s instead of %s`, content, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoveBlacklist(t *testing.T) {
|
||||||
|
html := `
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Test</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<article class="super-ad">Some content</article>
|
||||||
|
<article class="g-plus-crap">Some other thing</article>
|
||||||
|
<article class="stuff popupbody">And more</article>
|
||||||
|
<article class="legit">Valid!</article>
|
||||||
|
</body>
|
||||||
|
</html>`
|
||||||
|
want := `<div><div><articleclass="legit">Valid!</article></div></div>`
|
||||||
|
|
||||||
|
_, content, err := ExtractContent(strings.NewReader(html))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
content = strings.ReplaceAll(content, "\n", "")
|
||||||
|
content = strings.ReplaceAll(content, " ", "")
|
||||||
|
content = strings.ReplaceAll(content, "\t", "")
|
||||||
|
|
||||||
|
if content != want {
|
||||||
|
t.Errorf(`Invalid content, got %s instead of %s`, content, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkExtractContent(b *testing.B) {
|
||||||
|
var testCases = map[string][]byte{
|
||||||
|
"miniflux_github.html": {},
|
||||||
|
"miniflux_wikipedia.html": {},
|
||||||
|
}
|
||||||
|
for filename := range testCases {
|
||||||
|
data, err := os.ReadFile("testdata/" + filename)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf(`Unable to read file %q: %v`, filename, err)
|
||||||
|
}
|
||||||
|
testCases[filename] = data
|
||||||
|
}
|
||||||
|
for range b.N {
|
||||||
|
for _, v := range testCases {
|
||||||
|
ExtractContent(bytes.NewReader(v))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
1
internal/reader/readability/testdata
Symbolic link
1
internal/reader/readability/testdata
Symbolic link
|
@ -0,0 +1 @@
|
||||||
|
../../reader/sanitizer/testdata/
|
|
@ -19,7 +19,7 @@ func EstimateReadingTime(content string, defaultReadingSpeed, cjkReadingSpeed in
|
||||||
sanitizedContent := sanitizer.StripTags(content)
|
sanitizedContent := sanitizer.StripTags(content)
|
||||||
|
|
||||||
// Litterature on language detection says that around 100 signes is enough, we're safe here.
|
// Litterature on language detection says that around 100 signes is enough, we're safe here.
|
||||||
truncationPoint := int(math.Min(float64(len(sanitizedContent)), 250))
|
truncationPoint := min(len(sanitizedContent), 250)
|
||||||
|
|
||||||
// We're only interested in identifying Japanse/Chinese/Korean
|
// We're only interested in identifying Japanse/Chinese/Korean
|
||||||
options := whatlanggo.Options{
|
options := whatlanggo.Options{
|
||||||
|
|
67
internal/reader/rewrite/referer_override_test.go
Normal file
67
internal/reader/rewrite/referer_override_test.go
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package rewrite // import "miniflux.app/v2/internal/reader/rewrite"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetRefererForURL(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Weibo Image URL",
|
||||||
|
url: "https://wx1.sinaimg.cn/large/example.jpg",
|
||||||
|
expected: "https://weibo.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Pixiv Image URL",
|
||||||
|
url: "https://i.pximg.net/img-master/example.jpg",
|
||||||
|
expected: "https://www.pixiv.net",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "SSPai CDN URL",
|
||||||
|
url: "https://cdnfile.sspai.com/example.png",
|
||||||
|
expected: "https://sspai.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Instagram CDN URL",
|
||||||
|
url: "https://scontent-sjc3-1.cdninstagram.com/example.jpg",
|
||||||
|
expected: "https://www.instagram.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Piokok URL",
|
||||||
|
url: "https://sp1.piokok.com/example.jpg",
|
||||||
|
expected: "https://sp1.piokok.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Weibo Video URL",
|
||||||
|
url: "https://f.video.weibocdn.com/example.mp4",
|
||||||
|
expected: "https://weibo.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "HelloGithub Image URL",
|
||||||
|
url: "https://img.hellogithub.com/example.png",
|
||||||
|
expected: "https://hellogithub.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Non-matching URL",
|
||||||
|
url: "https://example.com/image.jpg",
|
||||||
|
expected: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result := GetRefererForURL(tc.url)
|
||||||
|
if result != tc.expected {
|
||||||
|
t.Errorf("GetRefererForURL(%s): expected %s, got %s",
|
||||||
|
tc.url, tc.expected, result)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
|
|
||||||
|
@ -23,11 +24,28 @@ var (
|
||||||
youtubeRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)$`)
|
youtubeRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)$`)
|
||||||
youtubeIdRegex = regexp.MustCompile(`youtube_id"?\s*[:=]\s*"([a-zA-Z0-9_-]{11})"`)
|
youtubeIdRegex = regexp.MustCompile(`youtube_id"?\s*[:=]\s*"([a-zA-Z0-9_-]{11})"`)
|
||||||
invidioRegex = regexp.MustCompile(`https?://(.*)/watch\?v=(.*)`)
|
invidioRegex = regexp.MustCompile(`https?://(.*)/watch\?v=(.*)`)
|
||||||
imgRegex = regexp.MustCompile(`<img [^>]+>`)
|
|
||||||
textLinkRegex = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`)
|
textLinkRegex = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`)
|
||||||
)
|
)
|
||||||
|
|
||||||
func addImageTitle(entryURL, entryContent string) string {
|
// titlelize returns a copy of the string s with all Unicode letters that begin words
|
||||||
|
// mapped to their Unicode title case.
|
||||||
|
func titlelize(s string) string {
|
||||||
|
// A closure is used here to remember the previous character
|
||||||
|
// so that we can check if there is a space preceding the current
|
||||||
|
// character.
|
||||||
|
previous := ' '
|
||||||
|
return strings.Map(
|
||||||
|
func(current rune) rune {
|
||||||
|
if unicode.IsSpace(previous) {
|
||||||
|
previous = current
|
||||||
|
return unicode.ToTitle(current)
|
||||||
|
}
|
||||||
|
previous = current
|
||||||
|
return current
|
||||||
|
}, strings.ToLower(s))
|
||||||
|
}
|
||||||
|
|
||||||
|
func addImageTitle(entryContent string) string {
|
||||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entryContent
|
return entryContent
|
||||||
|
@ -44,14 +62,14 @@ func addImageTitle(entryURL, entryContent string) string {
|
||||||
img.ReplaceWithHtml(`<figure><img src="` + srcAttr + `" alt="` + altAttr + `"/><figcaption><p>` + html.EscapeString(titleAttr) + `</p></figcaption></figure>`)
|
img.ReplaceWithHtml(`<figure><img src="` + srcAttr + `" alt="` + altAttr + `"/><figcaption><p>` + html.EscapeString(titleAttr) + `</p></figcaption></figure>`)
|
||||||
})
|
})
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
return entryContent
|
return entryContent
|
||||||
}
|
}
|
||||||
|
|
||||||
func addMailtoSubject(entryURL, entryContent string) string {
|
func addMailtoSubject(entryContent string) string {
|
||||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entryContent
|
return entryContent
|
||||||
|
@ -76,18 +94,19 @@ func addMailtoSubject(entryURL, entryContent string) string {
|
||||||
a.AppendHtml(" [" + html.EscapeString(subject) + "]")
|
a.AppendHtml(" [" + html.EscapeString(subject) + "]")
|
||||||
})
|
})
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
return entryContent
|
return entryContent
|
||||||
}
|
}
|
||||||
|
|
||||||
func addDynamicImage(entryURL, entryContent string) string {
|
func addDynamicImage(entryContent string) string {
|
||||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
parserHtml, err := nethtml.ParseWithOptions(strings.NewReader(entryContent), nethtml.ParseOptionEnableScripting(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entryContent
|
return entryContent
|
||||||
}
|
}
|
||||||
|
doc := goquery.NewDocumentFromNode(parserHtml)
|
||||||
|
|
||||||
// Ordered most preferred to least preferred.
|
// Ordered most preferred to least preferred.
|
||||||
candidateAttrs := []string{
|
candidateAttrs := []string{
|
||||||
|
@ -149,25 +168,22 @@ func addDynamicImage(entryURL, entryContent string) string {
|
||||||
|
|
||||||
if !changed {
|
if !changed {
|
||||||
doc.Find("noscript").Each(func(i int, noscript *goquery.Selection) {
|
doc.Find("noscript").Each(func(i int, noscript *goquery.Selection) {
|
||||||
matches := imgRegex.FindAllString(noscript.Text(), 2)
|
if img := noscript.Find("img"); img.Length() == 1 {
|
||||||
|
img.Unwrap()
|
||||||
if len(matches) == 1 {
|
|
||||||
changed = true
|
changed = true
|
||||||
|
|
||||||
noscript.ReplaceWithHtml(matches[0])
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if changed {
|
if changed {
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
return entryContent
|
return entryContent
|
||||||
}
|
}
|
||||||
|
|
||||||
func addDynamicIframe(entryURL, entryContent string) string {
|
func addDynamicIframe(entryContent string) string {
|
||||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entryContent
|
return entryContent
|
||||||
|
@ -197,14 +213,14 @@ func addDynamicIframe(entryURL, entryContent string) string {
|
||||||
})
|
})
|
||||||
|
|
||||||
if changed {
|
if changed {
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
return entryContent
|
return entryContent
|
||||||
}
|
}
|
||||||
|
|
||||||
func fixMediumImages(entryURL, entryContent string) string {
|
func fixMediumImages(entryContent string) string {
|
||||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entryContent
|
return entryContent
|
||||||
|
@ -217,11 +233,11 @@ func fixMediumImages(entryURL, entryContent string) string {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
func useNoScriptImages(entryURL, entryContent string) string {
|
func useNoScriptImages(entryContent string) string {
|
||||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entryContent
|
return entryContent
|
||||||
|
@ -239,7 +255,7 @@ func useNoScriptImages(entryURL, entryContent string) string {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -317,7 +333,7 @@ func removeCustom(entryContent string, selector string) string {
|
||||||
|
|
||||||
doc.Find(selector).Remove()
|
doc.Find(selector).Remove()
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -344,7 +360,7 @@ func applyFuncOnTextContent(entryContent string, selector string, repl func(stri
|
||||||
|
|
||||||
doc.Find(selector).Each(treatChildren)
|
doc.Find(selector).Each(treatChildren)
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -401,7 +417,7 @@ func addHackerNewsLinksUsing(entryContent, app string) string {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -420,7 +436,7 @@ func removeTables(entryContent string) string {
|
||||||
|
|
||||||
for _, selector := range selectors {
|
for _, selector := range selectors {
|
||||||
for {
|
for {
|
||||||
loopElement = doc.Find(selector).First()
|
loopElement = doc.FindMatcher(goquery.Single(selector))
|
||||||
|
|
||||||
if loopElement.Length() == 0 {
|
if loopElement.Length() == 0 {
|
||||||
break
|
break
|
||||||
|
@ -436,6 +452,6 @@ func removeTables(entryContent string) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output, _ := doc.Find("body").First().Html()
|
output, _ := doc.FindMatcher(goquery.Single("body")).Html()
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,9 +11,6 @@ import (
|
||||||
|
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
"miniflux.app/v2/internal/urllib"
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
|
||||||
"golang.org/x/text/cases"
|
|
||||||
"golang.org/x/text/language"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type rule struct {
|
type rule struct {
|
||||||
|
@ -24,13 +21,13 @@ type rule struct {
|
||||||
func (rule rule) applyRule(entryURL string, entry *model.Entry) {
|
func (rule rule) applyRule(entryURL string, entry *model.Entry) {
|
||||||
switch rule.name {
|
switch rule.name {
|
||||||
case "add_image_title":
|
case "add_image_title":
|
||||||
entry.Content = addImageTitle(entryURL, entry.Content)
|
entry.Content = addImageTitle(entry.Content)
|
||||||
case "add_mailto_subject":
|
case "add_mailto_subject":
|
||||||
entry.Content = addMailtoSubject(entryURL, entry.Content)
|
entry.Content = addMailtoSubject(entry.Content)
|
||||||
case "add_dynamic_image":
|
case "add_dynamic_image":
|
||||||
entry.Content = addDynamicImage(entryURL, entry.Content)
|
entry.Content = addDynamicImage(entry.Content)
|
||||||
case "add_dynamic_iframe":
|
case "add_dynamic_iframe":
|
||||||
entry.Content = addDynamicIframe(entryURL, entry.Content)
|
entry.Content = addDynamicIframe(entry.Content)
|
||||||
case "add_youtube_video":
|
case "add_youtube_video":
|
||||||
entry.Content = addYoutubeVideo(entryURL, entry.Content)
|
entry.Content = addYoutubeVideo(entryURL, entry.Content)
|
||||||
case "add_invidious_video":
|
case "add_invidious_video":
|
||||||
|
@ -46,9 +43,9 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) {
|
||||||
case "convert_text_link", "convert_text_links":
|
case "convert_text_link", "convert_text_links":
|
||||||
entry.Content = replaceTextLinks(entry.Content)
|
entry.Content = replaceTextLinks(entry.Content)
|
||||||
case "fix_medium_images":
|
case "fix_medium_images":
|
||||||
entry.Content = fixMediumImages(entryURL, entry.Content)
|
entry.Content = fixMediumImages(entry.Content)
|
||||||
case "use_noscript_figure_images":
|
case "use_noscript_figure_images":
|
||||||
entry.Content = useNoScriptImages(entryURL, entry.Content)
|
entry.Content = useNoScriptImages(entry.Content)
|
||||||
case "replace":
|
case "replace":
|
||||||
// Format: replace("search-term"|"replace-term")
|
// Format: replace("search-term"|"replace-term")
|
||||||
if len(rule.args) >= 2 {
|
if len(rule.args) >= 2 {
|
||||||
|
@ -94,7 +91,7 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) {
|
||||||
case "remove_tables":
|
case "remove_tables":
|
||||||
entry.Content = removeTables(entry.Content)
|
entry.Content = removeTables(entry.Content)
|
||||||
case "remove_clickbait":
|
case "remove_clickbait":
|
||||||
entry.Title = cases.Title(language.English).String(strings.ToLower(entry.Title))
|
entry.Title = titlelize(entry.Title)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -256,7 +256,7 @@ func TestRewriteWithNoLazyImage(t *testing.T) {
|
||||||
func TestRewriteWithLazyImage(t *testing.T) {
|
func TestRewriteWithLazyImage(t *testing.T) {
|
||||||
controlEntry := &model.Entry{
|
controlEntry := &model.Entry{
|
||||||
Title: `A title`,
|
Title: `A title`,
|
||||||
Content: `<img src="https://example.org/image.jpg" data-url="https://example.org/image.jpg" alt="Image"/><noscript><img src="https://example.org/fallback.jpg" alt="Fallback"></noscript>`,
|
Content: `<img src="https://example.org/image.jpg" data-url="https://example.org/image.jpg" alt="Image"/><noscript><img src="https://example.org/fallback.jpg" alt="Fallback"/></noscript>`,
|
||||||
}
|
}
|
||||||
testEntry := &model.Entry{
|
testEntry := &model.Entry{
|
||||||
Title: `A title`,
|
Title: `A title`,
|
||||||
|
@ -272,7 +272,7 @@ func TestRewriteWithLazyImage(t *testing.T) {
|
||||||
func TestRewriteWithLazyDivImage(t *testing.T) {
|
func TestRewriteWithLazyDivImage(t *testing.T) {
|
||||||
controlEntry := &model.Entry{
|
controlEntry := &model.Entry{
|
||||||
Title: `A title`,
|
Title: `A title`,
|
||||||
Content: `<img src="https://example.org/image.jpg" alt="Image"/><noscript><img src="https://example.org/fallback.jpg" alt="Fallback"></noscript>`,
|
Content: `<img src="https://example.org/image.jpg" alt="Image"/><noscript><img src="https://example.org/fallback.jpg" alt="Fallback"/></noscript>`,
|
||||||
}
|
}
|
||||||
testEntry := &model.Entry{
|
testEntry := &model.Entry{
|
||||||
Title: `A title`,
|
Title: `A title`,
|
||||||
|
|
|
@ -3,7 +3,10 @@
|
||||||
|
|
||||||
package rewrite // import "miniflux.app/v2/internal/reader/rewrite"
|
package rewrite // import "miniflux.app/v2/internal/reader/rewrite"
|
||||||
|
|
||||||
import "regexp"
|
import (
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
// List of predefined rewrite rules (alphabetically sorted)
|
// List of predefined rewrite rules (alphabetically sorted)
|
||||||
// Available rules: "add_image_title", "add_youtube_video"
|
// Available rules: "add_image_title", "add_youtube_video"
|
||||||
|
@ -39,49 +42,40 @@ var predefinedRules = map[string]string{
|
||||||
"youtube.com": "add_youtube_video",
|
"youtube.com": "add_youtube_video",
|
||||||
}
|
}
|
||||||
|
|
||||||
type RefererRule struct {
|
|
||||||
URLPattern *regexp.Regexp
|
|
||||||
Referer string
|
|
||||||
}
|
|
||||||
|
|
||||||
// List of predefined referer rules
|
|
||||||
var PredefinedRefererRules = []RefererRule{
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://\w+\.sinaimg\.cn`),
|
|
||||||
Referer: "https://weibo.com",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://i\.pximg\.net`),
|
|
||||||
Referer: "https://www.pixiv.net",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://cdnfile\.sspai\.com`),
|
|
||||||
Referer: "https://sspai.com",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://(?:\w|-)+\.cdninstagram\.com`),
|
|
||||||
Referer: "https://www.instagram.com",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://sp1\.piokok\.com`),
|
|
||||||
Referer: "https://sp1.piokok.com",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://f\.video\.weibocdn\.com`),
|
|
||||||
Referer: "https://weibo.com",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
URLPattern: regexp.MustCompile(`^https://img\.hellogithub\.com`),
|
|
||||||
Referer: "https://hellogithub.com",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetRefererForURL returns the referer for the given URL if it exists, otherwise an empty string.
|
// GetRefererForURL returns the referer for the given URL if it exists, otherwise an empty string.
|
||||||
func GetRefererForURL(url string) string {
|
func GetRefererForURL(u string) string {
|
||||||
for _, rule := range PredefinedRefererRules {
|
parsedUrl, err := url.Parse(u)
|
||||||
if rule.URLPattern.MatchString(url) {
|
if err != nil {
|
||||||
return rule.Referer
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
switch parsedUrl.Hostname() {
|
||||||
|
case "moyu.im":
|
||||||
|
return "https://i.jandan.net"
|
||||||
|
case "i.pximg.net":
|
||||||
|
return "https://www.pixiv.net"
|
||||||
|
case "sp1.piokok.com":
|
||||||
|
return "https://sp1.piokok.com"
|
||||||
|
case "cdnfile.sspai.com":
|
||||||
|
return "https://sspai.com"
|
||||||
|
case "f.video.weibocdn.com":
|
||||||
|
return "https://weibo.com"
|
||||||
|
case "img.hellogithub.com":
|
||||||
|
return "https://hellogithub.com"
|
||||||
|
case "bjp.org.cn":
|
||||||
|
return "https://bjp.org.cn"
|
||||||
|
case "appinn.com":
|
||||||
|
return "https://appinn.com"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(parsedUrl.Hostname(), ".sinaimg.cn"):
|
||||||
|
return "https://weibo.com"
|
||||||
|
case strings.HasSuffix(parsedUrl.Hostname(), ".cdninstagram.com"):
|
||||||
|
return "https://www.instagram.com"
|
||||||
|
case strings.HasSuffix(parsedUrl.Hostname(), ".moyu.im"):
|
||||||
|
return "https://i.jandan.net"
|
||||||
|
}
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,7 @@ package sanitizer // import "miniflux.app/v2/internal/reader/sanitizer"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
"regexp"
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -18,7 +18,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
youtubeEmbedRegex = regexp.MustCompile(`^(?:https?:)?//(?:www\.)?youtube\.com/embed/(.+)$`)
|
|
||||||
tagAllowList = map[string][]string{
|
tagAllowList = map[string][]string{
|
||||||
"a": {"href", "title", "id"},
|
"a": {"href", "title", "id"},
|
||||||
"abbr": {"title"},
|
"abbr": {"title"},
|
||||||
|
@ -397,9 +396,27 @@ func isValidIframeSource(baseURL, src string) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func rewriteIframeURL(link string) string {
|
func rewriteIframeURL(link string) string {
|
||||||
matches := youtubeEmbedRegex.FindStringSubmatch(link)
|
u, err := url.Parse(link)
|
||||||
if len(matches) == 2 {
|
if err != nil {
|
||||||
return config.Opts.YouTubeEmbedUrlOverride() + matches[1]
|
return link
|
||||||
|
}
|
||||||
|
|
||||||
|
switch strings.TrimPrefix(u.Hostname(), "www.") {
|
||||||
|
case "youtube.com":
|
||||||
|
if strings.HasPrefix(u.Path, "/embed/") {
|
||||||
|
if len(u.RawQuery) > 0 {
|
||||||
|
return config.Opts.YouTubeEmbedUrlOverride() + strings.TrimPrefix(u.Path, "/embed/") + "?" + u.RawQuery
|
||||||
|
}
|
||||||
|
return config.Opts.YouTubeEmbedUrlOverride() + strings.TrimPrefix(u.Path, "/embed/")
|
||||||
|
}
|
||||||
|
case "player.vimeo.com":
|
||||||
|
// See https://help.vimeo.com/hc/en-us/articles/12426260232977-About-Player-parameters
|
||||||
|
if strings.HasPrefix(u.Path, "/video/") {
|
||||||
|
if len(u.RawQuery) > 0 {
|
||||||
|
return link + "&dnt=1"
|
||||||
|
}
|
||||||
|
return link + "?dnt=1"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return link
|
return link
|
||||||
|
|
|
@ -611,9 +611,9 @@ func TestReplaceYoutubeURLWithCustomURL(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReplaceIframeURL(t *testing.T) {
|
func TestReplaceIframeVimedoDNTURL(t *testing.T) {
|
||||||
input := `<iframe src="https://player.vimeo.com/video/123456?title=0&byline=0"></iframe>`
|
input := `<iframe src="https://player.vimeo.com/video/123456?title=0&byline=0"></iframe>`
|
||||||
expected := `<iframe src="https://player.vimeo.com/video/123456?title=0&byline=0" sandbox="allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox" loading="lazy"></iframe>`
|
expected := `<iframe src="https://player.vimeo.com/video/123456?title=0&byline=0&dnt=1" sandbox="allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox" loading="lazy"></iframe>`
|
||||||
output := Sanitize("http://example.org/", input)
|
output := Sanitize("http://example.org/", input)
|
||||||
|
|
||||||
if expected != output {
|
if expected != output {
|
||||||
|
|
|
@ -75,7 +75,7 @@ func findContentUsingCustomRules(page io.Reader, rules string) (baseURL string,
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if hrefValue, exists := document.Find("head base").First().Attr("href"); exists {
|
if hrefValue, exists := document.FindMatcher(goquery.Single("head base")).Attr("href"); exists {
|
||||||
hrefValue = strings.TrimSpace(hrefValue)
|
hrefValue = strings.TrimSpace(hrefValue)
|
||||||
if urllib.IsAbsoluteURL(hrefValue) {
|
if urllib.IsAbsoluteURL(hrefValue) {
|
||||||
baseURL = hrefValue
|
baseURL = hrefValue
|
||||||
|
|
|
@ -26,7 +26,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
youtubeHostRegex = regexp.MustCompile(`youtube\.com$`)
|
|
||||||
youtubeChannelRegex = regexp.MustCompile(`channel/(.*)$`)
|
youtubeChannelRegex = regexp.MustCompile(`channel/(.*)$`)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -156,7 +155,7 @@ func (f *SubscriptionFinder) FindSubscriptionsFromWebPage(websiteURL, contentTyp
|
||||||
return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_parse_html_document", err)
|
return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_parse_html_document", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hrefValue, exists := doc.Find("head base").First().Attr("href"); exists {
|
if hrefValue, exists := doc.FindMatcher(goquery.Single("head base")).Attr("href"); exists {
|
||||||
hrefValue = strings.TrimSpace(hrefValue)
|
hrefValue = strings.TrimSpace(hrefValue)
|
||||||
if urllib.IsAbsoluteURL(hrefValue) {
|
if urllib.IsAbsoluteURL(hrefValue) {
|
||||||
websiteURL = hrefValue
|
websiteURL = hrefValue
|
||||||
|
@ -295,7 +294,7 @@ func (f *SubscriptionFinder) FindSubscriptionsFromYouTubeChannelPage(websiteURL
|
||||||
return nil, locale.NewLocalizedErrorWrapper(err, "error.invalid_site_url", err)
|
return nil, locale.NewLocalizedErrorWrapper(err, "error.invalid_site_url", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !youtubeHostRegex.MatchString(decodedUrl.Host) {
|
if !strings.HasSuffix(decodedUrl.Host, "youtube.com") {
|
||||||
slog.Debug("This website is not a YouTube page, the regex doesn't match", slog.String("website_url", websiteURL))
|
slog.Debug("This website is not a YouTube page, the regex doesn't match", slog.String("website_url", websiteURL))
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
@ -314,7 +313,7 @@ func (f *SubscriptionFinder) FindSubscriptionsFromYouTubePlaylistPage(websiteURL
|
||||||
return nil, locale.NewLocalizedErrorWrapper(err, "error.invalid_site_url", err)
|
return nil, locale.NewLocalizedErrorWrapper(err, "error.invalid_site_url", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !youtubeHostRegex.MatchString(decodedUrl.Host) {
|
if !strings.HasSuffix(decodedUrl.Host, "youtube.com") {
|
||||||
slog.Debug("This website is not a YouTube page, the regex doesn't match", slog.String("website_url", websiteURL))
|
slog.Debug("This website is not a YouTube page, the regex doesn't match", slog.String("website_url", websiteURL))
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -71,7 +71,7 @@ func (h *handler) showSettingsPage(w http.ResponseWriter, r *http.Request) {
|
||||||
"MarkAsReadOnlyOnPlayerCompletion": form.MarkAsReadOnlyOnPlayerCompletion,
|
"MarkAsReadOnlyOnPlayerCompletion": form.MarkAsReadOnlyOnPlayerCompletion,
|
||||||
})
|
})
|
||||||
view.Set("themes", model.Themes())
|
view.Set("themes", model.Themes())
|
||||||
view.Set("languages", locale.AvailableLanguages())
|
view.Set("languages", locale.AvailableLanguages)
|
||||||
view.Set("timezones", timezones)
|
view.Set("timezones", timezones)
|
||||||
view.Set("menu", "settings")
|
view.Set("menu", "settings")
|
||||||
view.Set("user", user)
|
view.Set("user", user)
|
||||||
|
|
|
@ -44,7 +44,7 @@ func (h *handler) updateSettings(w http.ResponseWriter, r *http.Request) {
|
||||||
view := view.New(h.tpl, r, sess)
|
view := view.New(h.tpl, r, sess)
|
||||||
view.Set("form", settingsForm)
|
view.Set("form", settingsForm)
|
||||||
view.Set("themes", model.Themes())
|
view.Set("themes", model.Themes())
|
||||||
view.Set("languages", locale.AvailableLanguages())
|
view.Set("languages", locale.AvailableLanguages)
|
||||||
view.Set("timezones", timezones)
|
view.Set("timezones", timezones)
|
||||||
view.Set("menu", "settings")
|
view.Set("menu", "settings")
|
||||||
view.Set("user", loggedUser)
|
view.Set("user", loggedUser)
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
|
|
||||||
html {
|
html {
|
||||||
-webkit-text-size-adjust: 100%;
|
-webkit-text-size-adjust: 100%;
|
||||||
-ms-text-size-adjust: 100%;
|
text-size-adjust: 100%;
|
||||||
}
|
}
|
||||||
|
|
||||||
body {
|
body {
|
||||||
|
@ -53,7 +53,6 @@ a:hover {
|
||||||
.sr-only {
|
.sr-only {
|
||||||
border: 0 !important;
|
border: 0 !important;
|
||||||
clip: rect(1px, 1px, 1px, 1px) !important;
|
clip: rect(1px, 1px, 1px, 1px) !important;
|
||||||
-webkit-clip-path: inset(50%) !important;
|
|
||||||
clip-path: inset(50%) !important;
|
clip-path: inset(50%) !important;
|
||||||
height: 1px !important;
|
height: 1px !important;
|
||||||
overflow: hidden !important;
|
overflow: hidden !important;
|
||||||
|
@ -749,7 +748,7 @@ template {
|
||||||
padding-left: 15px;
|
padding-left: 15px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.pagination-next {
|
.pagination-next, .pagination-last {
|
||||||
text-align: right;
|
text-align: right;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -757,10 +756,6 @@ template {
|
||||||
content: " ›";
|
content: " ›";
|
||||||
}
|
}
|
||||||
|
|
||||||
.pagination-last {
|
|
||||||
text-align: right;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pagination-last:after {
|
.pagination-last:after {
|
||||||
content: " »";
|
content: " »";
|
||||||
}
|
}
|
||||||
|
|
|
@ -751,11 +751,10 @@ function checkShareAPI(title, url) {
|
||||||
title: title,
|
title: title,
|
||||||
url: url
|
url: url
|
||||||
});
|
});
|
||||||
window.location.reload();
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
window.location.reload();
|
|
||||||
}
|
}
|
||||||
|
window.location.reload();
|
||||||
}
|
}
|
||||||
|
|
||||||
function getCsrfToken() {
|
function getCsrfToken() {
|
||||||
|
|
|
@ -155,7 +155,7 @@ func validateTheme(theme string) *locale.LocalizedError {
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateLanguage(language string) *locale.LocalizedError {
|
func validateLanguage(language string) *locale.LocalizedError {
|
||||||
languages := locale.AvailableLanguages()
|
languages := locale.AvailableLanguages
|
||||||
if _, found := languages[language]; !found {
|
if _, found := languages[language]; !found {
|
||||||
return locale.NewLocalizedError("error.invalid_language")
|
return locale.NewLocalizedError("error.invalid_language")
|
||||||
}
|
}
|
||||||
|
@ -219,7 +219,7 @@ func validateMediaPlaybackRate(mediaPlaybackRate float64) *locale.LocalizedError
|
||||||
|
|
||||||
func isValidFilterRules(filterEntryRules string, filterType string) *locale.LocalizedError {
|
func isValidFilterRules(filterEntryRules string, filterType string) *locale.LocalizedError {
|
||||||
// Valid Format: FieldName=RegEx\nFieldName=RegEx...
|
// Valid Format: FieldName=RegEx\nFieldName=RegEx...
|
||||||
fieldNames := []string{"EntryTitle", "EntryURL", "EntryCommentsURL", "EntryContent", "EntryAuthor", "EntryTag"}
|
fieldNames := []string{"EntryTitle", "EntryURL", "EntryCommentsURL", "EntryContent", "EntryAuthor", "EntryTag", "EntryDate"}
|
||||||
|
|
||||||
rules := strings.Split(filterEntryRules, "\n")
|
rules := strings.Split(filterEntryRules, "\n")
|
||||||
for i, rule := range rules {
|
for i, rule := range rules {
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
.\" Manpage for miniflux.
|
.\" Manpage for miniflux.
|
||||||
.TH "MINIFLUX" "1" "October 26, 2024" "\ \&" "\ \&"
|
.TH "MINIFLUX" "1" "December 7, 2024" "\ \&" "\ \&"
|
||||||
|
|
||||||
.SH NAME
|
.SH NAME
|
||||||
miniflux \- Minimalist and opinionated feed reader
|
miniflux \- Minimalist and opinionated feed reader
|
||||||
|
|
|
@ -4,7 +4,7 @@ ADD . /go/src/app
|
||||||
WORKDIR /go/src/app
|
WORKDIR /go/src/app
|
||||||
RUN make miniflux
|
RUN make miniflux
|
||||||
|
|
||||||
FROM docker.io/library/alpine:3.20
|
FROM docker.io/library/alpine:3.21
|
||||||
|
|
||||||
LABEL org.opencontainers.image.title=Miniflux
|
LABEL org.opencontainers.image.title=Miniflux
|
||||||
LABEL org.opencontainers.image.description="Miniflux is a minimalist and opinionated feed reader"
|
LABEL org.opencontainers.image.description="Miniflux is a minimalist and opinionated feed reader"
|
||||||
|
|
|
@ -5,7 +5,7 @@ WORKDIR /go/src/app
|
||||||
RUN make miniflux
|
RUN make miniflux
|
||||||
|
|
||||||
FROM rockylinux:9
|
FROM rockylinux:9
|
||||||
RUN dnf install -y rpm-build systemd
|
RUN dnf install --setopt=install_weak_deps=False -y rpm-build systemd-rpm-macros
|
||||||
RUN mkdir -p /root/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
RUN mkdir -p /root/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
||||||
RUN echo "%_topdir /root/rpmbuild" >> .rpmmacros
|
RUN echo "%_topdir /root/rpmbuild" >> .rpmmacros
|
||||||
COPY --from=build /go/src/app/miniflux /root/rpmbuild/SOURCES/miniflux
|
COPY --from=build /go/src/app/miniflux /root/rpmbuild/SOURCES/miniflux
|
||||||
|
|
|
@ -16,8 +16,7 @@ BuildRoot: %{_topdir}/BUILD/%{name}-%{version}-%{release}
|
||||||
BuildArch: x86_64
|
BuildArch: x86_64
|
||||||
Requires(pre): shadow-utils
|
Requires(pre): shadow-utils
|
||||||
|
|
||||||
%{?systemd_requires}
|
%{?systemd_ordering}
|
||||||
BuildRequires: systemd
|
|
||||||
|
|
||||||
AutoReqProv: no
|
AutoReqProv: no
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue