mirror of
https://github.com/miniflux/v2.git
synced 2025-08-01 17:38:37 +00:00
ef13756b1a
Before ```console $ go test -bench=. goos: linux goarch: arm64 pkg: miniflux.app/v2/internal/reader/readability BenchmarkExtractContent-8 34 86102474 ns/op BenchmarkGetWeight-8 10573 103045 ns/op PASS ok miniflux.app/v2/internal/reader/readability 5.409s ``` After ```console $ go test -bench=. goos: linux goarch: arm64 pkg: miniflux.app/v2/internal/reader/readability BenchmarkExtractContent-8 56 83130924 ns/op BenchmarkGetWeight-8 246541 5241 ns/op PASS ok miniflux.app/v2/internal/reader/readability 6.026s ``` This should make ProcessFeedEntries marginally faster, while saving an also marginal amount of memory.
222 lines
4.9 KiB
Go
222 lines
4.9 KiB
Go
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package readability // import "miniflux.app/v2/internal/reader/readability"
|
|
|
|
import (
|
|
"bytes"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestBaseURL(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<base href="https://example.org/ ">
|
|
</head>
|
|
<body>
|
|
<article>
|
|
Some content
|
|
</article>
|
|
</body>
|
|
</html>`
|
|
|
|
baseURL, _, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if baseURL != "https://example.org/" {
|
|
t.Errorf(`Unexpected base URL, got %q instead of "https://example.org/"`, baseURL)
|
|
}
|
|
}
|
|
|
|
func TestMultipleBaseURL(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<base href="https://example.org/ ">
|
|
<base href="https://example.com/ ">
|
|
</head>
|
|
<body>
|
|
<article>
|
|
Some content
|
|
</article>
|
|
</body>
|
|
</html>`
|
|
|
|
baseURL, _, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if baseURL != "https://example.org/" {
|
|
t.Errorf(`Unexpected base URL, got %q instead of "https://example.org/"`, baseURL)
|
|
}
|
|
}
|
|
|
|
func TestRelativeBaseURL(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<base href="/test/ ">
|
|
</head>
|
|
<body>
|
|
<article>
|
|
Some content
|
|
</article>
|
|
</body>
|
|
</html>`
|
|
|
|
baseURL, _, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if baseURL != "" {
|
|
t.Errorf(`Unexpected base URL, got %q`, baseURL)
|
|
}
|
|
}
|
|
|
|
func TestWithoutBaseURL(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<title>Test</title>
|
|
</head>
|
|
<body>
|
|
<article>
|
|
Some content
|
|
</article>
|
|
</body>
|
|
</html>`
|
|
|
|
baseURL, _, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if baseURL != "" {
|
|
t.Errorf(`Unexpected base URL, got %q instead of ""`, baseURL)
|
|
}
|
|
}
|
|
|
|
func TestRemoveStyleScript(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<title>Test</title>
|
|
<script src="tololo.js"></script>
|
|
</head>
|
|
<body>
|
|
<script src="tololo.js"></script>
|
|
<style>
|
|
h1 {color:red;}
|
|
p {color:blue;}
|
|
</style>
|
|
<article>Some content</article>
|
|
</body>
|
|
</html>`
|
|
want := `<div><div><article>Somecontent</article></div></div>`
|
|
|
|
_, content, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
content = strings.ReplaceAll(content, "\n", "")
|
|
content = strings.ReplaceAll(content, " ", "")
|
|
content = strings.ReplaceAll(content, "\t", "")
|
|
|
|
if content != want {
|
|
t.Errorf(`Invalid content, got %s instead of %s`, content, want)
|
|
}
|
|
}
|
|
|
|
func TestRemoveBlacklist(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<title>Test</title>
|
|
</head>
|
|
<body>
|
|
<article class="super-ad">Some content</article>
|
|
<article class="g-plus-crap">Some other thing</article>
|
|
<article class="stuff popupbody">And more</article>
|
|
<article class="legit">Valid!</article>
|
|
</body>
|
|
</html>`
|
|
want := `<div><div><articleclass="legit">Valid!</article></div></div>`
|
|
|
|
_, content, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
content = strings.ReplaceAll(content, "\n", "")
|
|
content = strings.ReplaceAll(content, " ", "")
|
|
content = strings.ReplaceAll(content, "\t", "")
|
|
|
|
if content != want {
|
|
t.Errorf(`Invalid content, got %s instead of %s`, content, want)
|
|
}
|
|
}
|
|
|
|
func TestNestedSpanInCodeBlock(t *testing.T) {
|
|
html := `
|
|
<html>
|
|
<head>
|
|
<title>Test</title>
|
|
</head>
|
|
<body>
|
|
<article><p>Some content</p><pre><code class="hljs-built_in">Code block with <span class="hljs-built_in">nested span</span> <span class="hljs-comment"># exit 1</span></code></pre></article>
|
|
</body>
|
|
</html>`
|
|
want := `<div><div><p>Some content</p><pre><code class="hljs-built_in">Code block with <span class="hljs-built_in">nested span</span> <span class="hljs-comment"># exit 1</span></code></pre></div></div>`
|
|
|
|
_, result, err := ExtractContent(strings.NewReader(html))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if result != want {
|
|
t.Errorf(`Invalid content, got %s instead of %s`, result, want)
|
|
}
|
|
}
|
|
|
|
func BenchmarkExtractContent(b *testing.B) {
|
|
var testCases = map[string][]byte{
|
|
"miniflux_github.html": {},
|
|
"miniflux_wikipedia.html": {},
|
|
}
|
|
for filename := range testCases {
|
|
data, err := os.ReadFile("testdata/" + filename)
|
|
if err != nil {
|
|
b.Fatalf(`Unable to read file %q: %v`, filename, err)
|
|
}
|
|
testCases[filename] = data
|
|
}
|
|
for range b.N {
|
|
for _, v := range testCases {
|
|
ExtractContent(bytes.NewReader(v))
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkGetWeight(b *testing.B) {
|
|
testCases := []string{
|
|
"p-3 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content",
|
|
"d-flex flex-column mb-3",
|
|
"AppHeader-search-control AppHeader-search-control-overflow",
|
|
"Button Button--iconOnly Button--invisible Button--medium mr-1 px-2 py-0 d-flex flex-items-center rounded-1 color-fg-muted",
|
|
"sr-only",
|
|
"validation-12753bbc-b4d1-4e10-bec6-92e585d1699d",
|
|
}
|
|
for range b.N {
|
|
for _, v := range testCases {
|
|
getWeight(v)
|
|
}
|
|
}
|
|
}
|