mirror of
https://github.com/miniflux/v2.git
synced 2025-06-27 16:36:00 +00:00
feat: resize favicons before storing them
Some websites are using images of O(10kB) when not )O(100kB) for their favicons. As miniflux only displays them with a 16x16 resolution, let's do our best to resize them before storing them in the database. This should make miniflux consume less bandwidth when serving pages, for the joy of mobile users on a small data plan. Of course, images that already are 16x16 aren't resized.
This commit is contained in:
parent
cfda948c3a
commit
777d0dd248
4 changed files with 114 additions and 0 deletions
1
go.mod
1
go.mod
|
@ -13,6 +13,7 @@ require (
|
||||||
github.com/prometheus/client_golang v1.20.5
|
github.com/prometheus/client_golang v1.20.5
|
||||||
github.com/tdewolff/minify/v2 v2.21.2
|
github.com/tdewolff/minify/v2 v2.21.2
|
||||||
golang.org/x/crypto v0.31.0
|
golang.org/x/crypto v0.31.0
|
||||||
|
golang.org/x/image v0.23.0
|
||||||
golang.org/x/net v0.32.0
|
golang.org/x/net v0.32.0
|
||||||
golang.org/x/oauth2 v0.24.0
|
golang.org/x/oauth2 v0.24.0
|
||||||
golang.org/x/term v0.27.0
|
golang.org/x/term v0.27.0
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -70,6 +70,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
|
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
|
||||||
|
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
|
|
@ -4,12 +4,18 @@
|
||||||
package icon // import "miniflux.app/v2/internal/reader/icon"
|
package icon // import "miniflux.app/v2/internal/reader/icon"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"image"
|
||||||
|
"image/gif"
|
||||||
|
"image/jpeg"
|
||||||
|
"image/png"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
|
@ -19,6 +25,7 @@ import (
|
||||||
"miniflux.app/v2/internal/urllib"
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"golang.org/x/image/draw"
|
||||||
"golang.org/x/net/html/charset"
|
"golang.org/x/net/html/charset"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -180,9 +187,59 @@ func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) {
|
||||||
Content: responseBody,
|
Content: responseBody,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
icon = resizeIcon(icon)
|
||||||
|
|
||||||
return icon, nil
|
return icon, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resizeIcon(icon *model.Icon) *model.Icon {
|
||||||
|
r := bytes.NewReader(icon.Content)
|
||||||
|
|
||||||
|
if !slices.Contains([]string{"image/jpeg", "image/png", "image/gif"}, icon.MimeType) {
|
||||||
|
slog.Info("icon isn't a png/gif/jpeg/ico, can't resize", slog.String("mimetype", icon.MimeType))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
|
// Don't resize icons that we can't decode, or that already have the right size.
|
||||||
|
config, _, err := image.DecodeConfig(r)
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("unable to decode the metadata of the icon", slog.Any("error", err))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
if config.Height <= 16 && config.Width <= 16 {
|
||||||
|
slog.Debug("icon don't need to be rescaled", slog.Int("height", config.Height), slog.Int("width", config.Width))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
|
r.Seek(0, io.SeekStart)
|
||||||
|
|
||||||
|
var src image.Image
|
||||||
|
switch icon.MimeType {
|
||||||
|
case "image/jpeg":
|
||||||
|
src, err = jpeg.Decode(r)
|
||||||
|
case "image/png":
|
||||||
|
src, err = png.Decode(r)
|
||||||
|
case "image/gif":
|
||||||
|
src, err = gif.Decode(r)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("unable to decode the icon", slog.Any("error", err))
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
|
dst := image.NewRGBA(image.Rect(0, 0, 16, 16))
|
||||||
|
draw.BiLinear.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
|
||||||
|
|
||||||
|
var b bytes.Buffer
|
||||||
|
if err = png.Encode(io.Writer(&b), dst); err != nil {
|
||||||
|
slog.Warn("unable to encode the new icon", slog.Any("error", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
icon.Content = b.Bytes()
|
||||||
|
icon.MimeType = "image/png"
|
||||||
|
return icon
|
||||||
|
}
|
||||||
|
|
||||||
func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string, error) {
|
func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string, error) {
|
||||||
queries := []string{
|
queries := []string{
|
||||||
"link[rel='icon' i]",
|
"link[rel='icon' i]",
|
||||||
|
|
|
@ -4,8 +4,13 @@
|
||||||
package icon // import "miniflux.app/v2/internal/reader/icon"
|
package icon // import "miniflux.app/v2/internal/reader/icon"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"image"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseImageDataURL(t *testing.T) {
|
func TestParseImageDataURL(t *testing.T) {
|
||||||
|
@ -125,3 +130,52 @@ func TestParseDocumentWithWhitespaceIconURL(t *testing.T) {
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURLs[0])
|
t.Errorf(`Invalid icon URL, got %q`, iconURLs[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResizeIconSmallGif(t *testing.T) {
|
||||||
|
data, err := base64.StdEncoding.DecodeString("R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
icon := model.Icon{
|
||||||
|
Content: data,
|
||||||
|
MimeType: "image/gif",
|
||||||
|
}
|
||||||
|
if !bytes.Equal(icon.Content, resizeIcon(&icon).Content) {
|
||||||
|
t.Fatalf("Converted gif smaller than 16x16")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResizeIconPng(t *testing.T) {
|
||||||
|
data, err := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAHElEQVR42mP8z/C/noFCwDhqyKgho4aMGkIlQwBrHSpf28Yx+gAAAABJRU5ErkJggg==")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
icon := model.Icon{
|
||||||
|
Content: data,
|
||||||
|
MimeType: "image/png",
|
||||||
|
}
|
||||||
|
resizedIcon := resizeIcon(&icon)
|
||||||
|
|
||||||
|
if bytes.Equal(data, resizedIcon.Content) {
|
||||||
|
t.Fatalf("Didn't convert png of 17x17")
|
||||||
|
}
|
||||||
|
|
||||||
|
config, _, err := image.DecodeConfig(bytes.NewReader(resizedIcon.Content))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Couln't decode resulting png: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.Height != 16 || config.Width != 16 {
|
||||||
|
t.Fatalf("Was expecting an image of 16x16, got %dx%d", config.Width, config.Height)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResizeInvalidImage(t *testing.T) {
|
||||||
|
icon := model.Icon{
|
||||||
|
Content: []byte("invalid data"),
|
||||||
|
MimeType: "image/gif",
|
||||||
|
}
|
||||||
|
if !bytes.Equal(icon.Content, resizeIcon(&icon).Content) {
|
||||||
|
t.Fatalf("Tried to convert an invalid image")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue