1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-09-15 18:57:04 +00:00

Move internal packages to an internal folder

For reference: https://go.dev/doc/go1.4#internalpackages
This commit is contained in:
Frédéric Guillot 2023-08-10 19:46:45 -07:00
parent c234903255
commit 168a870c02
433 changed files with 1121 additions and 1123 deletions

View file

@ -0,0 +1,122 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package proxy // import "miniflux.app/v2/internal/proxy"
import (
"strings"
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/url"
"github.com/PuerkitoBio/goquery"
"github.com/gorilla/mux"
)
type urlProxyRewriter func(router *mux.Router, url string) string
// ProxyRewriter replaces media URLs with internal proxy URLs.
func ProxyRewriter(router *mux.Router, data string) string {
return genericProxyRewriter(router, ProxifyURL, data)
}
// AbsoluteProxyRewriter do the same as ProxyRewriter except it uses absolute URLs.
func AbsoluteProxyRewriter(router *mux.Router, host, data string) string {
proxifyFunction := func(router *mux.Router, url string) string {
return AbsoluteProxifyURL(router, host, url)
}
return genericProxyRewriter(router, proxifyFunction, data)
}
func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string {
proxyOption := config.Opts.ProxyOption()
if proxyOption == "none" {
return data
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
if err != nil {
return data
}
for _, mediaType := range config.Opts.ProxyMediaTypes() {
switch mediaType {
case "image":
doc.Find("img").Each(func(i int, img *goquery.Selection) {
if srcAttrValue, ok := img.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
img.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
if srcsetAttrValue, ok := img.Attr("srcset"); ok {
proxifySourceSet(img, router, proxifyFunction, proxyOption, srcsetAttrValue)
}
})
doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) {
if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok {
proxifySourceSet(sourceElement, router, proxifyFunction, proxyOption, srcsetAttrValue)
}
})
case "audio":
doc.Find("audio").Each(func(i int, audio *goquery.Selection) {
if srcAttrValue, ok := audio.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
audio.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
doc.Find("audio source").Each(func(i int, sourceElement *goquery.Selection) {
if srcAttrValue, ok := sourceElement.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
sourceElement.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
case "video":
doc.Find("video").Each(func(i int, video *goquery.Selection) {
if srcAttrValue, ok := video.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
video.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
doc.Find("video source").Each(func(i int, sourceElement *goquery.Selection) {
if srcAttrValue, ok := sourceElement.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
sourceElement.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
}
}
output, err := doc.Find("body").First().Html()
if err != nil {
return data
}
return output
}
func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyOption, srcsetAttrValue string) {
imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue)
for _, imageCandidate := range imageCandidates {
if !isDataURL(imageCandidate.ImageURL) && (proxyOption == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) {
imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL)
}
}
element.SetAttr("srcset", imageCandidates.String())
}
func isDataURL(s string) bool {
return strings.HasPrefix(s, "data:")
}

View file

@ -0,0 +1,379 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package proxy // import "miniflux.app/v2/internal/proxy"
import (
"net/http"
"os"
"testing"
"github.com/gorilla/mux"
"miniflux.app/v2/internal/config"
)
func TestProxyFilterWithHttpDefault(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "http-only")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpsDefault(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "http-only")
os.Setenv("PROXY_MEDIA_TYPES", "image")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpNever(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "none")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := input
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpsNever(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "none")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := input
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpAlways(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpsAlways(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/LdPNR1GBDigeeNp2ArUQRyZsVqT_PWLfHGjYFrrWWIY=/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_URL", "https://proxy-example/proxy")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="https://proxy-example/proxy/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpInvalid(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "invalid")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithHttpsInvalid(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "invalid")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestProxyFilterWithSrcset(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" srcset="http://website/folder/image2.png 656w, http://website/folder/image3.png 360w" alt="test"></p>`
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/QgAmrJWiAud_nNAsz3F8OTxaIofwAiO36EDzH_YfMzo=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w" alt="test"/></p>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestProxyFilterWithEmptySrcset(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" srcset="" alt="test"></p>`
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="" alt="test"/></p>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestProxyFilterWithPictureSource(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<picture><source srcset="http://website/folder/image2.png 656w, http://website/folder/image3.png 360w, https://website/some,image.png 2x"></picture>`
expected := `<picture><source srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/QgAmrJWiAud_nNAsz3F8OTxaIofwAiO36EDzH_YfMzo=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w, /proxy/ZIw0hv8WhSTls5aSqhnFaCXlUrKIqTnBRaY0-NaLnds=/aHR0cHM6Ly93ZWJzaXRlL3NvbWUsaW1hZ2UucG5n 2x"/></picture>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "https")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<picture><source srcset="http://website/folder/image2.png 656w, https://website/some,image.png 2x"></picture>`
expected := `<picture><source srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, https://website/some,image.png 2x"/></picture>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestProxyWithImageDataURL(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<img src="data:image/gif;base64,test">`
expected := `<img src="data:image/gif;base64,test"/>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestProxyWithImageSourceDataURL(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<picture><source srcset="data:image/gif;base64,test"/></picture>`
expected := `<picture><source srcset="data:image/gif;base64,test"/></picture>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}

69
internal/proxy/proxy.go Normal file
View file

@ -0,0 +1,69 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package proxy // import "miniflux.app/v2/internal/proxy"
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"net/url"
"path"
"miniflux.app/v2/internal/http/route"
"github.com/gorilla/mux"
"miniflux.app/v2/internal/config"
)
// ProxifyURL generates a relative URL for a proxified resource.
func ProxifyURL(router *mux.Router, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write([]byte(link))
digest := mac.Sum(nil)
return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
}
proxyUrl, err := url.Parse(proxyImageUrl)
if err != nil {
return ""
}
proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link)))
return proxyUrl.String()
}
return ""
}
// AbsoluteProxifyURL generates an absolute URL for a proxified resource.
func AbsoluteProxifyURL(router *mux.Router, host, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write([]byte(link))
digest := mac.Sum(nil)
path := route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
if config.Opts.HTTPS {
return "https://" + host + path
} else {
return "http://" + host + path
}
}
proxyUrl, err := url.Parse(proxyImageUrl)
if err != nil {
return ""
}
proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link)))
return proxyUrl.String()
}
return ""
}