oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-08-01 17:38:37 +00:00
Code Wiki Activity

Make sure the scraper parse only HTML documents

Browse source
This commit is contained in:
Frédéric Guillot 2018-01-02 18:32:01 -08:00
parent aae3d75490
commit 3c3f397bf5
1 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
reader/scraper/scraper.go
View file

@ -6,6 +6,7 @@ package scraper
import ( import (
"errors" "errors"
"fmt"
"io" "io"
"strings" "strings"
@ -16,7 +17,7 @@ import (
"github.com/miniflux/miniflux/url" "github.com/miniflux/miniflux/url"
) )
// Fetch download a web page a returns relevant contents. // Fetch downloads a web page a returns relevant contents.
func Fetch(websiteURL, rules string) (string, error) { func Fetch(websiteURL, rules string) (string, error) {
client := http.NewClient(websiteURL) client := http.NewClient(websiteURL)
response, err := client.Get() response, err := client.Get()
@ -25,7 +26,11 @@ func Fetch(websiteURL, rules string) (string, error) {
} }
if response.HasServerFailure() { if response.HasServerFailure() {
return "", errors.New("unable to download web page") return "", errors.New("scraper: unable to download web page")
}
if !strings.Contains(response.ContentType, "text/html") {
return "", fmt.Errorf("scraper: this resource is not a HTML document (%s)", response.ContentType)
} }
page, err := response.NormalizeBodyEncoding() page, err := response.NormalizeBodyEncoding()
@ -33,7 +38,7 @@ func Fetch(websiteURL, rules string) (string, error) {
return "", err return "", err
} }
// The entry URL could be a redirect somewhere else. // The entry URL could redirect somewhere else.
websiteURL = response.EffectiveURL websiteURL = response.EffectiveURL
if rules == "" { if rules == "" {