mirror of
https://github.com/miniflux/v2.git
synced 2025-08-31 18:31:01 +00:00
Move packages http and url
This commit is contained in:
parent
2356ddad28
commit
6f5350a497
14 changed files with 11 additions and 11 deletions
|
@ -11,8 +11,8 @@ import (
|
|||
|
||||
"github.com/miniflux/miniflux2/errors"
|
||||
"github.com/miniflux/miniflux2/helper"
|
||||
"github.com/miniflux/miniflux2/http"
|
||||
"github.com/miniflux/miniflux2/model"
|
||||
"github.com/miniflux/miniflux2/reader/http"
|
||||
"github.com/miniflux/miniflux2/reader/icon"
|
||||
"github.com/miniflux/miniflux2/storage"
|
||||
)
|
||||
|
|
|
@ -1,115 +0,0 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/miniflux/miniflux2/helper"
|
||||
)
|
||||
|
||||
const userAgent = "Miniflux <https://miniflux.net/>"
|
||||
const requestTimeout = 300
|
||||
|
||||
// Client is a HTTP Client :)
|
||||
type Client struct {
|
||||
url string
|
||||
etagHeader string
|
||||
lastModifiedHeader string
|
||||
username string
|
||||
password string
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
// Get execute a GET HTTP request.
|
||||
func (c *Client) Get() (*Response, error) {
|
||||
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient:Get] url=%s", c.url))
|
||||
|
||||
client := c.buildClient()
|
||||
resp, err := client.Do(c.buildRequest())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response := &Response{
|
||||
Body: resp.Body,
|
||||
StatusCode: resp.StatusCode,
|
||||
EffectiveURL: resp.Request.URL.String(),
|
||||
LastModified: resp.Header.Get("Last-Modified"),
|
||||
ETag: resp.Header.Get("ETag"),
|
||||
ContentType: resp.Header.Get("Content-Type"),
|
||||
}
|
||||
|
||||
log.Println("[HttpClient:Get]",
|
||||
"OriginalURL:", c.url,
|
||||
"StatusCode:", response.StatusCode,
|
||||
"ETag:", response.ETag,
|
||||
"LastModified:", response.LastModified,
|
||||
"EffectiveURL:", response.EffectiveURL,
|
||||
)
|
||||
|
||||
return response, err
|
||||
}
|
||||
|
||||
func (c *Client) buildRequest() *http.Request {
|
||||
link, _ := url.Parse(c.url)
|
||||
request := &http.Request{
|
||||
URL: link,
|
||||
Method: http.MethodGet,
|
||||
Header: c.buildHeaders(),
|
||||
}
|
||||
|
||||
if c.username != "" && c.password != "" {
|
||||
request.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
|
||||
return request
|
||||
}
|
||||
|
||||
func (c *Client) buildClient() http.Client {
|
||||
client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)}
|
||||
if c.Insecure {
|
||||
client.Transport = &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
}
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func (c *Client) buildHeaders() http.Header {
|
||||
headers := make(http.Header)
|
||||
headers.Add("User-Agent", userAgent)
|
||||
|
||||
if c.etagHeader != "" {
|
||||
headers.Add("If-None-Match", c.etagHeader)
|
||||
}
|
||||
|
||||
if c.lastModifiedHeader != "" {
|
||||
headers.Add("If-Modified-Since", c.lastModifiedHeader)
|
||||
}
|
||||
|
||||
return headers
|
||||
}
|
||||
|
||||
// NewClient returns a new HTTP client.
|
||||
func NewClient(url string) *Client {
|
||||
return &Client{url: url, Insecure: false}
|
||||
}
|
||||
|
||||
// NewClientWithCredentials returns a new HTTP client that require authentication.
|
||||
func NewClientWithCredentials(url, username, password string) *Client {
|
||||
return &Client{url: url, Insecure: false, username: username, password: password}
|
||||
}
|
||||
|
||||
// NewClientWithCacheHeaders returns a new HTTP client that send cache headers.
|
||||
func NewClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *Client {
|
||||
return &Client{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
|
||||
}
|
|
@ -1,52 +0,0 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html/charset"
|
||||
)
|
||||
|
||||
// Response wraps a server response.
|
||||
type Response struct {
|
||||
Body io.Reader
|
||||
StatusCode int
|
||||
EffectiveURL string
|
||||
LastModified string
|
||||
ETag string
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// HasServerFailure returns true if the status code represents a failure.
|
||||
func (r *Response) HasServerFailure() bool {
|
||||
return r.StatusCode >= 400
|
||||
}
|
||||
|
||||
// IsModified returns true if the resource has been modified.
|
||||
func (r *Response) IsModified(etag, lastModified string) bool {
|
||||
if r.StatusCode == 304 {
|
||||
return false
|
||||
}
|
||||
|
||||
if r.ETag != "" && r.ETag == etag {
|
||||
return false
|
||||
}
|
||||
|
||||
if r.LastModified != "" && r.LastModified == lastModified {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// NormalizeBodyEncoding make sure the body is encoded in UTF-8.
|
||||
func (r *Response) NormalizeBodyEncoding() (io.Reader, error) {
|
||||
if strings.Contains(r.ContentType, "charset=") {
|
||||
return charset.NewReader(r.Body, r.ContentType)
|
||||
}
|
||||
return r.Body, nil
|
||||
}
|
|
@ -1,56 +0,0 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package http
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHasServerFailureWith200Status(t *testing.T) {
|
||||
r := &Response{StatusCode: 200}
|
||||
if r.HasServerFailure() {
|
||||
t.Error("200 is not a failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasServerFailureWith404Status(t *testing.T) {
|
||||
r := &Response{StatusCode: 404}
|
||||
if !r.HasServerFailure() {
|
||||
t.Error("404 is a failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasServerFailureWith500Status(t *testing.T) {
|
||||
r := &Response{StatusCode: 500}
|
||||
if !r.HasServerFailure() {
|
||||
t.Error("500 is a failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsModifiedWith304Status(t *testing.T) {
|
||||
r := &Response{StatusCode: 304}
|
||||
if r.IsModified("etag", "lastModified") {
|
||||
t.Error("The resource should not be considered modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsModifiedWithIdenticalEtag(t *testing.T) {
|
||||
r := &Response{StatusCode: 200, ETag: "etag"}
|
||||
if r.IsModified("etag", "lastModified") {
|
||||
t.Error("The resource should not be considered modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsModifiedWithIdenticalLastModified(t *testing.T) {
|
||||
r := &Response{StatusCode: 200, LastModified: "lastModified"}
|
||||
if r.IsModified("etag", "lastModified") {
|
||||
t.Error("The resource should not be considered modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsModifiedWithDifferentHeaders(t *testing.T) {
|
||||
r := &Response{StatusCode: 200, ETag: "some etag", LastModified: "some date"}
|
||||
if !r.IsModified("etag", "lastModified") {
|
||||
t.Error("The resource should be considered modified")
|
||||
}
|
||||
}
|
|
@ -11,9 +11,9 @@ import (
|
|||
"log"
|
||||
|
||||
"github.com/miniflux/miniflux2/helper"
|
||||
"github.com/miniflux/miniflux2/http"
|
||||
"github.com/miniflux/miniflux2/model"
|
||||
"github.com/miniflux/miniflux2/reader/http"
|
||||
"github.com/miniflux/miniflux2/reader/url"
|
||||
"github.com/miniflux/miniflux2/url"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
|
|
@ -10,7 +10,7 @@ import (
|
|||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/miniflux/miniflux2/reader/url"
|
||||
"github.com/miniflux/miniflux2/url"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
|
|
@ -13,9 +13,9 @@ import (
|
|||
|
||||
"github.com/miniflux/miniflux2/errors"
|
||||
"github.com/miniflux/miniflux2/helper"
|
||||
"github.com/miniflux/miniflux2/http"
|
||||
"github.com/miniflux/miniflux2/reader/feed"
|
||||
"github.com/miniflux/miniflux2/reader/http"
|
||||
"github.com/miniflux/miniflux2/reader/url"
|
||||
"github.com/miniflux/miniflux2/url"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
|
|
@ -1,71 +0,0 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package url
|
||||
|
||||
import "net/url"
|
||||
import "fmt"
|
||||
import "strings"
|
||||
|
||||
// AbsoluteURL converts the input URL as absolute URL if necessary.
|
||||
func AbsoluteURL(baseURL, input string) (string, error) {
|
||||
if strings.HasPrefix(input, "//") {
|
||||
input = "https://" + input[2:]
|
||||
}
|
||||
|
||||
u, err := url.Parse(input)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to parse input URL: %v", err)
|
||||
}
|
||||
|
||||
if u.IsAbs() {
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
base, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to parse base URL: %v", err)
|
||||
}
|
||||
|
||||
return base.ResolveReference(u).String(), nil
|
||||
}
|
||||
|
||||
// RootURL returns absolute URL without the path.
|
||||
func RootURL(websiteURL string) string {
|
||||
if strings.HasPrefix(websiteURL, "//") {
|
||||
websiteURL = "https://" + websiteURL[2:]
|
||||
}
|
||||
|
||||
absoluteURL, err := AbsoluteURL(websiteURL, "")
|
||||
if err != nil {
|
||||
return websiteURL
|
||||
}
|
||||
|
||||
u, err := url.Parse(absoluteURL)
|
||||
if err != nil {
|
||||
return absoluteURL
|
||||
}
|
||||
|
||||
return u.Scheme + "://" + u.Host + "/"
|
||||
}
|
||||
|
||||
// IsHTTPS returns true if the URL is using HTTPS.
|
||||
func IsHTTPS(websiteURL string) bool {
|
||||
parsedURL, err := url.Parse(websiteURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.ToLower(parsedURL.Scheme) == "https"
|
||||
}
|
||||
|
||||
// Domain returns only the domain part of the given URL.
|
||||
func Domain(websiteURL string) string {
|
||||
parsedURL, err := url.Parse(websiteURL)
|
||||
if err != nil {
|
||||
return websiteURL
|
||||
}
|
||||
|
||||
return parsedURL.Host
|
||||
}
|
|
@ -1,117 +0,0 @@
|
|||
package url
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGetAbsoluteURLWithAbsolutePath(t *testing.T) {
|
||||
expected := `https://example.org/path/file.ext`
|
||||
input := `/path/file.ext`
|
||||
output, err := AbsoluteURL("https://example.org/folder/", input)
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAbsoluteURLWithRelativePath(t *testing.T) {
|
||||
expected := `https://example.org/folder/path/file.ext`
|
||||
input := `path/file.ext`
|
||||
output, err := AbsoluteURL("https://example.org/folder/", input)
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAbsoluteURLWithRelativePaths(t *testing.T) {
|
||||
expected := `https://example.org/path/file.ext`
|
||||
input := `path/file.ext`
|
||||
output, err := AbsoluteURL("https://example.org/folder", input)
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhenInputIsAlreadyAbsolute(t *testing.T) {
|
||||
expected := `https://example.org/path/file.ext`
|
||||
input := `https://example.org/path/file.ext`
|
||||
output, err := AbsoluteURL("https://example.org/folder/", input)
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAbsoluteURLWithProtocolRelative(t *testing.T) {
|
||||
expected := `https://static.example.org/path/file.ext`
|
||||
input := `//static.example.org/path/file.ext`
|
||||
output, err := AbsoluteURL("https://www.example.org/", input)
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRootURL(t *testing.T) {
|
||||
expected := `https://example.org/`
|
||||
input := `https://example.org/path/file.ext`
|
||||
output := RootURL(input)
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRootURLWithProtocolRelativePath(t *testing.T) {
|
||||
expected := `https://static.example.org/`
|
||||
input := `//static.example.org/path/file.ext`
|
||||
output := RootURL(input)
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsHTTPS(t *testing.T) {
|
||||
if !IsHTTPS("https://example.org/") {
|
||||
t.Error("Unable to recognize HTTPS URL")
|
||||
}
|
||||
|
||||
if IsHTTPS("http://example.org/") {
|
||||
t.Error("Unable to recognize HTTP URL")
|
||||
}
|
||||
|
||||
if IsHTTPS("") {
|
||||
t.Error("Unable to recognize malformed URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDomain(t *testing.T) {
|
||||
expected := `static.example.org`
|
||||
input := `http://static.example.org/`
|
||||
output := Domain(input)
|
||||
|
||||
if expected != output {
|
||||
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue