Move packages http and url

http/client.go

@@ -0,0 +1,115 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/miniflux/miniflux2/helper"
+)
+
+const userAgent = "Miniflux <https://miniflux.net/>"
+const requestTimeout = 300
+
+// Client is a HTTP Client :)
+type Client struct {
+	url                string
+	etagHeader         string
+	lastModifiedHeader string
+	username           string
+	password           string
+	Insecure           bool
+}
+
+// Get execute a GET HTTP request.
+func (c *Client) Get() (*Response, error) {
+	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient:Get] url=%s", c.url))
+
+	client := c.buildClient()
+	resp, err := client.Do(c.buildRequest())
+	if err != nil {
+		return nil, err
+	}
+
+	response := &Response{
+		Body:         resp.Body,
+		StatusCode:   resp.StatusCode,
+		EffectiveURL: resp.Request.URL.String(),
+		LastModified: resp.Header.Get("Last-Modified"),
+		ETag:         resp.Header.Get("ETag"),
+		ContentType:  resp.Header.Get("Content-Type"),
+	}
+
+	log.Println("[HttpClient:Get]",
+		"OriginalURL:", c.url,
+		"StatusCode:", response.StatusCode,
+		"ETag:", response.ETag,
+		"LastModified:", response.LastModified,
+		"EffectiveURL:", response.EffectiveURL,
+	)
+
+	return response, err
+}
+
+func (c *Client) buildRequest() *http.Request {
+	link, _ := url.Parse(c.url)
+	request := &http.Request{
+		URL:    link,
+		Method: http.MethodGet,
+		Header: c.buildHeaders(),
+	}
+
+	if c.username != "" && c.password != "" {
+		request.SetBasicAuth(c.username, c.password)
+	}
+
+	return request
+}
+
+func (c *Client) buildClient() http.Client {
+	client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)}
+	if c.Insecure {
+		client.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+	}
+
+	return client
+}
+
+func (c *Client) buildHeaders() http.Header {
+	headers := make(http.Header)
+	headers.Add("User-Agent", userAgent)
+
+	if c.etagHeader != "" {
+		headers.Add("If-None-Match", c.etagHeader)
+	}
+
+	if c.lastModifiedHeader != "" {
+		headers.Add("If-Modified-Since", c.lastModifiedHeader)
+	}
+
+	return headers
+}
+
+// NewClient returns a new HTTP client.
+func NewClient(url string) *Client {
+	return &Client{url: url, Insecure: false}
+}
+
+// NewClientWithCredentials returns a new HTTP client that require authentication.
+func NewClientWithCredentials(url, username, password string) *Client {
+	return &Client{url: url, Insecure: false, username: username, password: password}
+}
+
+// NewClientWithCacheHeaders returns a new HTTP client that send cache headers.
+func NewClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *Client {
+	return &Client{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
+}

http/response.go

@@ -0,0 +1,52 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+	"io"
+	"strings"
+
+	"golang.org/x/net/html/charset"
+)
+
+// Response wraps a server response.
+type Response struct {
+	Body         io.Reader
+	StatusCode   int
+	EffectiveURL string
+	LastModified string
+	ETag         string
+	ContentType  string
+}
+
+// HasServerFailure returns true if the status code represents a failure.
+func (r *Response) HasServerFailure() bool {
+	return r.StatusCode >= 400
+}
+
+// IsModified returns true if the resource has been modified.
+func (r *Response) IsModified(etag, lastModified string) bool {
+	if r.StatusCode == 304 {
+		return false
+	}
+
+	if r.ETag != "" && r.ETag == etag {
+		return false
+	}
+
+	if r.LastModified != "" && r.LastModified == lastModified {
+		return false
+	}
+
+	return true
+}
+
+// NormalizeBodyEncoding make sure the body is encoded in UTF-8.
+func (r *Response) NormalizeBodyEncoding() (io.Reader, error) {
+	if strings.Contains(r.ContentType, "charset=") {
+		return charset.NewReader(r.Body, r.ContentType)
+	}
+	return r.Body, nil
+}

http/response_test.go

@@ -0,0 +1,56 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package http
+
+import "testing"
+
+func TestHasServerFailureWith200Status(t *testing.T) {
+	r := &Response{StatusCode: 200}
+	if r.HasServerFailure() {
+		t.Error("200 is not a failure")
+	}
+}
+
+func TestHasServerFailureWith404Status(t *testing.T) {
+	r := &Response{StatusCode: 404}
+	if !r.HasServerFailure() {
+		t.Error("404 is a failure")
+	}
+}
+
+func TestHasServerFailureWith500Status(t *testing.T) {
+	r := &Response{StatusCode: 500}
+	if !r.HasServerFailure() {
+		t.Error("500 is a failure")
+	}
+}
+
+func TestIsModifiedWith304Status(t *testing.T) {
+	r := &Response{StatusCode: 304}
+	if r.IsModified("etag", "lastModified") {
+		t.Error("The resource should not be considered modified")
+	}
+}
+
+func TestIsModifiedWithIdenticalEtag(t *testing.T) {
+	r := &Response{StatusCode: 200, ETag: "etag"}
+	if r.IsModified("etag", "lastModified") {
+		t.Error("The resource should not be considered modified")
+	}
+}
+
+func TestIsModifiedWithIdenticalLastModified(t *testing.T) {
+	r := &Response{StatusCode: 200, LastModified: "lastModified"}
+	if r.IsModified("etag", "lastModified") {
+		t.Error("The resource should not be considered modified")
+	}
+}
+
+func TestIsModifiedWithDifferentHeaders(t *testing.T) {
+	r := &Response{StatusCode: 200, ETag: "some etag", LastModified: "some date"}
+	if !r.IsModified("etag", "lastModified") {
+		t.Error("The resource should be considered modified")
+	}
+}