Normalize URL query string before executing HTTP requests

- Make sure query strings parameters are encoded
- As opposed to the standard library, do not append equal sign
for query parameters with empty value
- Strip URL fragments like Web browsers

url/url.go

@@ -4,9 +4,12 @@
 
 package url // import "miniflux.app/url"
 
-import "net/url"
-import "fmt"
-import "strings"
+import (
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+)
 
 // AbsoluteURL converts the input URL as absolute URL if necessary.
 func AbsoluteURL(baseURL, input string) (string, error) {
@@ -69,3 +72,51 @@ func Domain(websiteURL string) string {
 
 	return parsedURL.Host
 }
+
+// RequestURI returns the encoded URI to be used in HTTP requests.
+func RequestURI(websiteURL string) string {
+	u, err := url.Parse(websiteURL)
+	if err != nil {
+		return websiteURL
+	}
+
+	queryValues := u.Query()
+	u.RawQuery = "" // Clear RawQuery to make sure it's encoded properly.
+	u.Fragment = "" // Clear fragment because Web browsers strip #fragment before sending the URL to a web server.
+
+	var buf strings.Builder
+	buf.WriteString(u.String())
+
+	if len(queryValues) > 0 {
+		buf.WriteByte('?')
+
+		// Sort keys.
+		keys := make([]string, 0, len(queryValues))
+		for k := range queryValues {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		i := 0
+		for _, key := range keys {
+			keyEscaped := url.QueryEscape(key)
+			values := queryValues[key]
+			for _, value := range values {
+				if i > 0 {
+					buf.WriteByte('&')
+				}
+				buf.WriteString(keyEscaped)
+
+				// As opposed to the standard library, we append the = only if the value is not empty.
+				if value != "" {
+					buf.WriteByte('=')
+					buf.WriteString(url.QueryEscape(value))
+				}
+
+				i++
+			}
+		}
+	}
+
+	return buf.String()
+}

url/url_test.go

@@ -71,3 +71,25 @@ func TestDomain(t *testing.T) {
 		}
 	}
 }
+
+func TestRequestURI(t *testing.T) {
+	scenarios := map[string]string{
+		"https://www.example.org":                                                   "https://www.example.org",
+		"https://user:password@www.example.org":                                     "https://user:password@www.example.org",
+		"https://www.example.org/path with spaces":                                  "https://www.example.org/path%20with%20spaces",
+		"https://www.example.org/path#test":                                         "https://www.example.org/path",
+		"https://www.example.org/path?abc#test":                                     "https://www.example.org/path?abc",
+		"https://www.example.org/path?a=b&a=c":                                      "https://www.example.org/path?a=b&a=c",
+		"https://www.example.org/path?a=b&a=c&d":                                    "https://www.example.org/path?a=b&a=c&d",
+		"https://www.example.org/path?atom":                                         "https://www.example.org/path?atom",
+		"https://www.example.org/path?测试=测试":                                        "https://www.example.org/path?%E6%B5%8B%E8%AF%95=%E6%B5%8B%E8%AF%95",
+		"https://www.example.org/url=http%3A%2F%2Fwww.example.com%2Ffeed%2F&max=20": "https://www.example.org/url=http%3A%2F%2Fwww.example.com%2Ffeed%2F&max=20",
+	}
+
+	for input, expected := range scenarios {
+		actual := RequestURI(input)
+		if actual != expected {
+			t.Errorf(`Unexpected result, got %q instead of %q`, actual, expected)
+		}
+	}
+}