Refactor assets bundler and split Javascript files

vendor/github.com/tdewolff/parse/css/parse.go

@@ -70,6 +70,10 @@ type Token struct {
 	Data []byte
 }
 
+func (t Token) String() string {
+	return t.TokenType.String() + "('" + string(t.Data) + "')"
+}
+
 // Parser is the state for the parser.
 type Parser struct {
 	l     *Lexer

vendor/github.com/tdewolff/parse/js/lex.go

@@ -23,7 +23,8 @@ const (
 	UnknownToken                         // extra token when no token can be matched
 	WhitespaceToken                      // space \t \v \f
 	LineTerminatorToken                  // \r \n \r\n
-	CommentToken
+	SingleLineCommentToken
+	MultiLineCommentToken // token for comments with line terminators (not just any /*block*/)
 	IdentifierToken
 	PunctuatorToken /* { } ( ) [ ] . ; , < > <= >= == != === !==  + - * % ++ -- << >>
 	   >>> & | ^ ! ~ && || ? : = += -= *= %= <<= >>= >>>= &= |= ^= / /= >= */
@@ -68,8 +69,10 @@ func (tt TokenType) String() string {
 		return "Whitespace"
 	case LineTerminatorToken:
 		return "LineTerminator"
-	case CommentToken:
-		return "Comment"
+	case SingleLineCommentToken:
+		return "SingleLineComment"
+	case MultiLineCommentToken:
+		return "MultiLineComment"
 	case IdentifierToken:
 		return "Identifier"
 	case PunctuatorToken:
@@ -174,15 +177,15 @@ func (l *Lexer) Next() (TokenType, []byte) {
 		l.r.Move(1)
 		tt = PunctuatorToken
 	case '<', '>', '=', '!', '+', '-', '*', '%', '&', '|', '^':
-		if (c == '<' || (l.emptyLine && c == '-')) && l.consumeCommentToken() {
-			return CommentToken, l.r.Shift()
+		if l.consumeHTMLLikeCommentToken() {
+			return SingleLineCommentToken, l.r.Shift()
 		} else if l.consumeLongPunctuatorToken() {
 			l.state = ExprState
 			tt = PunctuatorToken
 		}
 	case '/':
-		if l.consumeCommentToken() {
-			return CommentToken, l.r.Shift()
+		if tt = l.consumeCommentToken(); tt != UnknownToken {
+			return tt, l.r.Shift()
 		} else if l.state == ExprState && l.consumeRegexpToken() {
 			l.state = SubscriptState
 			tt = RegexpToken
@@ -374,46 +377,54 @@ func (l *Lexer) consumeSingleLineComment() {
 
 ////////////////////////////////////////////////////////////////
 
-func (l *Lexer) consumeCommentToken() bool {
+func (l *Lexer) consumeHTMLLikeCommentToken() bool {
+	c := l.r.Peek(0)
+	if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
+		// opening HTML-style single line comment
+		l.r.Move(4)
+		l.consumeSingleLineComment()
+		return true
+	} else if l.emptyLine && c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
+		// closing HTML-style single line comment
+		// (only if current line didn't contain any meaningful tokens)
+		l.r.Move(3)
+		l.consumeSingleLineComment()
+		return true
+	}
+	return false
+}
+
+func (l *Lexer) consumeCommentToken() TokenType {
 	c := l.r.Peek(0)
 	if c == '/' {
 		c = l.r.Peek(1)
 		if c == '/' {
-			// single line
+			// single line comment
 			l.r.Move(2)
 			l.consumeSingleLineComment()
+			return SingleLineCommentToken
 		} else if c == '*' {
-			// multi line
+			// block comment (potentially multiline)
+			tt := SingleLineCommentToken
 			l.r.Move(2)
 			for {
 				c := l.r.Peek(0)
 				if c == '*' && l.r.Peek(1) == '/' {
 					l.r.Move(2)
-					return true
+					break
 				} else if c == 0 {
 					break
 				} else if l.consumeLineTerminator() {
+					tt = MultiLineCommentToken
 					l.emptyLine = true
 				} else {
 					l.r.Move(1)
 				}
 			}
-		} else {
-			return false
+			return tt
 		}
-	} else if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
-		// opening HTML-style single line comment
-		l.r.Move(4)
-		l.consumeSingleLineComment()
-	} else if c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
-		// closing HTML-style single line comment
-		// (only if current line didn't contain any meaningful tokens)
-		l.r.Move(3)
-		l.consumeSingleLineComment()
-	} else {
-		return false
 	}
-	return true
+	return UnknownToken
 }
 
 func (l *Lexer) consumeLongPunctuatorToken() bool {
@@ -643,6 +654,12 @@ func (l *Lexer) consumeTemplateToken() bool {
 			l.state = ExprState
 			l.r.Move(2)
 			return true
+		} else if c == '\\' {
+			l.r.Move(1)
+			if c := l.r.Peek(0); c != 0 {
+				l.r.Move(1)
+			}
+			continue
 		} else if c == 0 {
 			l.r.Rewind(mark)
 			return false

vendor/github.com/tdewolff/parse/js/lex_test.go

@@ -20,7 +20,7 @@ func TestTokens(t *testing.T) {
 		{"\n\r\r\n\u2028\u2029", TTs{LineTerminatorToken}},
 		{"5.2 .04 0x0F 5e99", TTs{NumericToken, NumericToken, NumericToken, NumericToken}},
 		{"a = 'string'", TTs{IdentifierToken, PunctuatorToken, StringToken}},
-		{"/*comment*/ //comment", TTs{CommentToken, CommentToken}},
+		{"/*comment*/ //comment", TTs{SingleLineCommentToken, SingleLineCommentToken}},
 		{"{ } ( ) [ ]", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}},
 		{". ; , < > <=", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}},
 		{">= == != === !==", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}},
@@ -31,12 +31,12 @@ func TestTokens(t *testing.T) {
 		{">>= >>>= &= |= ^= =>", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}},
 		{"a = /.*/g;", TTs{IdentifierToken, PunctuatorToken, RegexpToken, PunctuatorToken}},
 
-		{"/*co\nm\u2028m/*ent*/ //co//mment\u2029//comment", TTs{CommentToken, CommentToken, LineTerminatorToken, CommentToken}},
+		{"/*co\nm\u2028m/*ent*/ //co//mment\u2029//comment", TTs{MultiLineCommentToken, SingleLineCommentToken, LineTerminatorToken, SingleLineCommentToken}},
 		{"<!-", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken}},
-		{"1<!--2\n", TTs{NumericToken, CommentToken, LineTerminatorToken}},
+		{"1<!--2\n", TTs{NumericToken, SingleLineCommentToken, LineTerminatorToken}},
 		{"x=y-->10\n", TTs{IdentifierToken, PunctuatorToken, IdentifierToken, PunctuatorToken, PunctuatorToken, NumericToken, LineTerminatorToken}},
-		{"  /*comment*/ -->nothing\n", TTs{CommentToken, CommentToken, LineTerminatorToken}},
-		{"1 /*comment\nmultiline*/ -->nothing\n", TTs{NumericToken, CommentToken, CommentToken, LineTerminatorToken}},
+		{"  /*comment*/ -->nothing\n", TTs{SingleLineCommentToken, SingleLineCommentToken, LineTerminatorToken}},
+		{"1 /*comment\nmultiline*/ -->nothing\n", TTs{NumericToken, MultiLineCommentToken, SingleLineCommentToken, LineTerminatorToken}},
 		{"$ _\u200C \\u2000 \u200C", TTs{IdentifierToken, IdentifierToken, IdentifierToken, UnknownToken}},
 		{">>>=>>>>=", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken}},
 		{"1/", TTs{NumericToken, PunctuatorToken}},
@@ -63,7 +63,7 @@ func TestTokens(t *testing.T) {
 		{"'\n '\u2028", TTs{UnknownToken, LineTerminatorToken, UnknownToken, LineTerminatorToken}},
 		{"'str\\\U00100000ing\\0'", TTs{StringToken}},
 		{"'strin\\00g'", TTs{StringToken}},
-		{"/*comment", TTs{CommentToken}},
+		{"/*comment", TTs{SingleLineCommentToken}},
 		{"a=/regexp", TTs{IdentifierToken, PunctuatorToken, RegexpToken}},
 		{"\\u002", TTs{UnknownToken, IdentifierToken}},
 
@@ -97,6 +97,9 @@ func TestTokens(t *testing.T) {
 		{"function f(){}/1/g", TTs{IdentifierToken, IdentifierToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, RegexpToken}},
 		{"this.return/1/g", TTs{IdentifierToken, PunctuatorToken, IdentifierToken, PunctuatorToken, NumericToken, PunctuatorToken, IdentifierToken}},
 		{"(a+b)/1/g", TTs{PunctuatorToken, IdentifierToken, PunctuatorToken, IdentifierToken, PunctuatorToken, PunctuatorToken, NumericToken, PunctuatorToken, IdentifierToken}},
+		{"`\\``", TTs{TemplateToken}},
+		{"`\\${ 1 }`", TTs{TemplateToken}},
+		{"`\\\r\n`", TTs{TemplateToken}},
 
 		// go fuzz
 		{"`", TTs{UnknownToken}},

vendor/github.com/tdewolff/parse/strconv/int.go

@@ -1,6 +1,8 @@
 package strconv // import "github.com/tdewolff/parse/strconv"
 
-import "math"
+import (
+	"math"
+)
 
 // Int parses a byte-slice and returns the integer it represents.
 // If an invalid character is encountered, it will stop there.
@@ -34,6 +36,9 @@ func ParseInt(b []byte) (int64, int) {
 
 func LenInt(i int64) int {
 	if i < 0 {
+		if i == -9223372036854775808 {
+			return 19
+		}
 		i = -i
 	}
 	switch {

vendor/github.com/tdewolff/parse/strconv/int_test.go

@@ -41,6 +41,8 @@ func TestLenInt(t *testing.T) {
 		{1, 1},
 		{10, 2},
 		{99, 2},
+		{9223372036854775807, 19},
+		{-9223372036854775808, 19},
 
 		// coverage
 		{100, 3},

vendor/github.com/tdewolff/parse/strconv/price.go

@@ -0,0 +1,83 @@
+package strconv
+
+// AppendPrice will append an int64 formatted as a price, where the int64 is the price in cents.
+// It does not display whether a price is negative or not.
+func AppendPrice(b []byte, price int64, dec bool, milSeparator byte, decSeparator byte) []byte {
+	if price < 0 {
+		if price == -9223372036854775808 {
+			x := []byte("92 233 720 368 547 758 08")
+			x[2] = milSeparator
+			x[6] = milSeparator
+			x[10] = milSeparator
+			x[14] = milSeparator
+			x[18] = milSeparator
+			x[22] = decSeparator
+			return append(b, x...)
+		}
+		price = -price
+	}
+
+	// rounding
+	if !dec {
+		firstDec := (price / 10) % 10
+		if firstDec >= 5 {
+			price += 100
+		}
+	}
+
+	// calculate size
+	n := LenInt(price) - 2
+	if n > 0 {
+		n += (n - 1) / 3 // mil separator
+	} else {
+		n = 1
+	}
+	if dec {
+		n += 2 + 1 // decimals + dec separator
+	}
+
+	// resize byte slice
+	i := len(b)
+	if i+n > cap(b) {
+		b = append(b, make([]byte, n)...)
+	} else {
+		b = b[:i+n]
+	}
+
+	// print fractional-part
+	i += n - 1
+	if dec {
+		for j := 0; j < 2; j++ {
+			c := byte(price%10) + '0'
+			price /= 10
+			b[i] = c
+			i--
+		}
+		b[i] = decSeparator
+		i--
+	} else {
+		price /= 100
+	}
+
+	if price == 0 {
+		b[i] = '0'
+		return b
+	}
+
+	// print integer-part
+	j := 0
+	for price > 0 {
+		if j == 3 {
+			b[i] = milSeparator
+			i--
+			j = 0
+		}
+
+		c := byte(price%10) + '0'
+		price /= 10
+		b[i] = c
+		i--
+		j++
+	}
+	return b
+}

vendor/github.com/tdewolff/parse/strconv/price_test.go

@@ -0,0 +1,29 @@
+package strconv // import "github.com/tdewolff/parse/strconv"
+
+import (
+	"testing"
+
+	"github.com/tdewolff/test"
+)
+
+func TestAppendPrice(t *testing.T) {
+	priceTests := []struct {
+		price    int64
+		dec      bool
+		expected string
+	}{
+		{0, false, "0"},
+		{0, true, "0.00"},
+		{100, true, "1.00"},
+		{-100, true, "1.00"},
+		{100000, false, "1,000"},
+		{100000, true, "1,000.00"},
+		{123456789012, true, "1,234,567,890.12"},
+		{9223372036854775807, true, "92,233,720,368,547,758.07"},
+		{-9223372036854775808, true, "92,233,720,368,547,758.08"},
+	}
+	for _, tt := range priceTests {
+		price := AppendPrice([]byte{}, tt.price, tt.dec, ',', '.')
+		test.String(t, string(price), tt.expected, "for", tt.price)
+	}
+}