mirror of
https://github.com/miniflux/v2.git
synced 2025-09-15 18:57:04 +00:00
Migrate to go modules (Go 1.11)
This commit is contained in:
Frédéric Guillot
parent
a9e9c347f4
commit
a3f3f51c60
2088 changed files with 5130 additions and 908975 deletions
282
vendor/golang.org/x/text/runes/cond_test.go
generated
vendored
282
vendor/golang.org/x/text/runes/cond_test.go
generated
vendored
|
|
@ -1,282 +0,0 @@
|
|||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package runes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
var (
|
||||
toUpper = cases.Upper(language.Und)
|
||||
toLower = cases.Lower(language.Und)
|
||||
)
|
||||
|
||||
type spanformer interface {
|
||||
transform.SpanningTransformer
|
||||
}
|
||||
|
||||
func TestPredicate(t *testing.T) {
|
||||
testConditional(t, func(rt *unicode.RangeTable, t, f spanformer) spanformer {
|
||||
return If(Predicate(func(r rune) bool {
|
||||
return unicode.Is(rt, r)
|
||||
}), t, f)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIn(t *testing.T) {
|
||||
testConditional(t, func(rt *unicode.RangeTable, t, f spanformer) spanformer {
|
||||
return If(In(rt), t, f)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNotIn(t *testing.T) {
|
||||
testConditional(t, func(rt *unicode.RangeTable, t, f spanformer) spanformer {
|
||||
return If(NotIn(rt), f, t)
|
||||
})
|
||||
}
|
||||
|
||||
func testConditional(t *testing.T, f func(rt *unicode.RangeTable, t, f spanformer) spanformer) {
|
||||
lower := f(unicode.Latin, toLower, toLower)
|
||||
|
||||
for i, tt := range []transformTest{{
|
||||
desc: "empty",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "",
|
||||
out: "",
|
||||
outFull: "",
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "small",
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "B",
|
||||
out: "b",
|
||||
outFull: "b",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "short dst",
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "AAA",
|
||||
out: "aa",
|
||||
outFull: "aaa",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "short dst writing error",
|
||||
szDst: 1,
|
||||
atEOF: false,
|
||||
in: "A\x80",
|
||||
out: "a",
|
||||
outFull: "a\x80",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "short dst writing incomplete rune",
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "Σ\xc2",
|
||||
out: "Σ",
|
||||
outFull: "Σ\xc2",
|
||||
err: transform.ErrShortDst,
|
||||
t: f(unicode.Latin, toLower, nil),
|
||||
}, {
|
||||
desc: "short dst, longer",
|
||||
szDst: 5,
|
||||
atEOF: true,
|
||||
in: "Hellø",
|
||||
out: "Hell",
|
||||
outFull: "Hellø",
|
||||
err: transform.ErrShortDst,
|
||||
// idem is used to test short buffers by forcing processing of full-rune increments.
|
||||
t: f(unicode.Latin, Map(idem), nil),
|
||||
}, {
|
||||
desc: "short dst, longer, writing error",
|
||||
szDst: 6,
|
||||
atEOF: false,
|
||||
in: "\x80Hello\x80",
|
||||
out: "\x80Hello",
|
||||
outFull: "\x80Hello\x80",
|
||||
err: transform.ErrShortDst,
|
||||
t: f(unicode.Latin, Map(idem), nil),
|
||||
}, {
|
||||
desc: "short src",
|
||||
szDst: 2,
|
||||
atEOF: false,
|
||||
in: "A\xc2",
|
||||
out: "a",
|
||||
outFull: "a\xc2",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "short src no change",
|
||||
szDst: 2,
|
||||
atEOF: false,
|
||||
in: "a\xc2",
|
||||
out: "a",
|
||||
outFull: "a\xc2",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrShortSrc,
|
||||
nSpan: 1,
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "invalid input, atEOF",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\x80",
|
||||
out: "\x80",
|
||||
outFull: "\x80",
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "invalid input, !atEOF",
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\x80",
|
||||
out: "\x80",
|
||||
outFull: "\x80",
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "invalid input, incomplete rune atEOF",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\xc2",
|
||||
out: "\xc2",
|
||||
outFull: "\xc2",
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "nop",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "Hello World!",
|
||||
out: "Hello World!",
|
||||
outFull: "Hello World!",
|
||||
t: f(unicode.Latin, nil, nil),
|
||||
}, {
|
||||
desc: "nop in",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "THIS IS α ΤΕΣΤ",
|
||||
out: "this is α ΤΕΣΤ",
|
||||
outFull: "this is α ΤΕΣΤ",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: f(unicode.Greek, nil, toLower),
|
||||
}, {
|
||||
desc: "nop in latin",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "THIS IS α ΤΕΣΤ",
|
||||
out: "THIS IS α τεστ",
|
||||
outFull: "THIS IS α τεστ",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: f(unicode.Latin, nil, toLower),
|
||||
}, {
|
||||
desc: "nop not in",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "THIS IS α ΤΕΣΤ",
|
||||
out: "this is α ΤΕΣΤ",
|
||||
outFull: "this is α ΤΕΣΤ",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: f(unicode.Latin, toLower, nil),
|
||||
}, {
|
||||
desc: "pass atEOF is true when at end",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "hello",
|
||||
out: "HELLO",
|
||||
outFull: "HELLO",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: f(unicode.Latin, upperAtEOF{}, nil),
|
||||
}, {
|
||||
desc: "pass atEOF is true when at end of segment",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "hello ",
|
||||
out: "HELLO ",
|
||||
outFull: "HELLO ",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: f(unicode.Latin, upperAtEOF{}, nil),
|
||||
}, {
|
||||
desc: "don't pass atEOF is true when atEOF is false",
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "hello",
|
||||
out: "",
|
||||
outFull: "HELLO",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrShortSrc,
|
||||
t: f(unicode.Latin, upperAtEOF{}, nil),
|
||||
}, {
|
||||
desc: "pass atEOF is true when at end, no change",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "HELLO",
|
||||
out: "HELLO",
|
||||
outFull: "HELLO",
|
||||
t: f(unicode.Latin, upperAtEOF{}, nil),
|
||||
}, {
|
||||
desc: "pass atEOF is true when at end of segment, no change",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "HELLO ",
|
||||
out: "HELLO ",
|
||||
outFull: "HELLO ",
|
||||
t: f(unicode.Latin, upperAtEOF{}, nil),
|
||||
}, {
|
||||
desc: "large input ASCII",
|
||||
szDst: 12000,
|
||||
atEOF: false,
|
||||
in: strings.Repeat("HELLO", 2000),
|
||||
out: strings.Repeat("hello", 2000),
|
||||
outFull: strings.Repeat("hello", 2000),
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
err: nil,
|
||||
t: lower,
|
||||
}, {
|
||||
desc: "large input non-ASCII",
|
||||
szDst: 12000,
|
||||
atEOF: false,
|
||||
in: strings.Repeat("\u3333", 2000),
|
||||
out: strings.Repeat("\u3333", 2000),
|
||||
outFull: strings.Repeat("\u3333", 2000),
|
||||
err: nil,
|
||||
t: lower,
|
||||
}} {
|
||||
tt.check(t, i)
|
||||
}
|
||||
}
|
||||
|
||||
// upperAtEOF is a strange Transformer that converts text to uppercase, but only
|
||||
// if atEOF is true.
|
||||
type upperAtEOF struct{ transform.NopResetter }
|
||||
|
||||
func (upperAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
if !atEOF {
|
||||
return 0, 0, transform.ErrShortSrc
|
||||
}
|
||||
return toUpper.Transform(dst, src, atEOF)
|
||||
}
|
||||
|
||||
func (upperAtEOF) Span(src []byte, atEOF bool) (n int, err error) {
|
||||
if !atEOF {
|
||||
return 0, transform.ErrShortSrc
|
||||
}
|
||||
return toUpper.Span(src, atEOF)
|
||||
}
|
||||
|
||||
func BenchmarkConditional(b *testing.B) {
|
||||
doBench(b, If(In(unicode.Hangul), transform.Nop, transform.Nop))
|
||||
}
|
||||
60
vendor/golang.org/x/text/runes/example_test.go
generated
vendored
60
vendor/golang.org/x/text/runes/example_test.go
generated
vendored
|
|
@ -1,60 +0,0 @@
|
|||
// Copyright 2014 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package runes_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/runes"
|
||||
"golang.org/x/text/transform"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
"golang.org/x/text/width"
|
||||
)
|
||||
|
||||
func ExampleRemove() {
|
||||
t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
|
||||
s, _, _ := transform.String(t, "résumé")
|
||||
fmt.Println(s)
|
||||
|
||||
// Output:
|
||||
// resume
|
||||
}
|
||||
|
||||
func ExampleMap() {
|
||||
replaceHyphens := runes.Map(func(r rune) rune {
|
||||
if unicode.Is(unicode.Hyphen, r) {
|
||||
return '|'
|
||||
}
|
||||
return r
|
||||
})
|
||||
s, _, _ := transform.String(replaceHyphens, "a-b‐c⸗d﹣e")
|
||||
fmt.Println(s)
|
||||
|
||||
// Output:
|
||||
// a|b|c|d|e
|
||||
}
|
||||
|
||||
func ExampleIn() {
|
||||
// Convert Latin characters to their canonical form, while keeping other
|
||||
// width distinctions.
|
||||
t := runes.If(runes.In(unicode.Latin), width.Fold, nil)
|
||||
s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / アルアノリウ ｔｅｃｈ")
|
||||
fmt.Println(s)
|
||||
|
||||
// Output:
|
||||
// ｱﾙｱﾉﾘｳ tech / アルアノリウ tech
|
||||
}
|
||||
|
||||
func ExampleIf() {
|
||||
// Widen everything but ASCII.
|
||||
isASCII := func(r rune) bool { return r <= unicode.MaxASCII }
|
||||
t := runes.If(runes.Predicate(isASCII), nil, width.Widen)
|
||||
s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / 中國 / 5₩")
|
||||
fmt.Println(s)
|
||||
|
||||
// Output:
|
||||
// アルアノリウ tech / 中國 / 5￦
|
||||
}
|
||||
664
vendor/golang.org/x/text/runes/runes_test.go
generated
vendored
664
vendor/golang.org/x/text/runes/runes_test.go
generated
vendored
|
|
@ -1,664 +0,0 @@
|
|||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package runes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/internal/testtext"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
type transformTest struct {
|
||||
desc string
|
||||
szDst int
|
||||
atEOF bool
|
||||
repl string
|
||||
in string
|
||||
out string // result string of first call to Transform
|
||||
outFull string // transform of entire input string
|
||||
err error
|
||||
errSpan error
|
||||
nSpan int
|
||||
|
||||
t transform.SpanningTransformer
|
||||
}
|
||||
|
||||
const large = 10240
|
||||
|
||||
func (tt *transformTest) check(t *testing.T, i int) {
|
||||
if tt.t == nil {
|
||||
return
|
||||
}
|
||||
dst := make([]byte, tt.szDst)
|
||||
src := []byte(tt.in)
|
||||
nDst, nSrc, err := tt.t.Transform(dst, src, tt.atEOF)
|
||||
if err != tt.err {
|
||||
t.Errorf("%d:%s:error: got %v; want %v", i, tt.desc, err, tt.err)
|
||||
}
|
||||
if got := string(dst[:nDst]); got != tt.out {
|
||||
t.Errorf("%d:%s:out: got %q; want %q", i, tt.desc, got, tt.out)
|
||||
}
|
||||
|
||||
// Calls tt.t.Transform for the remainder of the input. We use this to test
|
||||
// the nSrc return value.
|
||||
out := make([]byte, large)
|
||||
n := copy(out, dst[:nDst])
|
||||
nDst, _, _ = tt.t.Transform(out[n:], src[nSrc:], true)
|
||||
if got, want := string(out[:n+nDst]), tt.outFull; got != want {
|
||||
t.Errorf("%d:%s:outFull: got %q; want %q", i, tt.desc, got, want)
|
||||
}
|
||||
|
||||
tt.t.Reset()
|
||||
p := 0
|
||||
for ; p < len(tt.in) && p < len(tt.outFull) && tt.in[p] == tt.outFull[p]; p++ {
|
||||
}
|
||||
if tt.nSpan != 0 {
|
||||
p = tt.nSpan
|
||||
}
|
||||
if n, err = tt.t.Span([]byte(tt.in), tt.atEOF); n != p || err != tt.errSpan {
|
||||
t.Errorf("%d:%s:span: got %d, %v; want %d, %v", i, tt.desc, n, err, p, tt.errSpan)
|
||||
}
|
||||
}
|
||||
|
||||
func idem(r rune) rune { return r }
|
||||
|
||||
func TestMap(t *testing.T) {
|
||||
runes := []rune{'a', 'ç', '中', '\U00012345', 'a'}
|
||||
// Default mapper used for this test.
|
||||
rotate := Map(func(r rune) rune {
|
||||
for i, m := range runes {
|
||||
if m == r {
|
||||
return runes[i+1]
|
||||
}
|
||||
}
|
||||
return r
|
||||
})
|
||||
|
||||
for i, tt := range []transformTest{{
|
||||
desc: "empty",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "",
|
||||
out: "",
|
||||
outFull: "",
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "no change",
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "b",
|
||||
out: "b",
|
||||
outFull: "b",
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short dst",
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "aaaa",
|
||||
out: "ç",
|
||||
outFull: "çççç",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short dst ascii, no change",
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "bbb",
|
||||
out: "bb",
|
||||
outFull: "bbb",
|
||||
err: transform.ErrShortDst,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short dst writing error",
|
||||
szDst: 2,
|
||||
atEOF: false,
|
||||
in: "a\x80",
|
||||
out: "ç",
|
||||
outFull: "ç\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short dst writing incomplete rune",
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "a\xc0",
|
||||
out: "ç",
|
||||
outFull: "ç\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short dst, longer",
|
||||
szDst: 5,
|
||||
atEOF: true,
|
||||
in: "Hellø",
|
||||
out: "Hell",
|
||||
outFull: "Hellø",
|
||||
err: transform.ErrShortDst,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short dst, single",
|
||||
szDst: 1,
|
||||
atEOF: false,
|
||||
in: "ø",
|
||||
out: "",
|
||||
outFull: "ø",
|
||||
err: transform.ErrShortDst,
|
||||
t: Map(idem),
|
||||
}, {
|
||||
desc: "short dst, longer, writing error",
|
||||
szDst: 8,
|
||||
atEOF: false,
|
||||
in: "\x80Hello\x80",
|
||||
out: "\ufffdHello",
|
||||
outFull: "\ufffdHello\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "short src",
|
||||
szDst: 2,
|
||||
atEOF: false,
|
||||
in: "a\xc2",
|
||||
out: "ç",
|
||||
outFull: "ç\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "invalid input, atEOF",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\x80",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "invalid input, !atEOF",
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\x80",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "incomplete rune !atEOF",
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\xc2",
|
||||
out: "",
|
||||
outFull: "\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrShortSrc,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "invalid input, incomplete rune atEOF",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\xc2",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "misc correct",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "a\U00012345 ç!",
|
||||
out: "ça 中!",
|
||||
outFull: "ça 中!",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "misc correct and invalid",
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "Hello\x80 w\x80orl\xc0d!\xc0",
|
||||
out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
|
||||
outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "misc correct and invalid, short src",
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "Hello\x80 w\x80orl\xc0d!\xc2",
|
||||
out: "Hello\ufffd w\ufffdorl\ufffdd!",
|
||||
outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: rotate,
|
||||
}, {
|
||||
desc: "misc correct and invalid, short src, replacing RuneError",
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2",
|
||||
out: "Hel?lo? w?orl?d!",
|
||||
outFull: "Hel?lo? w?orl?d!?",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
err: transform.ErrShortSrc,
|
||||
t: Map(func(r rune) rune {
|
||||
if r == utf8.RuneError {
|
||||
return '?'
|
||||
}
|
||||
return r
|
||||
}),
|
||||
}} {
|
||||
tt.check(t, i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemove(t *testing.T) {
|
||||
remove := Remove(Predicate(func(r rune) bool {
|
||||
return strings.ContainsRune("aeiou\u0300\uFF24\U00012345", r)
|
||||
}))
|
||||
|
||||
for i, tt := range []transformTest{
|
||||
0: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "",
|
||||
out: "",
|
||||
outFull: "",
|
||||
t: remove,
|
||||
},
|
||||
1: {
|
||||
szDst: 0,
|
||||
atEOF: true,
|
||||
in: "aaaa",
|
||||
out: "",
|
||||
outFull: "",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
2: {
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "aaaa",
|
||||
out: "",
|
||||
outFull: "",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
3: {
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "baaaa",
|
||||
out: "b",
|
||||
outFull: "b",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
4: {
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "açaaa",
|
||||
out: "ç",
|
||||
outFull: "ç",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
5: {
|
||||
szDst: 2,
|
||||
atEOF: true,
|
||||
in: "aaaç",
|
||||
out: "ç",
|
||||
outFull: "ç",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
6: {
|
||||
szDst: 2,
|
||||
atEOF: false,
|
||||
in: "a\x80",
|
||||
out: "",
|
||||
outFull: "\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
7: {
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "a\xc0",
|
||||
out: "",
|
||||
outFull: "\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
8: {
|
||||
szDst: 1,
|
||||
atEOF: false,
|
||||
in: "a\xc2",
|
||||
out: "",
|
||||
outFull: "\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
9: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\x80",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
10: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\x80",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
11: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\xc2",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
12: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\xc2",
|
||||
out: "",
|
||||
outFull: "\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrShortSrc,
|
||||
t: remove,
|
||||
},
|
||||
13: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "Hello \U00012345world!",
|
||||
out: "Hll wrld!",
|
||||
outFull: "Hll wrld!",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
14: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "Hello\x80 w\x80orl\xc0d!\xc0",
|
||||
out: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
|
||||
outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
15: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "Hello\x80 w\x80orl\xc0d!\xc2",
|
||||
out: "Hll\ufffd w\ufffdrl\ufffdd!",
|
||||
outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
16: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2",
|
||||
out: "Hello world!",
|
||||
outFull: "Hello world!",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: Remove(Predicate(func(r rune) bool { return r == utf8.RuneError })),
|
||||
},
|
||||
17: {
|
||||
szDst: 4,
|
||||
atEOF: true,
|
||||
in: "Hellø",
|
||||
out: "Hll",
|
||||
outFull: "Hllø",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
18: {
|
||||
szDst: 4,
|
||||
atEOF: false,
|
||||
in: "Hellø",
|
||||
out: "Hll",
|
||||
outFull: "Hllø",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
19: {
|
||||
szDst: 8,
|
||||
atEOF: false,
|
||||
in: "\x80Hello\uFF24\x80",
|
||||
out: "\ufffdHll",
|
||||
outFull: "\ufffdHll\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: remove,
|
||||
},
|
||||
20: {
|
||||
szDst: 8,
|
||||
atEOF: false,
|
||||
in: "Hllll",
|
||||
out: "Hllll",
|
||||
outFull: "Hllll",
|
||||
t: remove,
|
||||
}} {
|
||||
tt.check(t, i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplaceIllFormed(t *testing.T) {
|
||||
replace := ReplaceIllFormed()
|
||||
|
||||
for i, tt := range []transformTest{
|
||||
0: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "",
|
||||
out: "",
|
||||
outFull: "",
|
||||
t: replace,
|
||||
},
|
||||
1: {
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "aa",
|
||||
out: "a",
|
||||
outFull: "aa",
|
||||
err: transform.ErrShortDst,
|
||||
t: replace,
|
||||
},
|
||||
2: {
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "a\x80",
|
||||
out: "a",
|
||||
outFull: "a\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
3: {
|
||||
szDst: 1,
|
||||
atEOF: true,
|
||||
in: "a\xc2",
|
||||
out: "a",
|
||||
outFull: "a\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
4: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\x80",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
5: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\x80",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
6: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "\xc2",
|
||||
out: "\ufffd",
|
||||
outFull: "\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
7: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "\xc2",
|
||||
out: "",
|
||||
outFull: "\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrShortSrc,
|
||||
t: replace,
|
||||
},
|
||||
8: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "Hello world!",
|
||||
out: "Hello world!",
|
||||
outFull: "Hello world!",
|
||||
t: replace,
|
||||
},
|
||||
9: {
|
||||
szDst: large,
|
||||
atEOF: true,
|
||||
in: "Hello\x80 w\x80orl\xc2d!\xc2",
|
||||
out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
|
||||
outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
10: {
|
||||
szDst: large,
|
||||
atEOF: false,
|
||||
in: "Hello\x80 w\x80orl\xc2d!\xc2",
|
||||
out: "Hello\ufffd w\ufffdorl\ufffdd!",
|
||||
outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
|
||||
err: transform.ErrShortSrc,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
16: {
|
||||
szDst: 10,
|
||||
atEOF: false,
|
||||
in: "\x80Hello\x80",
|
||||
out: "\ufffdHello",
|
||||
outFull: "\ufffdHello\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
errSpan: transform.ErrEndOfSpan,
|
||||
t: replace,
|
||||
},
|
||||
17: {
|
||||
szDst: 10,
|
||||
atEOF: false,
|
||||
in: "\ufffdHello\ufffd",
|
||||
out: "\ufffdHello",
|
||||
outFull: "\ufffdHello\ufffd",
|
||||
err: transform.ErrShortDst,
|
||||
t: replace,
|
||||
},
|
||||
} {
|
||||
tt.check(t, i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapAlloc(t *testing.T) {
|
||||
if n := testtext.AllocsPerRun(3, func() {
|
||||
Map(idem).Transform(nil, nil, false)
|
||||
}); n > 0 {
|
||||
t.Errorf("got %f; want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func rmNop(r rune) bool { return false }
|
||||
|
||||
func TestRemoveAlloc(t *testing.T) {
|
||||
if n := testtext.AllocsPerRun(3, func() {
|
||||
Remove(Predicate(rmNop)).Transform(nil, nil, false)
|
||||
}); n > 0 {
|
||||
t.Errorf("got %f; want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplaceIllFormedAlloc(t *testing.T) {
|
||||
if n := testtext.AllocsPerRun(3, func() {
|
||||
ReplaceIllFormed().Transform(nil, nil, false)
|
||||
}); n > 0 {
|
||||
t.Errorf("got %f; want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func doBench(b *testing.B, t Transformer) {
|
||||
for _, bc := range []struct{ name, data string }{
|
||||
{"ascii", testtext.ASCII},
|
||||
{"3byte", testtext.ThreeByteUTF8},
|
||||
} {
|
||||
dst := make([]byte, 2*len(bc.data))
|
||||
src := []byte(bc.data)
|
||||
|
||||
testtext.Bench(b, bc.name+"/transform", func(b *testing.B) {
|
||||
b.SetBytes(int64(len(src)))
|
||||
for i := 0; i < b.N; i++ {
|
||||
t.Transform(dst, src, true)
|
||||
}
|
||||
})
|
||||
src = t.Bytes(src)
|
||||
t.Reset()
|
||||
testtext.Bench(b, bc.name+"/span", func(b *testing.B) {
|
||||
b.SetBytes(int64(len(src)))
|
||||
for i := 0; i < b.N; i++ {
|
||||
t.Span(src, true)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRemove(b *testing.B) {
|
||||
doBench(b, Remove(Predicate(func(r rune) bool { return r == 'e' })))
|
||||
}
|
||||
|
||||
func BenchmarkMapAll(b *testing.B) {
|
||||
doBench(b, Map(func(r rune) rune { return 'a' }))
|
||||
}
|
||||
|
||||
func BenchmarkMapNone(b *testing.B) {
|
||||
doBench(b, Map(func(r rune) rune { return r }))
|
||||
}
|
||||
|
||||
func BenchmarkReplaceIllFormed(b *testing.B) {
|
||||
doBench(b, ReplaceIllFormed())
|
||||
}
|
||||
|
||||
var (
|
||||
input = strings.Repeat("Thé qüick brøwn føx jumps øver the lazy døg. ", 100)
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue