First commit

vendor/golang.org/x/text/cmd/gotext/doc.go

@@ -0,0 +1,35 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// DO NOT EDIT THIS FILE. GENERATED BY go generate.
+// Edit the documentation in other files and rerun go generate to generate this one.
+
+// gotext is a tool for managing text in Go source code.
+//
+// Usage:
+//
+// 	gotext command [arguments]
+//
+// The commands are:
+//
+// 	extract     extract strings to be translated from code
+//
+// Use "go help [command]" for more information about a command.
+//
+// Additional help topics:
+//
+//
+// Use "gotext help [topic]" for more information about that topic.
+//
+//
+// Extract strings to be translated from code
+//
+// Usage:
+//
+// 	go extract <package>*
+//
+//
+//
+//
+package main

vendor/golang.org/x/text/cmd/gotext/extract.go

@@ -0,0 +1,195 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/constant"
+	"go/format"
+	"go/parser"
+	"go/types"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/tools/go/loader"
+)
+
+// TODO:
+// - merge information into existing files
+// - handle different file formats (PO, XLIFF)
+// - handle features (gender, plural)
+// - message rewriting
+
+var cmdExtract = &Command{
+	Run:       runExtract,
+	UsageLine: "extract <package>*",
+	Short:     "extract strings to be translated from code",
+}
+
+func runExtract(cmd *Command, args []string) error {
+	if len(args) == 0 {
+		args = []string{"."}
+	}
+
+	conf := loader.Config{
+		Build:      &build.Default,
+		ParserMode: parser.ParseComments,
+	}
+
+	// Use the initial packages from the command line.
+	args, err := conf.FromArgs(args, false)
+	if err != nil {
+		return err
+	}
+
+	// Load, parse and type-check the whole program.
+	iprog, err := conf.Load()
+	if err != nil {
+		return err
+	}
+
+	// print returns Go syntax for the specified node.
+	print := func(n ast.Node) string {
+		var buf bytes.Buffer
+		format.Node(&buf, conf.Fset, n)
+		return buf.String()
+	}
+
+	var translations []Translation
+
+	for _, info := range iprog.InitialPackages() {
+		for _, f := range info.Files {
+			// Associate comments with nodes.
+			cmap := ast.NewCommentMap(iprog.Fset, f, f.Comments)
+			getComment := func(n ast.Node) string {
+				cs := cmap.Filter(n).Comments()
+				if len(cs) > 0 {
+					return strings.TrimSpace(cs[0].Text())
+				}
+				return ""
+			}
+
+			// Find function calls.
+			ast.Inspect(f, func(n ast.Node) bool {
+				call, ok := n.(*ast.CallExpr)
+				if !ok {
+					return true
+				}
+
+				// Skip calls of functions other than
+				// (*message.Printer).{Sp,Fp,P}rintf.
+				sel, ok := call.Fun.(*ast.SelectorExpr)
+				if !ok {
+					return true
+				}
+				meth := info.Selections[sel]
+				if meth == nil || meth.Kind() != types.MethodVal {
+					return true
+				}
+				// TODO: remove cheap hack and check if the type either
+				// implements some interface or is specifically of type
+				// "golang.org/x/text/message".Printer.
+				m, ok := extractFuncs[path.Base(meth.Recv().String())]
+				if !ok {
+					return true
+				}
+
+				// argn is the index of the format string.
+				argn, ok := m[meth.Obj().Name()]
+				if !ok || argn >= len(call.Args) {
+					return true
+				}
+
+				// Skip calls with non-constant format string.
+				fmtstr := info.Types[call.Args[argn]].Value
+				if fmtstr == nil || fmtstr.Kind() != constant.String {
+					return true
+				}
+
+				posn := conf.Fset.Position(call.Lparen)
+				filepos := fmt.Sprintf("%s:%d:%d", filepath.Base(posn.Filename), posn.Line, posn.Column)
+
+				// TODO: identify the type of the format argument. If it is not
+				// a string, multiple keys may be defined.
+				var key []string
+
+				// TODO: replace substitutions (%v) with a translator friendly
+				// notation. For instance:
+				//     "%d files remaining" -> "{numFiles} files remaining", or
+				//     "%d files remaining" -> "{arg1} files remaining"
+				// Alternatively, this could be done at a later stage.
+				msg := constant.StringVal(fmtstr)
+
+				// Construct a Translation unit.
+				c := Translation{
+					Key:              key,
+					Position:         filepath.Join(info.Pkg.Path(), filepos),
+					Original:         Text{Msg: msg},
+					ExtractedComment: getComment(call.Args[0]),
+					// TODO(fix): this doesn't get the before comment.
+					// Comment: getComment(call),
+				}
+
+				for i, arg := range call.Args[argn+1:] {
+					var val string
+					if v := info.Types[arg].Value; v != nil {
+						val = v.ExactString()
+					}
+					posn := conf.Fset.Position(arg.Pos())
+					filepos := fmt.Sprintf("%s:%d:%d", filepath.Base(posn.Filename), posn.Line, posn.Column)
+					c.Args = append(c.Args, Argument{
+						ID:             i + 1,
+						Type:           info.Types[arg].Type.String(),
+						UnderlyingType: info.Types[arg].Type.Underlying().String(),
+						Expr:           print(arg),
+						Value:          val,
+						Comment:        getComment(arg),
+						Position:       filepath.Join(info.Pkg.Path(), filepos),
+						// TODO report whether it implements
+						// interfaces plural.Interface,
+						// gender.Interface.
+					})
+				}
+
+				translations = append(translations, c)
+				return true
+			})
+		}
+	}
+
+	data, err := json.MarshalIndent(translations, "", "    ")
+	if err != nil {
+		return err
+	}
+	for _, tag := range getLangs() {
+		// TODO: merge with existing files, don't overwrite.
+		os.MkdirAll(*dir, 0744)
+		file := filepath.Join(*dir, fmt.Sprintf("gotext_%v.out.json", tag))
+		if err := ioutil.WriteFile(file, data, 0744); err != nil {
+			return fmt.Errorf("could not create file: %v", err)
+		}
+	}
+	return nil
+}
+
+// extractFuncs indicates the types and methods for which to extract strings,
+// and which argument to extract.
+// TODO: use the types in conf.Import("golang.org/x/text/message") to extract
+// the correct instances.
+var extractFuncs = map[string]map[string]int{
+	// TODO: Printer -> *golang.org/x/text/message.Printer
+	"message.Printer": {
+		"Printf":  0,
+		"Sprintf": 0,
+		"Fprintf": 1,
+	},
+}

vendor/golang.org/x/text/cmd/gotext/main.go

@@ -0,0 +1,356 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go build -o gotext.latest
+//go:generate ./gotext.latest help gendocumentation
+//go:generate rm gotext.latest
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"go/build"
+	"go/format"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync"
+	"text/template"
+	"unicode"
+	"unicode/utf8"
+
+	"golang.org/x/text/language"
+	"golang.org/x/tools/go/buildutil"
+)
+
+func init() {
+	flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
+}
+
+var (
+	dir   = flag.String("dir", "textdata", "default subdirectory to store translation files")
+	langs = flag.String("lang", "en", "comma-separated list of languages to process")
+)
+
+// NOTE: the Command struct is copied from the go tool in core.
+
+// A Command is an implementation of a go command
+// like go build or go fix.
+type Command struct {
+	// Run runs the command.
+	// The args are the arguments after the command name.
+	Run func(cmd *Command, args []string) error
+
+	// UsageLine is the one-line usage message.
+	// The first word in the line is taken to be the command name.
+	UsageLine string
+
+	// Short is the short description shown in the 'go help' output.
+	Short string
+
+	// Long is the long message shown in the 'go help <this-command>' output.
+	Long string
+
+	// Flag is a set of flags specific to this command.
+	Flag flag.FlagSet
+}
+
+// Name returns the command's name: the first word in the usage line.
+func (c *Command) Name() string {
+	name := c.UsageLine
+	i := strings.Index(name, " ")
+	if i >= 0 {
+		name = name[:i]
+	}
+	return name
+}
+
+func (c *Command) Usage() {
+	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine)
+	fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long))
+	os.Exit(2)
+}
+
+// Runnable reports whether the command can be run; otherwise
+// it is a documentation pseudo-command such as importpath.
+func (c *Command) Runnable() bool {
+	return c.Run != nil
+}
+
+// Commands lists the available commands and help topics.
+// The order here is the order in which they are printed by 'go help'.
+var commands = []*Command{
+	cmdExtract,
+	// TODO:
+	// - generate code from translations.
+	// - update: full-cycle update of extraction, sending, and integration
+	// - report: report of freshness of translations
+}
+
+var exitStatus = 0
+var exitMu sync.Mutex
+
+func setExitStatus(n int) {
+	exitMu.Lock()
+	if exitStatus < n {
+		exitStatus = n
+	}
+	exitMu.Unlock()
+}
+
+var origEnv []string
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	log.SetFlags(0)
+
+	args := flag.Args()
+	if len(args) < 1 {
+		usage()
+	}
+
+	if args[0] == "help" {
+		help(args[1:])
+		return
+	}
+
+	for _, cmd := range commands {
+		if cmd.Name() == args[0] && cmd.Runnable() {
+			cmd.Flag.Usage = func() { cmd.Usage() }
+			cmd.Flag.Parse(args[1:])
+			args = cmd.Flag.Args()
+			if err := cmd.Run(cmd, args); err != nil {
+				fatalf("gotext: %v", err)
+			}
+			exit()
+			return
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "gotext: unknown subcommand %q\nRun 'go help' for usage.\n", args[0])
+	setExitStatus(2)
+	exit()
+}
+
+var usageTemplate = `gotext is a tool for managing text in Go source code.
+
+Usage:
+
+	gotext command [arguments]
+
+The commands are:
+{{range .}}{{if .Runnable}}
+	{{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}}
+
+Use "go help [command]" for more information about a command.
+
+Additional help topics:
+{{range .}}{{if not .Runnable}}
+	{{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}}
+
+Use "gotext help [topic]" for more information about that topic.
+
+`
+
+var helpTemplate = `{{if .Runnable}}usage: go {{.UsageLine}}
+
+{{end}}{{.Long | trim}}
+`
+
+var documentationTemplate = `{{range .}}{{if .Short}}{{.Short | capitalize}}
+
+{{end}}{{if .Runnable}}Usage:
+
+	go {{.UsageLine}}
+
+{{end}}{{.Long | trim}}
+
+
+{{end}}`
+
+// commentWriter writes a Go comment to the underlying io.Writer,
+// using line comment form (//).
+type commentWriter struct {
+	W            io.Writer
+	wroteSlashes bool // Wrote "//" at the beginning of the current line.
+}
+
+func (c *commentWriter) Write(p []byte) (int, error) {
+	var n int
+	for i, b := range p {
+		if !c.wroteSlashes {
+			s := "//"
+			if b != '\n' {
+				s = "// "
+			}
+			if _, err := io.WriteString(c.W, s); err != nil {
+				return n, err
+			}
+			c.wroteSlashes = true
+		}
+		n0, err := c.W.Write(p[i : i+1])
+		n += n0
+		if err != nil {
+			return n, err
+		}
+		if b == '\n' {
+			c.wroteSlashes = false
+		}
+	}
+	return len(p), nil
+}
+
+// An errWriter wraps a writer, recording whether a write error occurred.
+type errWriter struct {
+	w   io.Writer
+	err error
+}
+
+func (w *errWriter) Write(b []byte) (int, error) {
+	n, err := w.w.Write(b)
+	if err != nil {
+		w.err = err
+	}
+	return n, err
+}
+
+// tmpl executes the given template text on data, writing the result to w.
+func tmpl(w io.Writer, text string, data interface{}) {
+	t := template.New("top")
+	t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize})
+	template.Must(t.Parse(text))
+	ew := &errWriter{w: w}
+	err := t.Execute(ew, data)
+	if ew.err != nil {
+		// I/O error writing. Ignore write on closed pipe.
+		if strings.Contains(ew.err.Error(), "pipe") {
+			os.Exit(1)
+		}
+		fatalf("writing output: %v", ew.err)
+	}
+	if err != nil {
+		panic(err)
+	}
+}
+
+func capitalize(s string) string {
+	if s == "" {
+		return s
+	}
+	r, n := utf8.DecodeRuneInString(s)
+	return string(unicode.ToTitle(r)) + s[n:]
+}
+
+func printUsage(w io.Writer) {
+	bw := bufio.NewWriter(w)
+	tmpl(bw, usageTemplate, commands)
+	bw.Flush()
+}
+
+func usage() {
+	printUsage(os.Stderr)
+	os.Exit(2)
+}
+
+// help implements the 'help' command.
+func help(args []string) {
+	if len(args) == 0 {
+		printUsage(os.Stdout)
+		// not exit 2: succeeded at 'go help'.
+		return
+	}
+	if len(args) != 1 {
+		fmt.Fprintf(os.Stderr, "usage: go help command\n\nToo many arguments given.\n")
+		os.Exit(2) // failed at 'go help'
+	}
+
+	arg := args[0]
+
+	// 'go help documentation' generates doc.go.
+	if strings.HasSuffix(arg, "documentation") {
+		w := &bytes.Buffer{}
+
+		fmt.Fprintln(w, "// Copyright 2016 The Go Authors. All rights reserved.")
+		fmt.Fprintln(w, "// Use of this source code is governed by a BSD-style")
+		fmt.Fprintln(w, "// license that can be found in the LICENSE file.")
+		fmt.Fprintln(w)
+		fmt.Fprintln(w, "// DO NOT EDIT THIS FILE. GENERATED BY go generate.")
+		fmt.Fprintln(w, "// Edit the documentation in other files and rerun go generate to generate this one.")
+		fmt.Fprintln(w)
+		buf := new(bytes.Buffer)
+		printUsage(buf)
+		usage := &Command{Long: buf.String()}
+		tmpl(&commentWriter{W: w}, documentationTemplate, append([]*Command{usage}, commands...))
+		fmt.Fprintln(w, "package main")
+		if arg == "gendocumentation" {
+			b, err := format.Source(w.Bytes())
+			if err != nil {
+				errorf("Could not format generated docs: %v\n", err)
+			}
+			if err := ioutil.WriteFile("doc.go", b, 0666); err != nil {
+				errorf("Could not create file alldocs.go: %v\n", err)
+			}
+		} else {
+			fmt.Println(w.String())
+		}
+		return
+	}
+
+	for _, cmd := range commands {
+		if cmd.Name() == arg {
+			tmpl(os.Stdout, helpTemplate, cmd)
+			// not exit 2: succeeded at 'go help cmd'.
+			return
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'go help'.\n", arg)
+	os.Exit(2) // failed at 'go help cmd'
+}
+
+func getLangs() (tags []language.Tag) {
+	for _, t := range strings.Split(*langs, ",") {
+		tag, err := language.Parse(t)
+		if err != nil {
+			fatalf("gotext: could not parse language %q: %v", t, err)
+		}
+		tags = append(tags, tag)
+	}
+	return tags
+}
+
+var atexitFuncs []func()
+
+func atexit(f func()) {
+	atexitFuncs = append(atexitFuncs, f)
+}
+
+func exit() {
+	for _, f := range atexitFuncs {
+		f()
+	}
+	os.Exit(exitStatus)
+}
+
+func fatalf(format string, args ...interface{}) {
+	errorf(format, args...)
+	exit()
+}
+
+func errorf(format string, args ...interface{}) {
+	log.Printf(format, args...)
+	setExitStatus(1)
+}
+
+func exitIfErrors() {
+	if exitStatus != 0 {
+		exit()
+	}
+}

vendor/golang.org/x/text/cmd/gotext/message.go

@@ -0,0 +1,127 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// TODO: these definitions should be moved to a package so that the can be used
+// by other tools.
+
+// The file contains the structures used to define translations of a certain
+// messages.
+//
+// A translation may have multiple translations strings, or messages, depending
+// on the feature values of the various arguments. For instance, consider
+// a hypothetical translation from English to English, where the source defines
+// the format string "%d file(s) remaining". A completed translation, expressed
+// in JS, for this format string could look like:
+//
+// {
+//     "Key": [
+//         "\"%d files(s) remaining\""
+//     ],
+//     "Original": {
+//         "Msg": "\"%d files(s) remaining\""
+//     },
+//     "Translation": {
+// 	       "Select": {
+// 	           "Feature": "plural",
+//             "Arg": 1,
+//             "Case": {
+//                 "one":   { "Msg": "1 file remaining" },
+//                 "other": { "Msg": "%d files remaining" }
+//             },
+//         },
+//     },
+//     "Args": [
+//         {
+//             "ID": 2,
+//             "Type": "int",
+//             "UnderlyingType": "int",
+//             "Expr": "nFiles",
+//             "Comment": "number of files remaining",
+//             "Position": "golang.org/x/text/cmd/gotext/demo.go:34:3"
+//         }
+//     ],
+//     "Position": "golang.org/x/text/cmd/gotext/demo.go:33:10",
+// }
+//
+// Alternatively, the Translation section could be written as:
+//
+//     "Translation": {
+// 	       "Msg": "%d %[files]s remaining",
+//         "Var": {
+//             "files" : {
+//                 "Select": {
+//         	           "Feature": "plural",
+//                     "Arg": 1,
+//                     "Case": {
+//                         "one":   { "Msg": "file" },
+//                         "other": { "Msg": "files" }
+//                     }
+//                 }
+//             }
+//         }
+//     }
+
+// A Translation describes a translation for a single language for a single
+// message.
+type Translation struct {
+	// Key contains a list of identifiers for the message. If this list is empty
+	// Original is used as the key.
+	Key               []string `json:"key,omitempty"`
+	Original          Text     `json:"original"`
+	Translation       Text     `json:"translation"`
+	ExtractedComment  string   `json:"extractedComment,omitempty"`
+	TranslatorComment string   `json:"translatorComment,omitempty"`
+
+	Args []Argument `json:"args,omitempty"`
+
+	// Extraction information.
+	Position string `json:"position,omitempty"` // filePosition:line
+}
+
+// An Argument contains information about the arguments passed to a message.
+type Argument struct {
+	ID             interface{} `json:"id"` // An int for printf-style calls, but could be a string.
+	Type           string      `json:"type"`
+	UnderlyingType string      `json:"underlyingType"`
+	Expr           string      `json:"expr"`
+	Value          string      `json:"value,omitempty"`
+	Comment        string      `json:"comment,omitempty"`
+	Position       string      `json:"position,omitempty"`
+
+	// Features contains the features that are available for the implementation
+	// of this argument.
+	Features []Feature `json:"features,omitempty"`
+}
+
+// Feature holds information about a feature that can be implemented by
+// an Argument.
+type Feature struct {
+	Type string `json:"type"` // Right now this is only gender and plural.
+
+	// TODO: possible values and examples for the language under consideration.
+
+}
+
+// Text defines a message to be displayed.
+type Text struct {
+	// Msg and Select contains the message to be displayed. Within a Text value
+	// either Msg or Select is defined.
+	Msg    string  `json:"msg,omitempty"`
+	Select *Select `json:"select,omitempty"`
+	// Var defines a map of variables that may be substituted in the selected
+	// message.
+	Var map[string]Text `json:"var,omitempty"`
+	// Example contains an example message formatted with default values.
+	Example string `json:"example,omitempty"`
+}
+
+// Type Select selects a Text based on the feature value associated with
+// a feature of a certain argument.
+type Select struct {
+	Feature string          `json:"feature"` // Name of variable or Feature type
+	Arg     interface{}     `json:"arg"`     // The argument ID.
+	Cases   map[string]Text `json:"cases"`
+}