diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 13:15:26 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 13:15:26 +0000 |
commit | 82539ad8d59729fb45b0bb0edda8f2bddb719eb1 (patch) | |
tree | 58f0b58e6f44f0e04d4a6373132cf426fa835fa7 /src/go/parser | |
parent | Initial commit. (diff) | |
download | golang-1.17-upstream.tar.xz golang-1.17-upstream.zip |
Adding upstream version 1.17.13.upstream/1.17.13upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/go/parser')
28 files changed, 5741 insertions, 0 deletions
diff --git a/src/go/parser/error_test.go b/src/go/parser/error_test.go new file mode 100644 index 0000000..f4f0a52 --- /dev/null +++ b/src/go/parser/error_test.go @@ -0,0 +1,201 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a parser test harness. The files in the testdata +// directory are parsed and the errors reported are compared against the +// error messages expected in the test files. The test files must end in +// .src rather than .go so that they are not disturbed by gofmt runs. +// +// Expected errors are indicated in the test files by putting a comment +// of the form /* ERROR "rx" */ immediately following an offending token. +// The harness will verify that an error matching the regular expression +// rx is reported at that source position. +// +// For instance, the following test file indicates that a "not declared" +// error should be reported for the undeclared variable x: +// +// package p +// func f() { +// _ = x /* ERROR "not declared" */ + 1 +// } + +package parser + +import ( + "go/internal/typeparams" + "go/scanner" + "go/token" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +const testdata = "testdata" + +// getFile assumes that each filename occurs at most once +func getFile(fset *token.FileSet, filename string) (file *token.File) { + fset.Iterate(func(f *token.File) bool { + if f.Name() == filename { + if file != nil { + panic(filename + " used multiple times") + } + file = f + } + return true + }) + return file +} + +func getPos(fset *token.FileSet, filename string, offset int) token.Pos { + if f := getFile(fset, filename); f != nil { + return f.Pos(offset) + } + return token.NoPos +} + +// ERROR comments must be of the form /* ERROR "rx" */ and rx is +// a regular expression that matches the expected error message. +// The special form /* ERROR HERE "rx" */ must be used for error +// messages that appear immediately after a token, rather than at +// a token's position. +// +var errRx = regexp.MustCompile(`^/\* *ERROR *(HERE)? *"([^"]*)" *\*/$`) + +// expectedErrors collects the regular expressions of ERROR comments found +// in files and returns them as a map of error positions to error messages. +// +func expectedErrors(fset *token.FileSet, filename string, src []byte) map[token.Pos]string { + errors := make(map[token.Pos]string) + + var s scanner.Scanner + // file was parsed already - do not add it again to the file + // set otherwise the position information returned here will + // not match the position information collected by the parser + s.Init(getFile(fset, filename), src, nil, scanner.ScanComments) + var prev token.Pos // position of last non-comment, non-semicolon token + var here token.Pos // position immediately after the token at position prev + + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + return errors + case token.COMMENT: + s := errRx.FindStringSubmatch(lit) + if len(s) == 3 { + pos := prev + if s[1] == "HERE" { + pos = here + } + errors[pos] = s[2] + } + case token.SEMICOLON: + // don't use the position of auto-inserted (invisible) semicolons + if lit != ";" { + break + } + fallthrough + default: + prev = pos + var l int // token length + if tok.IsLiteral() { + l = len(lit) + } else { + l = len(tok.String()) + } + here = prev + token.Pos(l) + } + } +} + +// compareErrors compares the map of expected error messages with the list +// of found errors and reports discrepancies. +// +func compareErrors(t *testing.T, fset *token.FileSet, expected map[token.Pos]string, found scanner.ErrorList) { + t.Helper() + for _, error := range found { + // error.Pos is a token.Position, but we want + // a token.Pos so we can do a map lookup + pos := getPos(fset, error.Pos.Filename, error.Pos.Offset) + if msg, found := expected[pos]; found { + // we expect a message at pos; check if it matches + rx, err := regexp.Compile(msg) + if err != nil { + t.Errorf("%s: %v", error.Pos, err) + continue + } + if match := rx.MatchString(error.Msg); !match { + t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg) + continue + } + // we have a match - eliminate this error + delete(expected, pos) + } else { + // To keep in mind when analyzing failed test output: + // If the same error position occurs multiple times in errors, + // this message will be triggered (because the first error at + // the position removes this position from the expected errors). + t.Errorf("%s: unexpected error: %s", error.Pos, error.Msg) + } + } + + // there should be no expected errors left + if len(expected) > 0 { + t.Errorf("%d errors not reported:", len(expected)) + for pos, msg := range expected { + t.Errorf("%s: %s\n", fset.Position(pos), msg) + } + } +} + +func checkErrors(t *testing.T, filename string, input interface{}, mode Mode, expectErrors bool) { + t.Helper() + src, err := readSource(filename, input) + if err != nil { + t.Error(err) + return + } + + fset := token.NewFileSet() + _, err = ParseFile(fset, filename, src, mode) + found, ok := err.(scanner.ErrorList) + if err != nil && !ok { + t.Error(err) + return + } + found.RemoveMultiples() + + expected := map[token.Pos]string{} + if expectErrors { + // we are expecting the following errors + // (collect these after parsing a file so that it is found in the file set) + expected = expectedErrors(fset, filename, src) + } + + // verify errors returned by the parser + compareErrors(t, fset, expected, found) +} + +func TestErrors(t *testing.T) { + list, err := os.ReadDir(testdata) + if err != nil { + t.Fatal(err) + } + for _, d := range list { + name := d.Name() + if !d.IsDir() && !strings.HasPrefix(name, ".") && (strings.HasSuffix(name, ".src") || strings.HasSuffix(name, ".go2")) { + mode := DeclarationErrors | AllErrors + if strings.HasSuffix(name, ".go2") { + if !typeparams.Enabled { + continue + } + } else { + mode |= typeparams.DisallowParsing + } + checkErrors(t, filepath.Join(testdata, name), nil, mode, true) + } + } +} diff --git a/src/go/parser/example_test.go b/src/go/parser/example_test.go new file mode 100644 index 0000000..c2f7f29 --- /dev/null +++ b/src/go/parser/example_test.go @@ -0,0 +1,43 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser_test + +import ( + "fmt" + "go/parser" + "go/token" +) + +func ExampleParseFile() { + fset := token.NewFileSet() // positions are relative to fset + + src := `package foo + +import ( + "fmt" + "time" +) + +func bar() { + fmt.Println(time.Now()) +}` + + // Parse src but stop after processing the imports. + f, err := parser.ParseFile(fset, "", src, parser.ImportsOnly) + if err != nil { + fmt.Println(err) + return + } + + // Print the imports from the file's AST. + for _, s := range f.Imports { + fmt.Println(s.Path.Value) + } + + // output: + // + // "fmt" + // "time" +} diff --git a/src/go/parser/interface.go b/src/go/parser/interface.go new file mode 100644 index 0000000..eae429e --- /dev/null +++ b/src/go/parser/interface.go @@ -0,0 +1,244 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the exported entry points for invoking the parser. + +package parser + +import ( + "bytes" + "errors" + "go/ast" + "go/token" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// If src != nil, readSource converts src to a []byte if possible; +// otherwise it returns an error. If src == nil, readSource returns +// the result of reading the file specified by filename. +// +func readSource(filename string, src interface{}) ([]byte, error) { + if src != nil { + switch s := src.(type) { + case string: + return []byte(s), nil + case []byte: + return s, nil + case *bytes.Buffer: + // is io.Reader, but src is already available in []byte form + if s != nil { + return s.Bytes(), nil + } + case io.Reader: + return io.ReadAll(s) + } + return nil, errors.New("invalid source") + } + return os.ReadFile(filename) +} + +// A Mode value is a set of flags (or 0). +// They control the amount of source code parsed and other optional +// parser functionality. +// +type Mode uint + +const ( + PackageClauseOnly Mode = 1 << iota // stop parsing after package clause + ImportsOnly // stop parsing after import declarations + ParseComments // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors + SpuriousErrors // same as AllErrors, for backward-compatibility + SkipObjectResolution // don't resolve identifiers to objects - see ParseFile + AllErrors = SpuriousErrors // report all errors (not just the first 10 on different lines) +) + +// ParseFile parses the source code of a single Go source file and returns +// the corresponding ast.File node. The source code may be provided via +// the filename of the source file, or via the src parameter. +// +// If src != nil, ParseFile parses the source from src and the filename is +// only used when recording position information. The type of the argument +// for the src parameter must be string, []byte, or io.Reader. +// If src == nil, ParseFile parses the file specified by filename. +// +// The mode parameter controls the amount of source text parsed and other +// optional parser functionality. If the SkipObjectResolution mode bit is set, +// the object resolution phase of parsing will be skipped, causing File.Scope, +// File.Unresolved, and all Ident.Obj fields to be nil. +// +// Position information is recorded in the file set fset, which must not be +// nil. +// +// If the source couldn't be read, the returned AST is nil and the error +// indicates the specific failure. If the source was read but syntax +// errors were found, the result is a partial AST (with ast.Bad* nodes +// representing the fragments of erroneous source code). Multiple errors +// are returned via a scanner.ErrorList which is sorted by source position. +// +func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) (f *ast.File, err error) { + if fset == nil { + panic("parser.ParseFile: no token.FileSet provided (fset == nil)") + } + + // get source + text, err := readSource(filename, src) + if err != nil { + return nil, err + } + + var p parser + defer func() { + if e := recover(); e != nil { + // resume same panic if it's not a bailout + bail, ok := e.(bailout) + if !ok { + panic(e) + } else if bail.msg != "" { + p.errors.Add(p.file.Position(bail.pos), bail.msg) + } + } + + // set result values + if f == nil { + // source is not a valid Go source file - satisfy + // ParseFile API and return a valid (but) empty + // *ast.File + f = &ast.File{ + Name: new(ast.Ident), + Scope: ast.NewScope(nil), + } + } + + p.errors.Sort() + err = p.errors.Err() + }() + + // parse source + p.init(fset, filename, text, mode) + f = p.parseFile() + + return +} + +// ParseDir calls ParseFile for all files with names ending in ".go" in the +// directory specified by path and returns a map of package name -> package +// AST with all the packages found. +// +// If filter != nil, only the files with fs.FileInfo entries passing through +// the filter (and ending in ".go") are considered. The mode bits are passed +// to ParseFile unchanged. Position information is recorded in fset, which +// must not be nil. +// +// If the directory couldn't be read, a nil map and the respective error are +// returned. If a parse error occurred, a non-nil but incomplete map and the +// first error encountered are returned. +// +func ParseDir(fset *token.FileSet, path string, filter func(fs.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) { + list, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + pkgs = make(map[string]*ast.Package) + for _, d := range list { + if d.IsDir() || !strings.HasSuffix(d.Name(), ".go") { + continue + } + if filter != nil { + info, err := d.Info() + if err != nil { + return nil, err + } + if !filter(info) { + continue + } + } + filename := filepath.Join(path, d.Name()) + if src, err := ParseFile(fset, filename, nil, mode); err == nil { + name := src.Name.Name + pkg, found := pkgs[name] + if !found { + pkg = &ast.Package{ + Name: name, + Files: make(map[string]*ast.File), + } + pkgs[name] = pkg + } + pkg.Files[filename] = src + } else if first == nil { + first = err + } + } + + return +} + +// ParseExprFrom is a convenience function for parsing an expression. +// The arguments have the same meaning as for ParseFile, but the source must +// be a valid Go (type or value) expression. Specifically, fset must not +// be nil. +// +// If the source couldn't be read, the returned AST is nil and the error +// indicates the specific failure. If the source was read but syntax +// errors were found, the result is a partial AST (with ast.Bad* nodes +// representing the fragments of erroneous source code). Multiple errors +// are returned via a scanner.ErrorList which is sorted by source position. +// +func ParseExprFrom(fset *token.FileSet, filename string, src interface{}, mode Mode) (expr ast.Expr, err error) { + if fset == nil { + panic("parser.ParseExprFrom: no token.FileSet provided (fset == nil)") + } + + // get source + text, err := readSource(filename, src) + if err != nil { + return nil, err + } + + var p parser + defer func() { + if e := recover(); e != nil { + // resume same panic if it's not a bailout + bail, ok := e.(bailout) + if !ok { + panic(e) + } else if bail.msg != "" { + p.errors.Add(p.file.Position(bail.pos), bail.msg) + } + } + p.errors.Sort() + err = p.errors.Err() + }() + + // parse expr + p.init(fset, filename, text, mode) + expr = p.parseRhsOrType() + + // If a semicolon was inserted, consume it; + // report an error if there's more tokens. + if p.tok == token.SEMICOLON && p.lit == "\n" { + p.next() + } + p.expect(token.EOF) + + return +} + +// ParseExpr is a convenience function for obtaining the AST of an expression x. +// The position information recorded in the AST is undefined. The filename used +// in error messages is the empty string. +// +// If syntax errors were found, the result is a partial AST (with ast.Bad* nodes +// representing the fragments of erroneous source code). Multiple errors are +// returned via a scanner.ErrorList which is sorted by source position. +// +func ParseExpr(x string) (ast.Expr, error) { + return ParseExprFrom(token.NewFileSet(), "", []byte(x), 0) +} diff --git a/src/go/parser/parser.go b/src/go/parser/parser.go new file mode 100644 index 0000000..2c42b9f --- /dev/null +++ b/src/go/parser/parser.go @@ -0,0 +1,2696 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package parser implements a parser for Go source files. Input may be +// provided in a variety of forms (see the various Parse* functions); the +// output is an abstract syntax tree (AST) representing the Go source. The +// parser is invoked through one of the Parse* functions. +// +// The parser accepts a larger language than is syntactically permitted by +// the Go spec, for simplicity, and for improved robustness in the presence +// of syntax errors. For instance, in method declarations, the receiver is +// treated like an ordinary parameter list and thus may contain multiple +// entries where the spec permits exactly one. Consequently, the corresponding +// field in the AST (ast.FuncDecl.Recv) field is not restricted to one entry. +// +package parser + +import ( + "fmt" + "go/ast" + "go/internal/typeparams" + "go/scanner" + "go/token" + "strconv" + "strings" + "unicode" +) + +// The parser structure holds the parser's internal state. +type parser struct { + file *token.File + errors scanner.ErrorList + scanner scanner.Scanner + + // Tracing/debugging + mode Mode // parsing mode + trace bool // == (mode&Trace != 0) + indent int // indentation used for tracing output + + // Comments + comments []*ast.CommentGroup + leadComment *ast.CommentGroup // last lead comment + lineComment *ast.CommentGroup // last line comment + + // Next token + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + // Error recovery + // (used to limit the number of calls to parser.advance + // w/o making scanning progress - avoids potential endless + // loops across multiple parser functions during error recovery) + syncPos token.Pos // last synchronization position + syncCnt int // number of parser.advance calls without progress + + // Non-syntactic parser control + exprLev int // < 0: in control clause, >= 0: in expression + inRhs bool // if set, the parser is parsing a rhs expression + + imports []*ast.ImportSpec // list of imports + + // nestLev is used to track and limit the recursion depth + // during parsing. + nestLev int +} + +func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) { + p.file = fset.AddFile(filename, -1, len(src)) + var m scanner.Mode + if mode&ParseComments != 0 { + m = scanner.ScanComments + } + eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) } + p.scanner.Init(p.file, src, eh, m) + + p.mode = mode + p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) + p.next() +} + +func (p *parser) parseTypeParams() bool { + return typeparams.Enabled && p.mode&typeparams.DisallowParsing == 0 +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *parser) printTrace(a ...interface{}) { + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + pos := p.file.Position(p.pos) + fmt.Printf("%5d:%3d: ", pos.Line, pos.Column) + i := 2 * p.indent + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *parser, msg string) *parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *parser) { + p.indent-- + p.printTrace(")") +} + +// maxNestLev is the deepest we're willing to recurse during parsing +const maxNestLev int = 1e5 + +func incNestLev(p *parser) *parser { + p.nestLev++ + if p.nestLev > maxNestLev { + p.error(p.pos, "exceeded max nesting depth") + panic(bailout{}) + } + return p +} + +// decNestLev is used to track nesting depth during parsing to prevent stack exhaustion. +// It is used along with incNestLev in a similar fashion to how un and trace are used. +func decNestLev(p *parser) { + p.nestLev-- +} + +// Advance to the next token. +func (p *parser) next0() { + // Because of one-token look-ahead, print the previous token + // when tracing as it provides a more readable output. The + // very first token (!p.pos.IsValid()) is not initialized + // (it is token.ILLEGAL), so don't print it. + if p.trace && p.pos.IsValid() { + s := p.tok.String() + switch { + case p.tok.IsLiteral(): + p.printTrace(s, p.lit) + case p.tok.IsOperator(), p.tok.IsKeyword(): + p.printTrace("\"" + s + "\"") + default: + p.printTrace(s) + } + } + + p.pos, p.tok, p.lit = p.scanner.Scan() +} + +// Consume a comment and return it and the line on which it ends. +func (p *parser) consumeComment() (comment *ast.Comment, endline int) { + // /*-style comments may end on a different line than where they start. + // Scan the comment for '\n' chars and adjust endline accordingly. + endline = p.file.Line(p.pos) + if p.lit[1] == '*' { + // don't use range here - no need to decode Unicode code points + for i := 0; i < len(p.lit); i++ { + if p.lit[i] == '\n' { + endline++ + } + } + } + + comment = &ast.Comment{Slash: p.pos, Text: p.lit} + p.next0() + + return +} + +// Consume a group of adjacent comments, add it to the parser's +// comments list, and return it together with the line at which +// the last comment in the group ends. A non-comment token or n +// empty lines terminate a comment group. +// +func (p *parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) { + var list []*ast.Comment + endline = p.file.Line(p.pos) + for p.tok == token.COMMENT && p.file.Line(p.pos) <= endline+n { + var comment *ast.Comment + comment, endline = p.consumeComment() + list = append(list, comment) + } + + // add comment group to the comments list + comments = &ast.CommentGroup{List: list} + p.comments = append(p.comments, comments) + + return +} + +// Advance to the next non-comment token. In the process, collect +// any comment groups encountered, and remember the last lead and +// line comments. +// +// A lead comment is a comment group that starts and ends in a +// line without any other tokens and that is followed by a non-comment +// token on the line immediately after the comment group. +// +// A line comment is a comment group that follows a non-comment +// token on the same line, and that has no tokens after it on the line +// where it ends. +// +// Lead and line comments may be considered documentation that is +// stored in the AST. +// +func (p *parser) next() { + p.leadComment = nil + p.lineComment = nil + prev := p.pos + p.next0() + + if p.tok == token.COMMENT { + var comment *ast.CommentGroup + var endline int + + if p.file.Line(p.pos) == p.file.Line(prev) { + // The comment is on same line as the previous token; it + // cannot be a lead comment but may be a line comment. + comment, endline = p.consumeCommentGroup(0) + if p.file.Line(p.pos) != endline || p.tok == token.EOF { + // The next token is on a different line, thus + // the last comment group is a line comment. + p.lineComment = comment + } + } + + // consume successor comments, if any + endline = -1 + for p.tok == token.COMMENT { + comment, endline = p.consumeCommentGroup(1) + } + + if endline+1 == p.file.Line(p.pos) { + // The next token is following on the line immediately after the + // comment group, thus the last comment group is a lead comment. + p.leadComment = comment + } + } +} + +// A bailout panic is raised to indicate early termination. pos and msg are +// only populated when bailing out of object resolution. +type bailout struct { + pos token.Pos + msg string +} + +func (p *parser) error(pos token.Pos, msg string) { + if p.trace { + defer un(trace(p, "error: "+msg)) + } + + epos := p.file.Position(pos) + + // If AllErrors is not set, discard errors reported on the same line + // as the last recorded error and stop parsing if there are more than + // 10 errors. + if p.mode&AllErrors == 0 { + n := len(p.errors) + if n > 0 && p.errors[n-1].Pos.Line == epos.Line { + return // discard - likely a spurious error + } + if n > 10 { + panic(bailout{}) + } + } + + p.errors.Add(epos, msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + switch { + case p.tok == token.SEMICOLON && p.lit == "\n": + msg += ", found newline" + case p.tok.IsLiteral(): + // print 123 rather than 'INT', etc. + msg += ", found " + p.lit + default: + msg += ", found '" + p.tok.String() + "'" + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress + return pos +} + +// expect2 is like expect, but it returns an invalid position +// if the expected token is not found. +func (p *parser) expect2(tok token.Token) (pos token.Pos) { + if p.tok == tok { + pos = p.pos + } else { + p.errorExpected(p.pos, "'"+tok.String()+"'") + } + p.next() // make progress + return +} + +// expectClosing is like expect but provides a better error message +// for the common case of a missing comma before a newline. +// +func (p *parser) expectClosing(tok token.Token, context string) token.Pos { + if p.tok != tok && p.tok == token.SEMICOLON && p.lit == "\n" { + p.error(p.pos, "missing ',' before newline in "+context) + p.next() + } + return p.expect(tok) +} + +func (p *parser) expectSemi() { + // semicolon is optional before a closing ')' or '}' + if p.tok != token.RPAREN && p.tok != token.RBRACE { + switch p.tok { + case token.COMMA: + // permit a ',' instead of a ';' but complain + p.errorExpected(p.pos, "';'") + fallthrough + case token.SEMICOLON: + p.next() + default: + p.errorExpected(p.pos, "';'") + p.advance(stmtStart) + } + } +} + +func (p *parser) atComma(context string, follow token.Token) bool { + if p.tok == token.COMMA { + return true + } + if p.tok != follow { + msg := "missing ','" + if p.tok == token.SEMICOLON && p.lit == "\n" { + msg += " before newline" + } + p.error(p.pos, msg+" in "+context) + return true // "insert" comma and continue + } + return false +} + +func assert(cond bool, msg string) { + if !cond { + panic("go/parser internal error: " + msg) + } +} + +// advance consumes tokens until the current token p.tok +// is in the 'to' set, or token.EOF. For error recovery. +func (p *parser) advance(to map[token.Token]bool) { + for ; p.tok != token.EOF; p.next() { + if to[p.tok] { + // Return only if parser made some progress since last + // sync or if it has not reached 10 advance calls without + // progress. Otherwise consume at least one token to + // avoid an endless parser loop (it is possible that + // both parseOperand and parseStmt call advance and + // correctly do not advance, thus the need for the + // invocation limit p.syncCnt). + if p.pos == p.syncPos && p.syncCnt < 10 { + p.syncCnt++ + return + } + if p.pos > p.syncPos { + p.syncPos = p.pos + p.syncCnt = 0 + return + } + // Reaching here indicates a parser bug, likely an + // incorrect token list in this function, but it only + // leads to skipping of possibly correct code if a + // previous error is present, and thus is preferred + // over a non-terminating parse. + } + } +} + +var stmtStart = map[token.Token]bool{ + token.BREAK: true, + token.CONST: true, + token.CONTINUE: true, + token.DEFER: true, + token.FALLTHROUGH: true, + token.FOR: true, + token.GO: true, + token.GOTO: true, + token.IF: true, + token.RETURN: true, + token.SELECT: true, + token.SWITCH: true, + token.TYPE: true, + token.VAR: true, +} + +var declStart = map[token.Token]bool{ + token.CONST: true, + token.TYPE: true, + token.VAR: true, +} + +var exprEnd = map[token.Token]bool{ + token.COMMA: true, + token.COLON: true, + token.SEMICOLON: true, + token.RPAREN: true, + token.RBRACK: true, + token.RBRACE: true, +} + +// safePos returns a valid file position for a given position: If pos +// is valid to begin with, safePos returns pos. If pos is out-of-range, +// safePos returns the EOF position. +// +// This is hack to work around "artificial" end positions in the AST which +// are computed by adding 1 to (presumably valid) token positions. If the +// token positions are invalid due to parse errors, the resulting end position +// may be past the file's EOF position, which would lead to panics if used +// later on. +// +func (p *parser) safePos(pos token.Pos) (res token.Pos) { + defer func() { + if recover() != nil { + res = token.Pos(p.file.Base() + p.file.Size()) // EOF position + } + }() + _ = p.file.Offset(pos) // trigger a panic if position is out-of-range + return pos +} + +// ---------------------------------------------------------------------------- +// Identifiers + +func (p *parser) parseIdent() *ast.Ident { + pos := p.pos + name := "_" + if p.tok == token.IDENT { + name = p.lit + p.next() + } else { + p.expect(token.IDENT) // use expect() error handling + } + return &ast.Ident{NamePos: pos, Name: name} +} + +func (p *parser) parseIdentList() (list []*ast.Ident) { + if p.trace { + defer un(trace(p, "IdentList")) + } + + list = append(list, p.parseIdent()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseIdent()) + } + + return +} + +// ---------------------------------------------------------------------------- +// Common productions + +// If lhs is set, result list elements which are identifiers are not resolved. +func (p *parser) parseExprList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ExpressionList")) + } + + list = append(list, p.checkExpr(p.parseExpr())) + for p.tok == token.COMMA { + p.next() + list = append(list, p.checkExpr(p.parseExpr())) + } + + return +} + +func (p *parser) parseList(inRhs bool) []ast.Expr { + old := p.inRhs + p.inRhs = inRhs + list := p.parseExprList() + p.inRhs = old + return list +} + +// ---------------------------------------------------------------------------- +// Types + +func (p *parser) parseType() ast.Expr { + if p.trace { + defer un(trace(p, "Type")) + } + + typ := p.tryIdentOrType() + + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.advance(exprEnd) + return &ast.BadExpr{From: pos, To: p.pos} + } + + return typ +} + +func (p *parser) parseQualifiedIdent(ident *ast.Ident) ast.Expr { + if p.trace { + defer un(trace(p, "QualifiedIdent")) + } + + typ := p.parseTypeName(ident) + if p.tok == token.LBRACK && p.parseTypeParams() { + typ = p.parseTypeInstance(typ) + } + + return typ +} + +// If the result is an identifier, it is not resolved. +func (p *parser) parseTypeName(ident *ast.Ident) ast.Expr { + if p.trace { + defer un(trace(p, "TypeName")) + } + + if ident == nil { + ident = p.parseIdent() + } + + if p.tok == token.PERIOD { + // ident is a package name + p.next() + sel := p.parseIdent() + return &ast.SelectorExpr{X: ident, Sel: sel} + } + + return ident +} + +func (p *parser) parseArrayLen() ast.Expr { + if p.trace { + defer un(trace(p, "ArrayLen")) + } + + p.exprLev++ + var len ast.Expr + // always permit ellipsis for more fault-tolerant parsing + if p.tok == token.ELLIPSIS { + len = &ast.Ellipsis{Ellipsis: p.pos} + p.next() + } else if p.tok != token.RBRACK { + len = p.parseRhs() + } + p.exprLev-- + + return len +} + +func (p *parser) parseArrayFieldOrTypeInstance(x *ast.Ident) (*ast.Ident, ast.Expr) { + if p.trace { + defer un(trace(p, "ArrayFieldOrTypeInstance")) + } + + // TODO(gri) Should we allow a trailing comma in a type argument + // list such as T[P,]? (We do in parseTypeInstance). + lbrack := p.expect(token.LBRACK) + var args []ast.Expr + var firstComma token.Pos + // TODO(rfindley): consider changing parseRhsOrType so that this function variable + // is not needed. + argparser := p.parseRhsOrType + if !p.parseTypeParams() { + argparser = p.parseRhs + } + if p.tok != token.RBRACK { + p.exprLev++ + args = append(args, argparser()) + for p.tok == token.COMMA { + if !firstComma.IsValid() { + firstComma = p.pos + } + p.next() + args = append(args, argparser()) + } + p.exprLev-- + } + rbrack := p.expect(token.RBRACK) + + if len(args) == 0 { + // x []E + elt := p.parseType() + return x, &ast.ArrayType{Lbrack: lbrack, Elt: elt} + } + + // x [P]E or x[P] + if len(args) == 1 { + elt := p.tryIdentOrType() + if elt != nil { + // x [P]E + return x, &ast.ArrayType{Lbrack: lbrack, Len: args[0], Elt: elt} + } + if !p.parseTypeParams() { + p.error(rbrack, "missing element type in array type expression") + return nil, &ast.BadExpr{From: args[0].Pos(), To: args[0].End()} + } + } + + if !p.parseTypeParams() { + p.error(firstComma, "expected ']', found ','") + return x, &ast.BadExpr{From: args[0].Pos(), To: args[len(args)-1].End()} + } + + // x[P], x[P1, P2], ... + return nil, &ast.IndexExpr{X: x, Lbrack: lbrack, Index: typeparams.PackExpr(args), Rbrack: rbrack} +} + +func (p *parser) parseFieldDecl() *ast.Field { + if p.trace { + defer un(trace(p, "FieldDecl")) + } + + doc := p.leadComment + + var names []*ast.Ident + var typ ast.Expr + if p.tok == token.IDENT { + name := p.parseIdent() + if p.tok == token.PERIOD || p.tok == token.STRING || p.tok == token.SEMICOLON || p.tok == token.RBRACE { + // embedded type + typ = name + if p.tok == token.PERIOD { + typ = p.parseQualifiedIdent(name) + } + } else { + // name1, name2, ... T + names = []*ast.Ident{name} + for p.tok == token.COMMA { + p.next() + names = append(names, p.parseIdent()) + } + // Careful dance: We don't know if we have an embedded instantiated + // type T[P1, P2, ...] or a field T of array type []E or [P]E. + if len(names) == 1 && p.tok == token.LBRACK { + name, typ = p.parseArrayFieldOrTypeInstance(name) + if name == nil { + names = nil + } + } else { + // T P + typ = p.parseType() + } + } + } else { + // embedded, possibly generic type + // (using the enclosing parentheses to distinguish it from a named field declaration) + // TODO(rFindley) confirm that this doesn't allow parenthesized embedded type + typ = p.parseType() + } + + var tag *ast.BasicLit + if p.tok == token.STRING { + tag = &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + p.next() + } + + p.expectSemi() // call before accessing p.linecomment + + field := &ast.Field{Doc: doc, Names: names, Type: typ, Tag: tag, Comment: p.lineComment} + return field +} + +func (p *parser) parseStructType() *ast.StructType { + if p.trace { + defer un(trace(p, "StructType")) + } + + pos := p.expect(token.STRUCT) + lbrace := p.expect(token.LBRACE) + var list []*ast.Field + for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN { + // a field declaration cannot start with a '(' but we accept + // it here for more robust parsing and better error messages + // (parseFieldDecl will check and complain if necessary) + list = append(list, p.parseFieldDecl()) + } + rbrace := p.expect(token.RBRACE) + + return &ast.StructType{ + Struct: pos, + Fields: &ast.FieldList{ + Opening: lbrace, + List: list, + Closing: rbrace, + }, + } +} + +func (p *parser) parsePointerType() *ast.StarExpr { + if p.trace { + defer un(trace(p, "PointerType")) + } + + star := p.expect(token.MUL) + base := p.parseType() + + return &ast.StarExpr{Star: star, X: base} +} + +func (p *parser) parseDotsType() *ast.Ellipsis { + if p.trace { + defer un(trace(p, "DotsType")) + } + + pos := p.expect(token.ELLIPSIS) + elt := p.parseType() + + return &ast.Ellipsis{Ellipsis: pos, Elt: elt} +} + +type field struct { + name *ast.Ident + typ ast.Expr +} + +func (p *parser) parseParamDecl(name *ast.Ident) (f field) { + // TODO(rFindley) compare with parser.paramDeclOrNil in the syntax package + if p.trace { + defer un(trace(p, "ParamDeclOrNil")) + } + + ptok := p.tok + if name != nil { + p.tok = token.IDENT // force token.IDENT case in switch below + } + + switch p.tok { + case token.IDENT: + if name != nil { + f.name = name + p.tok = ptok + } else { + f.name = p.parseIdent() + } + switch p.tok { + case token.IDENT, token.MUL, token.ARROW, token.FUNC, token.CHAN, token.MAP, token.STRUCT, token.INTERFACE, token.LPAREN: + // name type + f.typ = p.parseType() + + case token.LBRACK: + // name[type1, type2, ...] or name []type or name [len]type + f.name, f.typ = p.parseArrayFieldOrTypeInstance(f.name) + + case token.ELLIPSIS: + // name ...type + f.typ = p.parseDotsType() + + case token.PERIOD: + // qualified.typename + f.typ = p.parseQualifiedIdent(f.name) + f.name = nil + } + + case token.MUL, token.ARROW, token.FUNC, token.LBRACK, token.CHAN, token.MAP, token.STRUCT, token.INTERFACE, token.LPAREN: + // type + f.typ = p.parseType() + + case token.ELLIPSIS: + // ...type + // (always accepted) + f.typ = p.parseDotsType() + + default: + p.errorExpected(p.pos, ")") + p.advance(exprEnd) + } + + return +} + +func (p *parser) parseParameterList(name0 *ast.Ident, closing token.Token, parseParamDecl func(*ast.Ident) field, tparams bool) (params []*ast.Field) { + if p.trace { + defer un(trace(p, "ParameterList")) + } + + pos := p.pos + if name0 != nil { + pos = name0.Pos() + } + + var list []field + var named int // number of parameters that have an explicit name and type + + for name0 != nil || p.tok != closing && p.tok != token.EOF { + par := parseParamDecl(name0) + name0 = nil // 1st name was consumed if present + if par.name != nil || par.typ != nil { + list = append(list, par) + if par.name != nil && par.typ != nil { + named++ + } + } + if !p.atComma("parameter list", closing) { + break + } + p.next() + } + + if len(list) == 0 { + return // not uncommon + } + + // TODO(gri) parameter distribution and conversion to []*ast.Field + // can be combined and made more efficient + + // distribute parameter types + if named == 0 { + // all unnamed => found names are type names + for i := 0; i < len(list); i++ { + par := &list[i] + if typ := par.name; typ != nil { + par.typ = typ + par.name = nil + } + } + if tparams { + p.error(pos, "all type parameters must be named") + } + } else if named != len(list) { + // some named => all must be named + ok := true + var typ ast.Expr + for i := len(list) - 1; i >= 0; i-- { + if par := &list[i]; par.typ != nil { + typ = par.typ + if par.name == nil { + ok = false + n := ast.NewIdent("_") + n.NamePos = typ.Pos() // correct position + par.name = n + } + } else if typ != nil { + par.typ = typ + } else { + // par.typ == nil && typ == nil => we only have a par.name + ok = false + par.typ = &ast.BadExpr{From: par.name.Pos(), To: p.pos} + } + } + if !ok { + if tparams { + p.error(pos, "all type parameters must be named") + } else { + p.error(pos, "mixed named and unnamed parameters") + } + } + } + + // convert list []*ast.Field + if named == 0 { + // parameter list consists of types only + for _, par := range list { + assert(par.typ != nil, "nil type in unnamed parameter list") + params = append(params, &ast.Field{Type: par.typ}) + } + return + } + + // parameter list consists of named parameters with types + var names []*ast.Ident + var typ ast.Expr + addParams := func() { + assert(typ != nil, "nil type in named parameter list") + field := &ast.Field{Names: names, Type: typ} + params = append(params, field) + names = nil + } + for _, par := range list { + if par.typ != typ { + if len(names) > 0 { + addParams() + } + typ = par.typ + } + names = append(names, par.name) + } + if len(names) > 0 { + addParams() + } + return +} + +func (p *parser) parseParameters(acceptTParams bool) (tparams, params *ast.FieldList) { + if p.trace { + defer un(trace(p, "Parameters")) + } + + if p.parseTypeParams() && acceptTParams && p.tok == token.LBRACK { + opening := p.pos + p.next() + // [T any](params) syntax + list := p.parseParameterList(nil, token.RBRACK, p.parseParamDecl, true) + rbrack := p.expect(token.RBRACK) + tparams = &ast.FieldList{Opening: opening, List: list, Closing: rbrack} + // Type parameter lists must not be empty. + if tparams.NumFields() == 0 { + p.error(tparams.Closing, "empty type parameter list") + tparams = nil // avoid follow-on errors + } + } + + opening := p.expect(token.LPAREN) + + var fields []*ast.Field + if p.tok != token.RPAREN { + fields = p.parseParameterList(nil, token.RPAREN, p.parseParamDecl, false) + } + + rparen := p.expect(token.RPAREN) + params = &ast.FieldList{Opening: opening, List: fields, Closing: rparen} + + return +} + +func (p *parser) parseResult() *ast.FieldList { + if p.trace { + defer un(trace(p, "Result")) + } + + if p.tok == token.LPAREN { + _, results := p.parseParameters(false) + return results + } + + typ := p.tryIdentOrType() + if typ != nil { + list := make([]*ast.Field, 1) + list[0] = &ast.Field{Type: typ} + return &ast.FieldList{List: list} + } + + return nil +} + +func (p *parser) parseFuncType() *ast.FuncType { + if p.trace { + defer un(trace(p, "FuncType")) + } + + pos := p.expect(token.FUNC) + tparams, params := p.parseParameters(true) + if tparams != nil { + p.error(tparams.Pos(), "function type cannot have type parameters") + } + results := p.parseResult() + + return &ast.FuncType{Func: pos, Params: params, Results: results} +} + +func (p *parser) parseMethodSpec() *ast.Field { + if p.trace { + defer un(trace(p, "MethodSpec")) + } + + doc := p.leadComment + var idents []*ast.Ident + var typ ast.Expr + x := p.parseTypeName(nil) + if ident, _ := x.(*ast.Ident); ident != nil { + switch { + case p.tok == token.LBRACK && p.parseTypeParams(): + // generic method or embedded instantiated type + lbrack := p.pos + p.next() + p.exprLev++ + x := p.parseExpr() + p.exprLev-- + if name0, _ := x.(*ast.Ident); name0 != nil && p.tok != token.COMMA && p.tok != token.RBRACK { + // generic method m[T any] + list := p.parseParameterList(name0, token.RBRACK, p.parseParamDecl, true) + rbrack := p.expect(token.RBRACK) + tparams := &ast.FieldList{Opening: lbrack, List: list, Closing: rbrack} + // TODO(rfindley) refactor to share code with parseFuncType. + _, params := p.parseParameters(false) + results := p.parseResult() + idents = []*ast.Ident{ident} + typ = &ast.FuncType{Func: token.NoPos, Params: params, Results: results} + typeparams.Set(typ, tparams) + } else { + // embedded instantiated type + // TODO(rfindley) should resolve all identifiers in x. + list := []ast.Expr{x} + if p.atComma("type argument list", token.RBRACK) { + p.exprLev++ + for p.tok != token.RBRACK && p.tok != token.EOF { + list = append(list, p.parseType()) + if !p.atComma("type argument list", token.RBRACK) { + break + } + p.next() + } + p.exprLev-- + } + rbrack := p.expectClosing(token.RBRACK, "type argument list") + typ = &ast.IndexExpr{X: ident, Lbrack: lbrack, Index: typeparams.PackExpr(list), Rbrack: rbrack} + } + case p.tok == token.LPAREN: + // ordinary method + // TODO(rfindley) refactor to share code with parseFuncType. + _, params := p.parseParameters(false) + results := p.parseResult() + idents = []*ast.Ident{ident} + typ = &ast.FuncType{Func: token.NoPos, Params: params, Results: results} + default: + // embedded type + typ = x + } + } else { + // embedded, possibly instantiated type + typ = x + if p.tok == token.LBRACK && p.parseTypeParams() { + // embedded instantiated interface + typ = p.parseTypeInstance(typ) + } + } + p.expectSemi() // call before accessing p.linecomment + + spec := &ast.Field{Doc: doc, Names: idents, Type: typ, Comment: p.lineComment} + + return spec +} + +func (p *parser) parseInterfaceType() *ast.InterfaceType { + if p.trace { + defer un(trace(p, "InterfaceType")) + } + + pos := p.expect(token.INTERFACE) + lbrace := p.expect(token.LBRACE) + var list []*ast.Field + for p.tok == token.IDENT || p.parseTypeParams() && p.tok == token.TYPE { + if p.tok == token.IDENT { + list = append(list, p.parseMethodSpec()) + } else { + // all types in a type list share the same field name "type" + // (since type is a keyword, a Go program cannot have that field name) + name := []*ast.Ident{{NamePos: p.pos, Name: "type"}} + p.next() + // add each type as a field named "type" + for _, typ := range p.parseTypeList() { + list = append(list, &ast.Field{Names: name, Type: typ}) + } + p.expectSemi() + } + } + // TODO(rfindley): the error produced here could be improved, since we could + // accept a identifier, 'type', or a '}' at this point. + rbrace := p.expect(token.RBRACE) + + return &ast.InterfaceType{ + Interface: pos, + Methods: &ast.FieldList{ + Opening: lbrace, + List: list, + Closing: rbrace, + }, + } +} + +func (p *parser) parseMapType() *ast.MapType { + if p.trace { + defer un(trace(p, "MapType")) + } + + pos := p.expect(token.MAP) + p.expect(token.LBRACK) + key := p.parseType() + p.expect(token.RBRACK) + value := p.parseType() + + return &ast.MapType{Map: pos, Key: key, Value: value} +} + +func (p *parser) parseChanType() *ast.ChanType { + if p.trace { + defer un(trace(p, "ChanType")) + } + + pos := p.pos + dir := ast.SEND | ast.RECV + var arrow token.Pos + if p.tok == token.CHAN { + p.next() + if p.tok == token.ARROW { + arrow = p.pos + p.next() + dir = ast.SEND + } + } else { + arrow = p.expect(token.ARROW) + p.expect(token.CHAN) + dir = ast.RECV + } + value := p.parseType() + + return &ast.ChanType{Begin: pos, Arrow: arrow, Dir: dir, Value: value} +} + +func (p *parser) parseTypeInstance(typ ast.Expr) ast.Expr { + assert(p.parseTypeParams(), "parseTypeInstance while not parsing type params") + if p.trace { + defer un(trace(p, "TypeInstance")) + } + + opening := p.expect(token.LBRACK) + + p.exprLev++ + var list []ast.Expr + for p.tok != token.RBRACK && p.tok != token.EOF { + list = append(list, p.parseType()) + if !p.atComma("type argument list", token.RBRACK) { + break + } + p.next() + } + p.exprLev-- + + closing := p.expectClosing(token.RBRACK, "type argument list") + + return &ast.IndexExpr{X: typ, Lbrack: opening, Index: typeparams.PackExpr(list), Rbrack: closing} +} + +func (p *parser) tryIdentOrType() ast.Expr { + defer decNestLev(incNestLev(p)) + + switch p.tok { + case token.IDENT: + typ := p.parseTypeName(nil) + if p.tok == token.LBRACK && p.parseTypeParams() { + typ = p.parseTypeInstance(typ) + } + return typ + case token.LBRACK: + lbrack := p.expect(token.LBRACK) + alen := p.parseArrayLen() + p.expect(token.RBRACK) + elt := p.parseType() + return &ast.ArrayType{Lbrack: lbrack, Len: alen, Elt: elt} + case token.STRUCT: + return p.parseStructType() + case token.MUL: + return p.parsePointerType() + case token.FUNC: + typ := p.parseFuncType() + return typ + case token.INTERFACE: + return p.parseInterfaceType() + case token.MAP: + return p.parseMapType() + case token.CHAN, token.ARROW: + return p.parseChanType() + case token.LPAREN: + lparen := p.pos + p.next() + typ := p.parseType() + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{Lparen: lparen, X: typ, Rparen: rparen} + } + + // no type found + return nil +} + +// ---------------------------------------------------------------------------- +// Blocks + +func (p *parser) parseStmtList() (list []ast.Stmt) { + if p.trace { + defer un(trace(p, "StatementList")) + } + + for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseStmt()) + } + + return +} + +func (p *parser) parseBody() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "Body")) + } + + lbrace := p.expect(token.LBRACE) + list := p.parseStmtList() + rbrace := p.expect2(token.RBRACE) + + return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} +} + +func (p *parser) parseBlockStmt() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "BlockStmt")) + } + + lbrace := p.expect(token.LBRACE) + list := p.parseStmtList() + rbrace := p.expect2(token.RBRACE) + + return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} +} + +// ---------------------------------------------------------------------------- +// Expressions + +func (p *parser) parseFuncTypeOrLit() ast.Expr { + if p.trace { + defer un(trace(p, "FuncTypeOrLit")) + } + + typ := p.parseFuncType() + if p.tok != token.LBRACE { + // function type only + return typ + } + + p.exprLev++ + body := p.parseBody() + p.exprLev-- + + return &ast.FuncLit{Type: typ, Body: body} +} + +// parseOperand may return an expression or a raw type (incl. array +// types of the form [...]T. Callers must verify the result. +// +func (p *parser) parseOperand() ast.Expr { + if p.trace { + defer un(trace(p, "Operand")) + } + + switch p.tok { + case token.IDENT: + x := p.parseIdent() + return x + + case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: + x := &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + p.next() + return x + + case token.LPAREN: + lparen := p.pos + p.next() + p.exprLev++ + x := p.parseRhsOrType() // types may be parenthesized: (some type) + p.exprLev-- + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{Lparen: lparen, X: x, Rparen: rparen} + + case token.FUNC: + return p.parseFuncTypeOrLit() + } + + if typ := p.tryIdentOrType(); typ != nil { // do not consume trailing type parameters + // could be type for composite literal or conversion + _, isIdent := typ.(*ast.Ident) + assert(!isIdent, "type cannot be identifier") + return typ + } + + // we have an error + pos := p.pos + p.errorExpected(pos, "operand") + p.advance(stmtStart) + return &ast.BadExpr{From: pos, To: p.pos} +} + +func (p *parser) parseSelector(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "Selector")) + } + + sel := p.parseIdent() + + return &ast.SelectorExpr{X: x, Sel: sel} +} + +func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeAssertion")) + } + + lparen := p.expect(token.LPAREN) + var typ ast.Expr + if p.tok == token.TYPE { + // type switch: typ == nil + p.next() + } else { + typ = p.parseType() + } + rparen := p.expect(token.RPAREN) + + return &ast.TypeAssertExpr{X: x, Type: typ, Lparen: lparen, Rparen: rparen} +} + +func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "parseIndexOrSliceOrInstance")) + } + + lbrack := p.expect(token.LBRACK) + if p.tok == token.RBRACK { + // empty index, slice or index expressions are not permitted; + // accept them for parsing tolerance, but complain + p.errorExpected(p.pos, "operand") + rbrack := p.pos + p.next() + return &ast.IndexExpr{ + X: x, + Lbrack: lbrack, + Index: &ast.BadExpr{From: rbrack, To: rbrack}, + Rbrack: rbrack, + } + } + p.exprLev++ + + const N = 3 // change the 3 to 2 to disable 3-index slices + var args []ast.Expr + var index [N]ast.Expr + var colons [N - 1]token.Pos + var firstComma token.Pos + if p.tok != token.COLON { + // We can't know if we have an index expression or a type instantiation; + // so even if we see a (named) type we are not going to be in type context. + index[0] = p.parseRhsOrType() + } + ncolons := 0 + switch p.tok { + case token.COLON: + // slice expression + for p.tok == token.COLON && ncolons < len(colons) { + colons[ncolons] = p.pos + ncolons++ + p.next() + if p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF { + index[ncolons] = p.parseRhs() + } + } + case token.COMMA: + firstComma = p.pos + // instance expression + args = append(args, index[0]) + for p.tok == token.COMMA { + p.next() + if p.tok != token.RBRACK && p.tok != token.EOF { + args = append(args, p.parseType()) + } + } + } + + p.exprLev-- + rbrack := p.expect(token.RBRACK) + + if ncolons > 0 { + // slice expression + slice3 := false + if ncolons == 2 { + slice3 = true + // Check presence of 2nd and 3rd index here rather than during type-checking + // to prevent erroneous programs from passing through gofmt (was issue 7305). + if index[1] == nil { + p.error(colons[0], "2nd index required in 3-index slice") + index[1] = &ast.BadExpr{From: colons[0] + 1, To: colons[1]} + } + if index[2] == nil { + p.error(colons[1], "3rd index required in 3-index slice") + index[2] = &ast.BadExpr{From: colons[1] + 1, To: rbrack} + } + } + return &ast.SliceExpr{X: x, Lbrack: lbrack, Low: index[0], High: index[1], Max: index[2], Slice3: slice3, Rbrack: rbrack} + } + + if len(args) == 0 { + // index expression + return &ast.IndexExpr{X: x, Lbrack: lbrack, Index: index[0], Rbrack: rbrack} + } + + if !p.parseTypeParams() { + p.error(firstComma, "expected ']' or ':', found ','") + return &ast.BadExpr{From: args[0].Pos(), To: args[len(args)-1].End()} + } + + // instance expression + return &ast.IndexExpr{X: x, Lbrack: lbrack, Index: typeparams.PackExpr(args), Rbrack: rbrack} +} + +func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { + if p.trace { + defer un(trace(p, "CallOrConversion")) + } + + lparen := p.expect(token.LPAREN) + p.exprLev++ + var list []ast.Expr + var ellipsis token.Pos + for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() { + list = append(list, p.parseRhsOrType()) // builtins may expect a type: make(some type, ...) + if p.tok == token.ELLIPSIS { + ellipsis = p.pos + p.next() + } + if !p.atComma("argument list", token.RPAREN) { + break + } + p.next() + } + p.exprLev-- + rparen := p.expectClosing(token.RPAREN, "argument list") + + return &ast.CallExpr{Fun: fun, Lparen: lparen, Args: list, Ellipsis: ellipsis, Rparen: rparen} +} + +func (p *parser) parseValue() ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + if p.tok == token.LBRACE { + return p.parseLiteralValue(nil) + } + + x := p.checkExpr(p.parseExpr()) + + return x +} + +func (p *parser) parseElement() ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + x := p.parseValue() + if p.tok == token.COLON { + colon := p.pos + p.next() + x = &ast.KeyValueExpr{Key: x, Colon: colon, Value: p.parseValue()} + } + + return x +} + +func (p *parser) parseElementList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ElementList")) + } + + for p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseElement()) + if !p.atComma("composite literal", token.RBRACE) { + break + } + p.next() + } + + return +} + +func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "LiteralValue")) + } + + lbrace := p.expect(token.LBRACE) + var elts []ast.Expr + p.exprLev++ + if p.tok != token.RBRACE { + elts = p.parseElementList() + } + p.exprLev-- + rbrace := p.expectClosing(token.RBRACE, "composite literal") + return &ast.CompositeLit{Type: typ, Lbrace: lbrace, Elts: elts, Rbrace: rbrace} +} + +// checkExpr checks that x is an expression (and not a type). +func (p *parser) checkExpr(x ast.Expr) ast.Expr { + switch unparen(x).(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.BasicLit: + case *ast.FuncLit: + case *ast.CompositeLit: + case *ast.ParenExpr: + panic("unreachable") + case *ast.SelectorExpr: + case *ast.IndexExpr: + case *ast.SliceExpr: + case *ast.TypeAssertExpr: + // If t.Type == nil we have a type assertion of the form + // y.(type), which is only allowed in type switch expressions. + // It's hard to exclude those but for the case where we are in + // a type switch. Instead be lenient and test this in the type + // checker. + case *ast.CallExpr: + case *ast.StarExpr: + case *ast.UnaryExpr: + case *ast.BinaryExpr: + default: + // all other nodes are not proper expressions + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{From: x.Pos(), To: p.safePos(x.End())} + } + return x +} + +// If x is of the form (T), unparen returns unparen(T), otherwise it returns x. +func unparen(x ast.Expr) ast.Expr { + if p, isParen := x.(*ast.ParenExpr); isParen { + x = unparen(p.X) + } + return x +} + +// checkExprOrType checks that x is an expression or a type +// (and not a raw type such as [...]T). +// +func (p *parser) checkExprOrType(x ast.Expr) ast.Expr { + switch t := unparen(x).(type) { + case *ast.ParenExpr: + panic("unreachable") + case *ast.ArrayType: + if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis { + p.error(len.Pos(), "expected array length, found '...'") + x = &ast.BadExpr{From: x.Pos(), To: p.safePos(x.End())} + } + } + + // all other nodes are expressions or types + return x +} + +func (p *parser) parsePrimaryExpr() (x ast.Expr) { + if p.trace { + defer un(trace(p, "PrimaryExpr")) + } + + x = p.parseOperand() + // We track the nesting here rather than at the entry for the function, + // since it can iteratively produce a nested output, and we want to + // limit how deep a structure we generate. + var n int + defer func() { p.nestLev -= n }() + for n = 1; ; n++ { + incNestLev(p) + switch p.tok { + case token.PERIOD: + p.next() + switch p.tok { + case token.IDENT: + x = p.parseSelector(p.checkExprOrType(x)) + case token.LPAREN: + x = p.parseTypeAssertion(p.checkExpr(x)) + default: + pos := p.pos + p.errorExpected(pos, "selector or type assertion") + // TODO(rFindley) The check for token.RBRACE below is a targeted fix + // to error recovery sufficient to make the x/tools tests to + // pass with the new parsing logic introduced for type + // parameters. Remove this once error recovery has been + // more generally reconsidered. + if p.tok != token.RBRACE { + p.next() // make progress + } + sel := &ast.Ident{NamePos: pos, Name: "_"} + x = &ast.SelectorExpr{X: x, Sel: sel} + } + case token.LBRACK: + x = p.parseIndexOrSliceOrInstance(p.checkExpr(x)) + case token.LPAREN: + x = p.parseCallOrConversion(p.checkExprOrType(x)) + case token.LBRACE: + // operand may have returned a parenthesized complit + // type; accept it but complain if we have a complit + t := unparen(x) + // determine if '{' belongs to a composite literal or a block statement + switch t.(type) { + case *ast.BadExpr, *ast.Ident, *ast.SelectorExpr: + if p.exprLev < 0 { + return + } + // x is possibly a composite literal type + case *ast.IndexExpr: + if p.exprLev < 0 { + return + } + // x is possibly a composite literal type + case *ast.ArrayType, *ast.StructType, *ast.MapType: + // x is a composite literal type + default: + return + } + if t != x { + p.error(t.Pos(), "cannot parenthesize type in composite literal") + // already progressed, no need to advance + } + x = p.parseLiteralValue(x) + default: + return + } + } +} + +func (p *parser) parseUnaryExpr() ast.Expr { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "UnaryExpr")) + } + + switch p.tok { + case token.ADD, token.SUB, token.NOT, token.XOR, token.AND: + pos, op := p.pos, p.tok + p.next() + x := p.parseUnaryExpr() + return &ast.UnaryExpr{OpPos: pos, Op: op, X: p.checkExpr(x)} + + case token.ARROW: + // channel type or receive expression + arrow := p.pos + p.next() + + // If the next token is token.CHAN we still don't know if it + // is a channel type or a receive operation - we only know + // once we have found the end of the unary expression. There + // are two cases: + // + // <- type => (<-type) must be channel type + // <- expr => <-(expr) is a receive from an expression + // + // In the first case, the arrow must be re-associated with + // the channel type parsed already: + // + // <- (chan type) => (<-chan type) + // <- (chan<- type) => (<-chan (<-type)) + + x := p.parseUnaryExpr() + + // determine which case we have + if typ, ok := x.(*ast.ChanType); ok { + // (<-type) + + // re-associate position info and <- + dir := ast.SEND + for ok && dir == ast.SEND { + if typ.Dir == ast.RECV { + // error: (<-type) is (<-(<-chan T)) + p.errorExpected(typ.Arrow, "'chan'") + } + arrow, typ.Begin, typ.Arrow = typ.Arrow, arrow, arrow + dir, typ.Dir = typ.Dir, ast.RECV + typ, ok = typ.Value.(*ast.ChanType) + } + if dir == ast.SEND { + p.errorExpected(arrow, "channel type") + } + + return x + } + + // <-(expr) + return &ast.UnaryExpr{OpPos: arrow, Op: token.ARROW, X: p.checkExpr(x)} + + case token.MUL: + // pointer type or unary "*" expression + pos := p.pos + p.next() + x := p.parseUnaryExpr() + return &ast.StarExpr{Star: pos, X: p.checkExprOrType(x)} + } + + return p.parsePrimaryExpr() +} + +func (p *parser) tokPrec() (token.Token, int) { + tok := p.tok + if p.inRhs && tok == token.ASSIGN { + tok = token.EQL + } + return tok, tok.Precedence() +} + +func (p *parser) parseBinaryExpr(prec1 int) ast.Expr { + if p.trace { + defer un(trace(p, "BinaryExpr")) + } + + x := p.parseUnaryExpr() + // We track the nesting here rather than at the entry for the function, + // since it can iteratively produce a nested output, and we want to + // limit how deep a structure we generate. + var n int + defer func() { p.nestLev -= n }() + for n = 1; ; n++ { + incNestLev(p) + op, oprec := p.tokPrec() + if oprec < prec1 { + return x + } + pos := p.expect(op) + y := p.parseBinaryExpr(oprec + 1) + x = &ast.BinaryExpr{X: p.checkExpr(x), OpPos: pos, Op: op, Y: p.checkExpr(y)} + } +} + +// The result may be a type or even a raw type ([...]int). Callers must +// check the result (using checkExpr or checkExprOrType), depending on +// context. +func (p *parser) parseExpr() ast.Expr { + if p.trace { + defer un(trace(p, "Expression")) + } + + return p.parseBinaryExpr(token.LowestPrec + 1) +} + +func (p *parser) parseRhs() ast.Expr { + old := p.inRhs + p.inRhs = true + x := p.checkExpr(p.parseExpr()) + p.inRhs = old + return x +} + +func (p *parser) parseRhsOrType() ast.Expr { + old := p.inRhs + p.inRhs = true + x := p.checkExprOrType(p.parseExpr()) + p.inRhs = old + return x +} + +// ---------------------------------------------------------------------------- +// Statements + +// Parsing modes for parseSimpleStmt. +const ( + basic = iota + labelOk + rangeOk +) + +// parseSimpleStmt returns true as 2nd result if it parsed the assignment +// of a range clause (with mode == rangeOk). The returned statement is an +// assignment with a right-hand side that is a single unary expression of +// the form "range x". No guarantees are given for the left-hand side. +func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) { + if p.trace { + defer un(trace(p, "SimpleStmt")) + } + + x := p.parseList(false) + + switch p.tok { + case + token.DEFINE, token.ASSIGN, token.ADD_ASSIGN, + token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN, + token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN, + token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN: + // assignment statement, possibly part of a range clause + pos, tok := p.pos, p.tok + p.next() + var y []ast.Expr + isRange := false + if mode == rangeOk && p.tok == token.RANGE && (tok == token.DEFINE || tok == token.ASSIGN) { + pos := p.pos + p.next() + y = []ast.Expr{&ast.UnaryExpr{OpPos: pos, Op: token.RANGE, X: p.parseRhs()}} + isRange = true + } else { + y = p.parseList(true) + } + as := &ast.AssignStmt{Lhs: x, TokPos: pos, Tok: tok, Rhs: y} + if tok == token.DEFINE { + p.checkAssignStmt(as) + } + return as, isRange + } + + if len(x) > 1 { + p.errorExpected(x[0].Pos(), "1 expression") + // continue with first expression + } + + switch p.tok { + case token.COLON: + // labeled statement + colon := p.pos + p.next() + if label, isIdent := x[0].(*ast.Ident); mode == labelOk && isIdent { + // Go spec: The scope of a label is the body of the function + // in which it is declared and excludes the body of any nested + // function. + stmt := &ast.LabeledStmt{Label: label, Colon: colon, Stmt: p.parseStmt()} + return stmt, false + } + // The label declaration typically starts at x[0].Pos(), but the label + // declaration may be erroneous due to a token after that position (and + // before the ':'). If SpuriousErrors is not set, the (only) error + // reported for the line is the illegal label error instead of the token + // before the ':' that caused the problem. Thus, use the (latest) colon + // position for error reporting. + p.error(colon, "illegal label declaration") + return &ast.BadStmt{From: x[0].Pos(), To: colon + 1}, false + + case token.ARROW: + // send statement + arrow := p.pos + p.next() + y := p.parseRhs() + return &ast.SendStmt{Chan: x[0], Arrow: arrow, Value: y}, false + + case token.INC, token.DEC: + // increment or decrement + s := &ast.IncDecStmt{X: x[0], TokPos: p.pos, Tok: p.tok} + p.next() + return s, false + } + + // expression + return &ast.ExprStmt{X: x[0]}, false +} + +func (p *parser) checkAssignStmt(as *ast.AssignStmt) { + for _, x := range as.Lhs { + if _, isIdent := x.(*ast.Ident); !isIdent { + p.errorExpected(x.Pos(), "identifier on left side of :=") + } + } +} + +func (p *parser) parseCallExpr(callType string) *ast.CallExpr { + x := p.parseRhsOrType() // could be a conversion: (some type)(x) + if call, isCall := x.(*ast.CallExpr); isCall { + return call + } + if _, isBad := x.(*ast.BadExpr); !isBad { + // only report error if it's a new one + p.error(p.safePos(x.End()), fmt.Sprintf("function must be invoked in %s statement", callType)) + } + return nil +} + +func (p *parser) parseGoStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "GoStmt")) + } + + pos := p.expect(token.GO) + call := p.parseCallExpr("go") + p.expectSemi() + if call == nil { + return &ast.BadStmt{From: pos, To: pos + 2} // len("go") + } + + return &ast.GoStmt{Go: pos, Call: call} +} + +func (p *parser) parseDeferStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "DeferStmt")) + } + + pos := p.expect(token.DEFER) + call := p.parseCallExpr("defer") + p.expectSemi() + if call == nil { + return &ast.BadStmt{From: pos, To: pos + 5} // len("defer") + } + + return &ast.DeferStmt{Defer: pos, Call: call} +} + +func (p *parser) parseReturnStmt() *ast.ReturnStmt { + if p.trace { + defer un(trace(p, "ReturnStmt")) + } + + pos := p.pos + p.expect(token.RETURN) + var x []ast.Expr + if p.tok != token.SEMICOLON && p.tok != token.RBRACE { + x = p.parseList(true) + } + p.expectSemi() + + return &ast.ReturnStmt{Return: pos, Results: x} +} + +func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt { + if p.trace { + defer un(trace(p, "BranchStmt")) + } + + pos := p.expect(tok) + var label *ast.Ident + if tok != token.FALLTHROUGH && p.tok == token.IDENT { + label = p.parseIdent() + } + p.expectSemi() + + return &ast.BranchStmt{TokPos: pos, Tok: tok, Label: label} +} + +func (p *parser) makeExpr(s ast.Stmt, want string) ast.Expr { + if s == nil { + return nil + } + if es, isExpr := s.(*ast.ExprStmt); isExpr { + return p.checkExpr(es.X) + } + found := "simple statement" + if _, isAss := s.(*ast.AssignStmt); isAss { + found = "assignment" + } + p.error(s.Pos(), fmt.Sprintf("expected %s, found %s (missing parentheses around composite literal?)", want, found)) + return &ast.BadExpr{From: s.Pos(), To: p.safePos(s.End())} +} + +// parseIfHeader is an adjusted version of parser.header +// in cmd/compile/internal/syntax/parser.go, which has +// been tuned for better error handling. +func (p *parser) parseIfHeader() (init ast.Stmt, cond ast.Expr) { + if p.tok == token.LBRACE { + p.error(p.pos, "missing condition in if statement") + cond = &ast.BadExpr{From: p.pos, To: p.pos} + return + } + // p.tok != token.LBRACE + + prevLev := p.exprLev + p.exprLev = -1 + + if p.tok != token.SEMICOLON { + // accept potential variable declaration but complain + if p.tok == token.VAR { + p.next() + p.error(p.pos, "var declaration not allowed in 'IF' initializer") + } + init, _ = p.parseSimpleStmt(basic) + } + + var condStmt ast.Stmt + var semi struct { + pos token.Pos + lit string // ";" or "\n"; valid if pos.IsValid() + } + if p.tok != token.LBRACE { + if p.tok == token.SEMICOLON { + semi.pos = p.pos + semi.lit = p.lit + p.next() + } else { + p.expect(token.SEMICOLON) + } + if p.tok != token.LBRACE { + condStmt, _ = p.parseSimpleStmt(basic) + } + } else { + condStmt = init + init = nil + } + + if condStmt != nil { + cond = p.makeExpr(condStmt, "boolean expression") + } else if semi.pos.IsValid() { + if semi.lit == "\n" { + p.error(semi.pos, "unexpected newline, expecting { after if clause") + } else { + p.error(semi.pos, "missing condition in if statement") + } + } + + // make sure we have a valid AST + if cond == nil { + cond = &ast.BadExpr{From: p.pos, To: p.pos} + } + + p.exprLev = prevLev + return +} + +func (p *parser) parseIfStmt() *ast.IfStmt { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + + init, cond := p.parseIfHeader() + body := p.parseBlockStmt() + + var else_ ast.Stmt + if p.tok == token.ELSE { + p.next() + switch p.tok { + case token.IF: + else_ = p.parseIfStmt() + case token.LBRACE: + else_ = p.parseBlockStmt() + p.expectSemi() + default: + p.errorExpected(p.pos, "if statement or block") + else_ = &ast.BadStmt{From: p.pos, To: p.pos} + } + } else { + p.expectSemi() + } + + return &ast.IfStmt{If: pos, Init: init, Cond: cond, Body: body, Else: else_} +} + +func (p *parser) parseTypeList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "TypeList")) + } + + list = append(list, p.parseType()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseType()) + } + + return +} + +func (p *parser) parseCaseClause(typeSwitch bool) *ast.CaseClause { + if p.trace { + defer un(trace(p, "CaseClause")) + } + + pos := p.pos + var list []ast.Expr + if p.tok == token.CASE { + p.next() + if typeSwitch { + list = p.parseTypeList() + } else { + list = p.parseList(true) + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + + return &ast.CaseClause{Case: pos, List: list, Colon: colon, Body: body} +} + +func isTypeSwitchAssert(x ast.Expr) bool { + a, ok := x.(*ast.TypeAssertExpr) + return ok && a.Type == nil +} + +func (p *parser) isTypeSwitchGuard(s ast.Stmt) bool { + switch t := s.(type) { + case *ast.ExprStmt: + // x.(type) + return isTypeSwitchAssert(t.X) + case *ast.AssignStmt: + // v := x.(type) + if len(t.Lhs) == 1 && len(t.Rhs) == 1 && isTypeSwitchAssert(t.Rhs[0]) { + switch t.Tok { + case token.ASSIGN: + // permit v = x.(type) but complain + p.error(t.TokPos, "expected ':=', found '='") + fallthrough + case token.DEFINE: + return true + } + } + } + return false +} + +func (p *parser) parseSwitchStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "SwitchStmt")) + } + + pos := p.expect(token.SWITCH) + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2, _ = p.parseSimpleStmt(basic) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + // A TypeSwitchGuard may declare a variable in addition + // to the variable declared in the initial SimpleStmt. + // Introduce extra scope to avoid redeclaration errors: + // + // switch t := 0; t := x.(T) { ... } + // + // (this code is not valid Go because the first t + // cannot be accessed and thus is never used, the extra + // scope is needed for the correct error message). + // + // If we don't have a type switch, s2 must be an expression. + // Having the extra nested but empty scope won't affect it. + s2, _ = p.parseSimpleStmt(basic) + } + } + p.exprLev = prevLev + } + + typeSwitch := p.isTypeSwitchGuard(s2) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCaseClause(typeSwitch)) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} + + if typeSwitch { + return &ast.TypeSwitchStmt{Switch: pos, Init: s1, Assign: s2, Body: body} + } + + return &ast.SwitchStmt{Switch: pos, Init: s1, Tag: p.makeExpr(s2, "switch expression"), Body: body} +} + +func (p *parser) parseCommClause() *ast.CommClause { + if p.trace { + defer un(trace(p, "CommClause")) + } + + pos := p.pos + var comm ast.Stmt + if p.tok == token.CASE { + p.next() + lhs := p.parseList(false) + if p.tok == token.ARROW { + // SendStmt + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + arrow := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.SendStmt{Chan: lhs[0], Arrow: arrow, Value: rhs} + } else { + // RecvStmt + if tok := p.tok; tok == token.ASSIGN || tok == token.DEFINE { + // RecvStmt with assignment + if len(lhs) > 2 { + p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") + // continue with first two expressions + lhs = lhs[0:2] + } + pos := p.pos + p.next() + rhs := p.parseRhs() + as := &ast.AssignStmt{Lhs: lhs, TokPos: pos, Tok: tok, Rhs: []ast.Expr{rhs}} + if tok == token.DEFINE { + p.checkAssignStmt(as) + } + comm = as + } else { + // lhs must be single receive operation + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + comm = &ast.ExprStmt{X: lhs[0]} + } + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + + return &ast.CommClause{Case: pos, Comm: comm, Colon: colon, Body: body} +} + +func (p *parser) parseSelectStmt() *ast.SelectStmt { + if p.trace { + defer un(trace(p, "SelectStmt")) + } + + pos := p.expect(token.SELECT) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCommClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} + + return &ast.SelectStmt{Select: pos, Body: body} +} + +func (p *parser) parseForStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "ForStmt")) + } + + pos := p.expect(token.FOR) + + var s1, s2, s3 ast.Stmt + var isRange bool + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + if p.tok == token.RANGE { + // "for range x" (nil lhs in assignment) + pos := p.pos + p.next() + y := []ast.Expr{&ast.UnaryExpr{OpPos: pos, Op: token.RANGE, X: p.parseRhs()}} + s2 = &ast.AssignStmt{Rhs: y} + isRange = true + } else { + s2, isRange = p.parseSimpleStmt(rangeOk) + } + } + if !isRange && p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2, _ = p.parseSimpleStmt(basic) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3, _ = p.parseSimpleStmt(basic) + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + p.expectSemi() + + if isRange { + as := s2.(*ast.AssignStmt) + // check lhs + var key, value ast.Expr + switch len(as.Lhs) { + case 0: + // nothing to do + case 1: + key = as.Lhs[0] + case 2: + key, value = as.Lhs[0], as.Lhs[1] + default: + p.errorExpected(as.Lhs[len(as.Lhs)-1].Pos(), "at most 2 expressions") + return &ast.BadStmt{From: pos, To: p.safePos(body.End())} + } + // parseSimpleStmt returned a right-hand side that + // is a single unary expression of the form "range x" + x := as.Rhs[0].(*ast.UnaryExpr).X + return &ast.RangeStmt{ + For: pos, + Key: key, + Value: value, + TokPos: as.TokPos, + Tok: as.Tok, + X: x, + Body: body, + } + } + + // regular for statement + return &ast.ForStmt{ + For: pos, + Init: s1, + Cond: p.makeExpr(s2, "boolean or range expression"), + Post: s3, + Body: body, + } +} + +func (p *parser) parseStmt() (s ast.Stmt) { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "Statement")) + } + + switch p.tok { + case token.CONST, token.TYPE, token.VAR: + s = &ast.DeclStmt{Decl: p.parseDecl(stmtStart)} + case + // tokens that may start an expression + token.IDENT, token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operands + token.LBRACK, token.STRUCT, token.MAP, token.CHAN, token.INTERFACE, // composite types + token.ADD, token.SUB, token.MUL, token.AND, token.XOR, token.ARROW, token.NOT: // unary operators + s, _ = p.parseSimpleStmt(labelOk) + // because of the required look-ahead, labeled statements are + // parsed by parseSimpleStmt - don't expect a semicolon after + // them + if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt { + p.expectSemi() + } + case token.GO: + s = p.parseGoStmt() + case token.DEFER: + s = p.parseDeferStmt() + case token.RETURN: + s = p.parseReturnStmt() + case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH: + s = p.parseBranchStmt(p.tok) + case token.LBRACE: + s = p.parseBlockStmt() + p.expectSemi() + case token.IF: + s = p.parseIfStmt() + case token.SWITCH: + s = p.parseSwitchStmt() + case token.SELECT: + s = p.parseSelectStmt() + case token.FOR: + s = p.parseForStmt() + case token.SEMICOLON: + // Is it ever possible to have an implicit semicolon + // producing an empty statement in a valid program? + // (handle correctly anyway) + s = &ast.EmptyStmt{Semicolon: p.pos, Implicit: p.lit == "\n"} + p.next() + case token.RBRACE: + // a semicolon may be omitted before a closing "}" + s = &ast.EmptyStmt{Semicolon: p.pos, Implicit: true} + default: + // no statement found + pos := p.pos + p.errorExpected(pos, "statement") + p.advance(stmtStart) + s = &ast.BadStmt{From: pos, To: p.pos} + } + + return +} + +// ---------------------------------------------------------------------------- +// Declarations + +type parseSpecFunction func(doc *ast.CommentGroup, pos token.Pos, keyword token.Token, iota int) ast.Spec + +func isValidImport(lit string) bool { + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + s, _ := strconv.Unquote(lit) // go/scanner returns a legal string literal + for _, r := range s { + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return false + } + } + return s != "" +} + +func (p *parser) parseImportSpec(doc *ast.CommentGroup, _ token.Pos, _ token.Token, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "ImportSpec")) + } + + var ident *ast.Ident + switch p.tok { + case token.PERIOD: + ident = &ast.Ident{NamePos: p.pos, Name: "."} + p.next() + case token.IDENT: + ident = p.parseIdent() + } + + pos := p.pos + var path string + if p.tok == token.STRING { + path = p.lit + if !isValidImport(path) { + p.error(pos, "invalid import path: "+path) + } + p.next() + } else { + p.expect(token.STRING) // use expect() error handling + } + p.expectSemi() // call before accessing p.linecomment + + // collect imports + spec := &ast.ImportSpec{ + Doc: doc, + Name: ident, + Path: &ast.BasicLit{ValuePos: pos, Kind: token.STRING, Value: path}, + Comment: p.lineComment, + } + p.imports = append(p.imports, spec) + + return spec +} + +func (p *parser) parseValueSpec(doc *ast.CommentGroup, _ token.Pos, keyword token.Token, iota int) ast.Spec { + if p.trace { + defer un(trace(p, keyword.String()+"Spec")) + } + + pos := p.pos + idents := p.parseIdentList() + typ := p.tryIdentOrType() + var values []ast.Expr + // always permit optional initialization for more tolerant parsing + if p.tok == token.ASSIGN { + p.next() + values = p.parseList(true) + } + p.expectSemi() // call before accessing p.linecomment + + switch keyword { + case token.VAR: + if typ == nil && values == nil { + p.error(pos, "missing variable type or initialization") + } + case token.CONST: + if values == nil && (iota == 0 || typ != nil) { + p.error(pos, "missing constant value") + } + } + + spec := &ast.ValueSpec{ + Doc: doc, + Names: idents, + Type: typ, + Values: values, + Comment: p.lineComment, + } + return spec +} + +func (p *parser) parseGenericType(spec *ast.TypeSpec, openPos token.Pos, name0 *ast.Ident, closeTok token.Token) { + list := p.parseParameterList(name0, closeTok, p.parseParamDecl, true) + closePos := p.expect(closeTok) + typeparams.Set(spec, &ast.FieldList{Opening: openPos, List: list, Closing: closePos}) + // Type alias cannot have type parameters. Accept them for robustness but complain. + if p.tok == token.ASSIGN { + p.error(p.pos, "generic type cannot be alias") + p.next() + } + spec.Type = p.parseType() +} + +func (p *parser) parseTypeSpec(doc *ast.CommentGroup, _ token.Pos, _ token.Token, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "TypeSpec")) + } + + ident := p.parseIdent() + spec := &ast.TypeSpec{Doc: doc, Name: ident} + + switch p.tok { + case token.LBRACK: + lbrack := p.pos + p.next() + if p.tok == token.IDENT { + // array type or generic type [T any] + p.exprLev++ + x := p.parseExpr() + p.exprLev-- + if name0, _ := x.(*ast.Ident); p.parseTypeParams() && name0 != nil && p.tok != token.RBRACK { + // generic type [T any]; + p.parseGenericType(spec, lbrack, name0, token.RBRACK) + } else { + // array type + // TODO(rfindley) should resolve all identifiers in x. + p.expect(token.RBRACK) + elt := p.parseType() + spec.Type = &ast.ArrayType{Lbrack: lbrack, Len: x, Elt: elt} + } + } else { + // array type + alen := p.parseArrayLen() + p.expect(token.RBRACK) + elt := p.parseType() + spec.Type = &ast.ArrayType{Lbrack: lbrack, Len: alen, Elt: elt} + } + + default: + // no type parameters + if p.tok == token.ASSIGN { + // type alias + spec.Assign = p.pos + p.next() + } + spec.Type = p.parseType() + } + + p.expectSemi() // call before accessing p.linecomment + spec.Comment = p.lineComment + + return spec +} + +func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl { + if p.trace { + defer un(trace(p, "GenDecl("+keyword.String()+")")) + } + + doc := p.leadComment + pos := p.expect(keyword) + var lparen, rparen token.Pos + var list []ast.Spec + if p.tok == token.LPAREN { + lparen = p.pos + p.next() + for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ { + list = append(list, f(p.leadComment, pos, keyword, iota)) + } + rparen = p.expect(token.RPAREN) + p.expectSemi() + } else { + list = append(list, f(nil, pos, keyword, 0)) + } + + return &ast.GenDecl{ + Doc: doc, + TokPos: pos, + Tok: keyword, + Lparen: lparen, + Specs: list, + Rparen: rparen, + } +} + +func (p *parser) parseFuncDecl() *ast.FuncDecl { + if p.trace { + defer un(trace(p, "FunctionDecl")) + } + + doc := p.leadComment + pos := p.expect(token.FUNC) + + var recv *ast.FieldList + if p.tok == token.LPAREN { + _, recv = p.parseParameters(false) + } + + ident := p.parseIdent() + + tparams, params := p.parseParameters(true) + results := p.parseResult() + + var body *ast.BlockStmt + if p.tok == token.LBRACE { + body = p.parseBody() + p.expectSemi() + } else if p.tok == token.SEMICOLON { + p.next() + if p.tok == token.LBRACE { + // opening { of function declaration on next line + p.error(p.pos, "unexpected semicolon or newline before {") + body = p.parseBody() + p.expectSemi() + } + } else { + p.expectSemi() + } + + decl := &ast.FuncDecl{ + Doc: doc, + Recv: recv, + Name: ident, + Type: &ast.FuncType{ + Func: pos, + Params: params, + Results: results, + }, + Body: body, + } + typeparams.Set(decl.Type, tparams) + return decl +} + +func (p *parser) parseDecl(sync map[token.Token]bool) ast.Decl { + if p.trace { + defer un(trace(p, "Declaration")) + } + + var f parseSpecFunction + switch p.tok { + case token.CONST, token.VAR: + f = p.parseValueSpec + + case token.TYPE: + f = p.parseTypeSpec + + case token.FUNC: + return p.parseFuncDecl() + + default: + pos := p.pos + p.errorExpected(pos, "declaration") + p.advance(sync) + return &ast.BadDecl{From: pos, To: p.pos} + } + + return p.parseGenDecl(p.tok, f) +} + +// ---------------------------------------------------------------------------- +// Source files + +func (p *parser) parseFile() *ast.File { + if p.trace { + defer un(trace(p, "File")) + } + + // Don't bother parsing the rest if we had errors scanning the first token. + // Likely not a Go source file at all. + if p.errors.Len() != 0 { + return nil + } + + // package clause + doc := p.leadComment + pos := p.expect(token.PACKAGE) + // Go spec: The package clause is not a declaration; + // the package name does not appear in any scope. + ident := p.parseIdent() + if ident.Name == "_" && p.mode&DeclarationErrors != 0 { + p.error(p.pos, "invalid package name _") + } + p.expectSemi() + + // Don't bother parsing the rest if we had errors parsing the package clause. + // Likely not a Go source file at all. + if p.errors.Len() != 0 { + return nil + } + + var decls []ast.Decl + if p.mode&PackageClauseOnly == 0 { + // import decls + for p.tok == token.IMPORT { + decls = append(decls, p.parseGenDecl(token.IMPORT, p.parseImportSpec)) + } + + if p.mode&ImportsOnly == 0 { + // rest of package body + for p.tok != token.EOF { + decls = append(decls, p.parseDecl(declStart)) + } + } + } + + f := &ast.File{ + Doc: doc, + Package: pos, + Name: ident, + Decls: decls, + Imports: p.imports, + Comments: p.comments, + } + var declErr func(token.Pos, string) + if p.mode&DeclarationErrors != 0 { + declErr = p.error + } + if p.mode&SkipObjectResolution == 0 { + resolveFile(f, p.file, declErr) + } + + return f +} diff --git a/src/go/parser/parser_test.go b/src/go/parser/parser_test.go new file mode 100644 index 0000000..1a46c87 --- /dev/null +++ b/src/go/parser/parser_test.go @@ -0,0 +1,748 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "bytes" + "fmt" + "go/ast" + "go/token" + "io/fs" + "runtime" + "strings" + "testing" +) + +var validFiles = []string{ + "parser.go", + "parser_test.go", + "error_test.go", + "short_test.go", +} + +func TestParse(t *testing.T) { + for _, filename := range validFiles { + _, err := ParseFile(token.NewFileSet(), filename, nil, DeclarationErrors) + if err != nil { + t.Fatalf("ParseFile(%s): %v", filename, err) + } + } +} + +func nameFilter(filename string) bool { + switch filename { + case "parser.go", "interface.go", "parser_test.go": + return true + case "parser.go.orig": + return true // permit but should be ignored by ParseDir + } + return false +} + +func dirFilter(f fs.FileInfo) bool { return nameFilter(f.Name()) } + +func TestParseFile(t *testing.T) { + src := "package p\nvar _=s[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]" + _, err := ParseFile(token.NewFileSet(), "", src, 0) + if err == nil { + t.Errorf("ParseFile(%s) succeeded unexpectedly", src) + } +} + +func TestParseExprFrom(t *testing.T) { + src := "s[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]" + _, err := ParseExprFrom(token.NewFileSet(), "", src, 0) + if err == nil { + t.Errorf("ParseExprFrom(%s) succeeded unexpectedly", src) + } +} + +func TestParseDir(t *testing.T) { + path := "." + pkgs, err := ParseDir(token.NewFileSet(), path, dirFilter, 0) + if err != nil { + t.Fatalf("ParseDir(%s): %v", path, err) + } + if n := len(pkgs); n != 1 { + t.Errorf("got %d packages; want 1", n) + } + pkg := pkgs["parser"] + if pkg == nil { + t.Errorf(`package "parser" not found`) + return + } + if n := len(pkg.Files); n != 3 { + t.Errorf("got %d package files; want 3", n) + } + for filename := range pkg.Files { + if !nameFilter(filename) { + t.Errorf("unexpected package file: %s", filename) + } + } +} + +func TestIssue42951(t *testing.T) { + path := "./testdata/issue42951" + _, err := ParseDir(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Errorf("ParseDir(%s): %v", path, err) + } +} + +func TestParseExpr(t *testing.T) { + // just kicking the tires: + // a valid arithmetic expression + src := "a + b" + x, err := ParseExpr(src) + if err != nil { + t.Errorf("ParseExpr(%q): %v", src, err) + } + // sanity check + if _, ok := x.(*ast.BinaryExpr); !ok { + t.Errorf("ParseExpr(%q): got %T, want *ast.BinaryExpr", src, x) + } + + // a valid type expression + src = "struct{x *int}" + x, err = ParseExpr(src) + if err != nil { + t.Errorf("ParseExpr(%q): %v", src, err) + } + // sanity check + if _, ok := x.(*ast.StructType); !ok { + t.Errorf("ParseExpr(%q): got %T, want *ast.StructType", src, x) + } + + // an invalid expression + src = "a + *" + x, err = ParseExpr(src) + if err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + if x == nil { + t.Errorf("ParseExpr(%q): got no (partial) result", src) + } + if _, ok := x.(*ast.BinaryExpr); !ok { + t.Errorf("ParseExpr(%q): got %T, want *ast.BinaryExpr", src, x) + } + + // a valid expression followed by extra tokens is invalid + src = "a[i] := x" + if _, err := ParseExpr(src); err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + + // a semicolon is not permitted unless automatically inserted + src = "a + b\n" + if _, err := ParseExpr(src); err != nil { + t.Errorf("ParseExpr(%q): got error %s", src, err) + } + src = "a + b;" + if _, err := ParseExpr(src); err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + + // various other stuff following a valid expression + const validExpr = "a + b" + const anything = "dh3*#D)#_" + for _, c := range "!)]};," { + src := validExpr + string(c) + anything + if _, err := ParseExpr(src); err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + } + + // ParseExpr must not crash + for _, src := range valids { + ParseExpr(src) + } +} + +func TestColonEqualsScope(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", `package p; func f() { x, y, z := x, y, z }`, 0) + if err != nil { + t.Fatal(err) + } + + // RHS refers to undefined globals; LHS does not. + as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.AssignStmt) + for _, v := range as.Rhs { + id := v.(*ast.Ident) + if id.Obj != nil { + t.Errorf("rhs %s has Obj, should not", id.Name) + } + } + for _, v := range as.Lhs { + id := v.(*ast.Ident) + if id.Obj == nil { + t.Errorf("lhs %s does not have Obj, should", id.Name) + } + } +} + +func TestVarScope(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", `package p; func f() { var x, y, z = x, y, z }`, 0) + if err != nil { + t.Fatal(err) + } + + // RHS refers to undefined globals; LHS does not. + as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.DeclStmt).Decl.(*ast.GenDecl).Specs[0].(*ast.ValueSpec) + for _, v := range as.Values { + id := v.(*ast.Ident) + if id.Obj != nil { + t.Errorf("rhs %s has Obj, should not", id.Name) + } + } + for _, id := range as.Names { + if id.Obj == nil { + t.Errorf("lhs %s does not have Obj, should", id.Name) + } + } +} + +func TestObjects(t *testing.T) { + const src = ` +package p +import fmt "fmt" +const pi = 3.14 +type T struct{} +var x int +func f() { L: } +` + + f, err := ParseFile(token.NewFileSet(), "", src, 0) + if err != nil { + t.Fatal(err) + } + + objects := map[string]ast.ObjKind{ + "p": ast.Bad, // not in a scope + "fmt": ast.Bad, // not resolved yet + "pi": ast.Con, + "T": ast.Typ, + "x": ast.Var, + "int": ast.Bad, // not resolved yet + "f": ast.Fun, + "L": ast.Lbl, + } + + ast.Inspect(f, func(n ast.Node) bool { + if ident, ok := n.(*ast.Ident); ok { + obj := ident.Obj + if obj == nil { + if objects[ident.Name] != ast.Bad { + t.Errorf("no object for %s", ident.Name) + } + return true + } + if obj.Name != ident.Name { + t.Errorf("names don't match: obj.Name = %s, ident.Name = %s", obj.Name, ident.Name) + } + kind := objects[ident.Name] + if obj.Kind != kind { + t.Errorf("%s: obj.Kind = %s; want %s", ident.Name, obj.Kind, kind) + } + } + return true + }) +} + +func TestUnresolved(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", ` +package p +// +func f1a(int) +func f2a(byte, int, float) +func f3a(a, b int, c float) +func f4a(...complex) +func f5a(a s1a, b ...complex) +// +func f1b(*int) +func f2b([]byte, (int), *float) +func f3b(a, b *int, c []float) +func f4b(...*complex) +func f5b(a s1a, b ...[]complex) +// +type s1a struct { int } +type s2a struct { byte; int; s1a } +type s3a struct { a, b int; c float } +// +type s1b struct { *int } +type s2b struct { byte; int; *float } +type s3b struct { a, b *s3b; c []float } +`, 0) + if err != nil { + t.Fatal(err) + } + + want := "int " + // f1a + "byte int float " + // f2a + "int float " + // f3a + "complex " + // f4a + "complex " + // f5a + // + "int " + // f1b + "byte int float " + // f2b + "int float " + // f3b + "complex " + // f4b + "complex " + // f5b + // + "int " + // s1a + "byte int " + // s2a + "int float " + // s3a + // + "int " + // s1a + "byte int float " + // s2a + "float " // s3a + + // collect unresolved identifiers + var buf bytes.Buffer + for _, u := range f.Unresolved { + buf.WriteString(u.Name) + buf.WriteByte(' ') + } + got := buf.String() + + if got != want { + t.Errorf("\ngot: %s\nwant: %s", got, want) + } +} + +var imports = map[string]bool{ + `"a"`: true, + "`a`": true, + `"a/b"`: true, + `"a.b"`: true, + `"m\x61th"`: true, + `"greek/αβ"`: true, + `""`: false, + + // Each of these pairs tests both `` vs "" strings + // and also use of invalid characters spelled out as + // escape sequences and written directly. + // For example `"\x00"` tests import "\x00" + // while "`\x00`" tests import `<actual-NUL-byte>`. + `"\x00"`: false, + "`\x00`": false, + `"\x7f"`: false, + "`\x7f`": false, + `"a!"`: false, + "`a!`": false, + `"a b"`: false, + "`a b`": false, + `"a\\b"`: false, + "`a\\b`": false, + "\"`a`\"": false, + "`\"a\"`": false, + `"\x80\x80"`: false, + "`\x80\x80`": false, + `"\xFFFD"`: false, + "`\xFFFD`": false, +} + +func TestImports(t *testing.T) { + for path, isValid := range imports { + src := fmt.Sprintf("package p; import %s", path) + _, err := ParseFile(token.NewFileSet(), "", src, 0) + switch { + case err != nil && isValid: + t.Errorf("ParseFile(%s): got %v; expected no error", src, err) + case err == nil && !isValid: + t.Errorf("ParseFile(%s): got no error; expected one", src) + } + } +} + +func TestCommentGroups(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", ` +package p /* 1a */ /* 1b */ /* 1c */ // 1d +/* 2a +*/ +// 2b +const pi = 3.1415 +/* 3a */ // 3b +/* 3c */ const e = 2.7182 + +// Example from issue 3139 +func ExampleCount() { + fmt.Println(strings.Count("cheese", "e")) + fmt.Println(strings.Count("five", "")) // before & after each rune + // Output: + // 3 + // 5 +} +`, ParseComments) + if err != nil { + t.Fatal(err) + } + expected := [][]string{ + {"/* 1a */", "/* 1b */", "/* 1c */", "// 1d"}, + {"/* 2a\n*/", "// 2b"}, + {"/* 3a */", "// 3b", "/* 3c */"}, + {"// Example from issue 3139"}, + {"// before & after each rune"}, + {"// Output:", "// 3", "// 5"}, + } + if len(f.Comments) != len(expected) { + t.Fatalf("got %d comment groups; expected %d", len(f.Comments), len(expected)) + } + for i, exp := range expected { + got := f.Comments[i].List + if len(got) != len(exp) { + t.Errorf("got %d comments in group %d; expected %d", len(got), i, len(exp)) + continue + } + for j, exp := range exp { + got := got[j].Text + if got != exp { + t.Errorf("got %q in group %d; expected %q", got, i, exp) + } + } + } +} + +func getField(file *ast.File, fieldname string) *ast.Field { + parts := strings.Split(fieldname, ".") + for _, d := range file.Decls { + if d, ok := d.(*ast.GenDecl); ok && d.Tok == token.TYPE { + for _, s := range d.Specs { + if s, ok := s.(*ast.TypeSpec); ok && s.Name.Name == parts[0] { + if s, ok := s.Type.(*ast.StructType); ok { + for _, f := range s.Fields.List { + for _, name := range f.Names { + if name.Name == parts[1] { + return f + } + } + } + } + } + } + } + } + return nil +} + +// Don't use ast.CommentGroup.Text() - we want to see exact comment text. +func commentText(c *ast.CommentGroup) string { + var buf bytes.Buffer + if c != nil { + for _, c := range c.List { + buf.WriteString(c.Text) + } + } + return buf.String() +} + +func checkFieldComments(t *testing.T, file *ast.File, fieldname, lead, line string) { + f := getField(file, fieldname) + if f == nil { + t.Fatalf("field not found: %s", fieldname) + } + if got := commentText(f.Doc); got != lead { + t.Errorf("got lead comment %q; expected %q", got, lead) + } + if got := commentText(f.Comment); got != line { + t.Errorf("got line comment %q; expected %q", got, line) + } +} + +func TestLeadAndLineComments(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", ` +package p +type T struct { + /* F1 lead comment */ + // + F1 int /* F1 */ // line comment + // F2 lead + // comment + F2 int // F2 line comment + // f3 lead comment + f3 int // f3 line comment +} +`, ParseComments) + if err != nil { + t.Fatal(err) + } + checkFieldComments(t, f, "T.F1", "/* F1 lead comment *///", "/* F1 */// line comment") + checkFieldComments(t, f, "T.F2", "// F2 lead// comment", "// F2 line comment") + checkFieldComments(t, f, "T.f3", "// f3 lead comment", "// f3 line comment") + ast.FileExports(f) + checkFieldComments(t, f, "T.F1", "/* F1 lead comment *///", "/* F1 */// line comment") + checkFieldComments(t, f, "T.F2", "// F2 lead// comment", "// F2 line comment") + if getField(f, "T.f3") != nil { + t.Error("not expected to find T.f3") + } +} + +// TestIssue9979 verifies that empty statements are contained within their enclosing blocks. +func TestIssue9979(t *testing.T) { + for _, src := range []string{ + "package p; func f() {;}", + "package p; func f() {L:}", + "package p; func f() {L:;}", + "package p; func f() {L:\n}", + "package p; func f() {L:\n;}", + "package p; func f() { ; }", + "package p; func f() { L: }", + "package p; func f() { L: ; }", + "package p; func f() { L: \n}", + "package p; func f() { L: \n; }", + } { + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, 0) + if err != nil { + t.Fatal(err) + } + + var pos, end token.Pos + ast.Inspect(f, func(x ast.Node) bool { + switch s := x.(type) { + case *ast.BlockStmt: + pos, end = s.Pos()+1, s.End()-1 // exclude "{", "}" + case *ast.LabeledStmt: + pos, end = s.Pos()+2, s.End() // exclude "L:" + case *ast.EmptyStmt: + // check containment + if s.Pos() < pos || s.End() > end { + t.Errorf("%s: %T[%d, %d] not inside [%d, %d]", src, s, s.Pos(), s.End(), pos, end) + } + // check semicolon + offs := fset.Position(s.Pos()).Offset + if ch := src[offs]; ch != ';' != s.Implicit { + want := "want ';'" + if s.Implicit { + want = "but ';' is implicit" + } + t.Errorf("%s: found %q at offset %d; %s", src, ch, offs, want) + } + } + return true + }) + } +} + +// TestIncompleteSelection ensures that an incomplete selector +// expression is parsed as a (blank) *ast.SelectorExpr, not a +// *ast.BadExpr. +func TestIncompleteSelection(t *testing.T) { + for _, src := range []string{ + "package p; var _ = fmt.", // at EOF + "package p; var _ = fmt.\ntype X int", // not at EOF + } { + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, 0) + if err == nil { + t.Errorf("ParseFile(%s) succeeded unexpectedly", src) + continue + } + + const wantErr = "expected selector or type assertion" + if !strings.Contains(err.Error(), wantErr) { + t.Errorf("ParseFile returned wrong error %q, want %q", err, wantErr) + } + + var sel *ast.SelectorExpr + ast.Inspect(f, func(n ast.Node) bool { + if n, ok := n.(*ast.SelectorExpr); ok { + sel = n + } + return true + }) + if sel == nil { + t.Error("found no *ast.SelectorExpr") + continue + } + const wantSel = "&{fmt _}" + if fmt.Sprint(sel) != wantSel { + t.Errorf("found selector %s, want %s", sel, wantSel) + continue + } + } +} + +func TestLastLineComment(t *testing.T) { + const src = `package main +type x int // comment +` + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, ParseComments) + if err != nil { + t.Fatal(err) + } + comment := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec).Comment.List[0].Text + if comment != "// comment" { + t.Errorf("got %q, want %q", comment, "// comment") + } +} + +var parseDepthTests = []struct { + name string + format string + // multipler is used when a single statement may result in more than one + // change in the depth level, for instance "1+(..." produces a BinaryExpr + // followed by a UnaryExpr, which increments the depth twice. The test + // case comment explains which nodes are triggering the multiple depth + // changes. + parseMultiplier int + // scope is true if we should also test the statement for the resolver scope + // depth limit. + scope bool + // scopeMultiplier does the same as parseMultiplier, but for the scope + // depths. + scopeMultiplier int +}{ + // The format expands the part inside « » many times. + // A second set of brackets nested inside the first stops the repetition, + // so that for example «(«1»)» expands to (((...((((1))))...))). + {name: "array", format: "package main; var x «[1]»int"}, + {name: "slice", format: "package main; var x «[]»int"}, + {name: "struct", format: "package main; var x «struct { X «int» }»", scope: true}, + {name: "pointer", format: "package main; var x «*»int"}, + {name: "func", format: "package main; var x «func()»int", scope: true}, + {name: "chan", format: "package main; var x «chan »int"}, + {name: "chan2", format: "package main; var x «<-chan »int"}, + {name: "interface", format: "package main; var x «interface { M() «int» }»", scope: true, scopeMultiplier: 2}, // Scopes: InterfaceType, FuncType + {name: "map", format: "package main; var x «map[int]»int"}, + {name: "slicelit", format: "package main; var x = «[]any{«»}»", parseMultiplier: 2}, // Parser nodes: UnaryExpr, CompositeLit + {name: "arraylit", format: "package main; var x = «[1]any{«nil»}»", parseMultiplier: 2}, // Parser nodes: UnaryExpr, CompositeLit + {name: "structlit", format: "package main; var x = «struct{x any}{«nil»}»", parseMultiplier: 2}, // Parser nodes: UnaryExpr, CompositeLit + {name: "maplit", format: "package main; var x = «map[int]any{1:«nil»}»", parseMultiplier: 2}, // Parser nodes: CompositeLit, KeyValueExpr + {name: "dot", format: "package main; var x = «x.»x"}, + {name: "index", format: "package main; var x = x«[1]»"}, + {name: "slice", format: "package main; var x = x«[1:2]»"}, + {name: "slice3", format: "package main; var x = x«[1:2:3]»"}, + {name: "dottype", format: "package main; var x = x«.(any)»"}, + {name: "callseq", format: "package main; var x = x«()»"}, + {name: "methseq", format: "package main; var x = x«.m()»", parseMultiplier: 2}, // Parser nodes: SelectorExpr, CallExpr + {name: "binary", format: "package main; var x = «1+»1"}, + {name: "binaryparen", format: "package main; var x = «1+(«1»)»", parseMultiplier: 2}, // Parser nodes: BinaryExpr, ParenExpr + {name: "unary", format: "package main; var x = «^»1"}, + {name: "addr", format: "package main; var x = «& »x"}, + {name: "star", format: "package main; var x = «*»x"}, + {name: "recv", format: "package main; var x = «<-»x"}, + {name: "call", format: "package main; var x = «f(«1»)»", parseMultiplier: 2}, // Parser nodes: Ident, CallExpr + {name: "conv", format: "package main; var x = «(*T)(«1»)»", parseMultiplier: 2}, // Parser nodes: ParenExpr, CallExpr + {name: "label", format: "package main; func main() { «Label:» }"}, + {name: "if", format: "package main; func main() { «if true { «» }»}", parseMultiplier: 2, scope: true, scopeMultiplier: 2}, // Parser nodes: IfStmt, BlockStmt. Scopes: IfStmt, BlockStmt + {name: "ifelse", format: "package main; func main() { «if true {} else » {} }", scope: true}, + {name: "switch", format: "package main; func main() { «switch { default: «» }»}", scope: true, scopeMultiplier: 2}, // Scopes: TypeSwitchStmt, CaseClause + {name: "typeswitch", format: "package main; func main() { «switch x.(type) { default: «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: TypeSwitchStmt, CaseClause + {name: "for0", format: "package main; func main() { «for { «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: ForStmt, BlockStmt + {name: "for1", format: "package main; func main() { «for x { «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: ForStmt, BlockStmt + {name: "for3", format: "package main; func main() { «for f(); g(); h() { «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: ForStmt, BlockStmt + {name: "forrange0", format: "package main; func main() { «for range x { «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: RangeStmt, BlockStmt + {name: "forrange1", format: "package main; func main() { «for x = range z { «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: RangeStmt, BlockStmt + {name: "forrange2", format: "package main; func main() { «for x, y = range z { «» }» }", scope: true, scopeMultiplier: 2}, // Scopes: RangeStmt, BlockStmt + {name: "go", format: "package main; func main() { «go func() { «» }()» }", parseMultiplier: 2, scope: true}, // Parser nodes: GoStmt, FuncLit + {name: "defer", format: "package main; func main() { «defer func() { «» }()» }", parseMultiplier: 2, scope: true}, // Parser nodes: DeferStmt, FuncLit + {name: "select", format: "package main; func main() { «select { default: «» }» }", scope: true}, +} + +// split splits pre«mid»post into pre, mid, post. +// If the string does not have that form, split returns x, "", "". +func split(x string) (pre, mid, post string) { + start, end := strings.Index(x, "«"), strings.LastIndex(x, "»") + if start < 0 || end < 0 { + return x, "", "" + } + return x[:start], x[start+len("«") : end], x[end+len("»"):] +} + +func TestParseDepthLimit(t *testing.T) { + if runtime.GOARCH == "wasm" { + t.Skip("causes call stack exhaustion on js/wasm") + } + for _, tt := range parseDepthTests { + for _, size := range []string{"small", "big"} { + t.Run(tt.name+"/"+size, func(t *testing.T) { + n := maxNestLev + 1 + if tt.parseMultiplier > 0 { + n /= tt.parseMultiplier + } + if size == "small" { + // Decrease the number of statements by 10, in order to check + // that we do not fail when under the limit. 10 is used to + // provide some wiggle room for cases where the surrounding + // scaffolding syntax adds some noise to the depth that changes + // on a per testcase basis. + n -= 10 + } + + pre, mid, post := split(tt.format) + if strings.Contains(mid, "«") { + left, base, right := split(mid) + mid = strings.Repeat(left, n) + base + strings.Repeat(right, n) + } else { + mid = strings.Repeat(mid, n) + } + input := pre + mid + post + + fset := token.NewFileSet() + _, err := ParseFile(fset, "", input, ParseComments|SkipObjectResolution) + if size == "small" { + if err != nil { + t.Errorf("ParseFile(...): %v (want success)", err) + } + } else { + expected := "exceeded max nesting depth" + if err == nil || !strings.HasSuffix(err.Error(), expected) { + t.Errorf("ParseFile(...) = _, %v, want %q", err, expected) + } + } + }) + } + } +} + +func TestScopeDepthLimit(t *testing.T) { + if runtime.GOARCH == "wasm" { + t.Skip("causes call stack exhaustion on js/wasm") + } + for _, tt := range parseDepthTests { + if !tt.scope { + continue + } + for _, size := range []string{"small", "big"} { + t.Run(tt.name+"/"+size, func(t *testing.T) { + n := maxScopeDepth + 1 + if tt.scopeMultiplier > 0 { + n /= tt.scopeMultiplier + } + if size == "small" { + // Decrease the number of statements by 10, in order to check + // that we do not fail when under the limit. 10 is used to + // provide some wiggle room for cases where the surrounding + // scaffolding syntax adds some noise to the depth that changes + // on a per testcase basis. + n -= 10 + } + + pre, mid, post := split(tt.format) + if strings.Contains(mid, "«") { + left, base, right := split(mid) + mid = strings.Repeat(left, n) + base + strings.Repeat(right, n) + } else { + mid = strings.Repeat(mid, n) + } + input := pre + mid + post + + fset := token.NewFileSet() + _, err := ParseFile(fset, "", input, DeclarationErrors) + if size == "small" { + if err != nil { + t.Errorf("ParseFile(...): %v (want success)", err) + } + } else { + expected := "exceeded max scope depth during object resolution" + if err == nil || !strings.HasSuffix(err.Error(), expected) { + t.Errorf("ParseFile(...) = _, %v, want %q", err, expected) + } + } + }) + } + } +} diff --git a/src/go/parser/performance_test.go b/src/go/parser/performance_test.go new file mode 100644 index 0000000..6f8a977 --- /dev/null +++ b/src/go/parser/performance_test.go @@ -0,0 +1,56 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "go/token" + "os" + "testing" +) + +// TODO(rFindley): use a testdata file or file from another package here, to +// avoid a moving target. +var src = readFile("parser.go") + +func readFile(filename string) []byte { + data, err := os.ReadFile(filename) + if err != nil { + panic(err) + } + return data +} + +func BenchmarkParse(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + if _, err := ParseFile(token.NewFileSet(), "", src, ParseComments); err != nil { + b.Fatalf("benchmark failed due to parse error: %s", err) + } + } +} + +func BenchmarkParseOnly(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + if _, err := ParseFile(token.NewFileSet(), "", src, ParseComments|SkipObjectResolution); err != nil { + b.Fatalf("benchmark failed due to parse error: %s", err) + } + } +} + +func BenchmarkResolve(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + b.StopTimer() + fset := token.NewFileSet() + file, err := ParseFile(fset, "", src, SkipObjectResolution) + if err != nil { + b.Fatalf("benchmark failed due to parse error: %s", err) + } + b.StartTimer() + handle := fset.File(file.Package) + resolveFile(file, handle, nil) + } +} diff --git a/src/go/parser/resolver.go b/src/go/parser/resolver.go new file mode 100644 index 0000000..f55bdb7 --- /dev/null +++ b/src/go/parser/resolver.go @@ -0,0 +1,565 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "fmt" + "go/ast" + "go/internal/typeparams" + "go/token" +) + +const debugResolve = false + +// resolveFile walks the given file to resolve identifiers within the file +// scope, updating ast.Ident.Obj fields with declaration information. +// +// If declErr is non-nil, it is used to report declaration errors during +// resolution. tok is used to format position in error messages. +func resolveFile(file *ast.File, handle *token.File, declErr func(token.Pos, string)) { + pkgScope := ast.NewScope(nil) + r := &resolver{ + handle: handle, + declErr: declErr, + topScope: pkgScope, + pkgScope: pkgScope, + depth: 1, + } + + for _, decl := range file.Decls { + ast.Walk(r, decl) + } + + r.closeScope() + assert(r.topScope == nil, "unbalanced scopes") + assert(r.labelScope == nil, "unbalanced label scopes") + + // resolve global identifiers within the same file + i := 0 + for _, ident := range r.unresolved { + // i <= index for current ident + assert(ident.Obj == unresolved, "object already resolved") + ident.Obj = r.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel + if ident.Obj == nil { + r.unresolved[i] = ident + i++ + } else if debugResolve { + pos := ident.Obj.Decl.(interface{ Pos() token.Pos }).Pos() + r.dump("resolved %s@%v to package object %v", ident.Name, ident.Pos(), pos) + } + } + file.Scope = r.pkgScope + file.Unresolved = r.unresolved[0:i] +} + +const maxScopeDepth int = 1e3 + +type resolver struct { + handle *token.File + declErr func(token.Pos, string) + + // Ordinary identifier scopes + pkgScope *ast.Scope // pkgScope.Outer == nil + topScope *ast.Scope // top-most scope; may be pkgScope + unresolved []*ast.Ident // unresolved identifiers + depth int // scope depth + + // Label scopes + // (maintained by open/close LabelScope) + labelScope *ast.Scope // label scope for current function + targetStack [][]*ast.Ident // stack of unresolved labels +} + +func (r *resolver) dump(format string, args ...interface{}) { + fmt.Println(">>> " + r.sprintf(format, args...)) +} + +func (r *resolver) sprintf(format string, args ...interface{}) string { + for i, arg := range args { + switch arg := arg.(type) { + case token.Pos: + args[i] = r.handle.Position(arg) + } + } + return fmt.Sprintf(format, args...) +} + +func (r *resolver) openScope(pos token.Pos) { + r.depth++ + if r.depth > maxScopeDepth { + panic(bailout{pos: pos, msg: "exceeded max scope depth during object resolution"}) + } + if debugResolve { + r.dump("opening scope @%v", pos) + } + r.topScope = ast.NewScope(r.topScope) +} + +func (r *resolver) closeScope() { + r.depth-- + if debugResolve { + r.dump("closing scope") + } + r.topScope = r.topScope.Outer +} + +func (r *resolver) openLabelScope() { + r.labelScope = ast.NewScope(r.labelScope) + r.targetStack = append(r.targetStack, nil) +} + +func (r *resolver) closeLabelScope() { + // resolve labels + n := len(r.targetStack) - 1 + scope := r.labelScope + for _, ident := range r.targetStack[n] { + ident.Obj = scope.Lookup(ident.Name) + if ident.Obj == nil && r.declErr != nil { + r.declErr(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name)) + } + } + // pop label scope + r.targetStack = r.targetStack[0:n] + r.labelScope = r.labelScope.Outer +} + +func (r *resolver) declare(decl, data interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) { + for _, ident := range idents { + // "type" is used for type lists in interfaces, and is otherwise an invalid + // identifier. The 'type' identifier is also artificially duplicated in the + // type list, so could cause panics below if we were to proceed. + if ident.Name == "type" { + continue + } + assert(ident.Obj == nil, "identifier already declared or resolved") + obj := ast.NewObj(kind, ident.Name) + // remember the corresponding declaration for redeclaration + // errors and global variable resolution/typechecking phase + obj.Decl = decl + obj.Data = data + ident.Obj = obj + if ident.Name != "_" { + if debugResolve { + r.dump("declaring %s@%v", ident.Name, ident.Pos()) + } + if alt := scope.Insert(obj); alt != nil && r.declErr != nil { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", r.handle.Position(pos)) + } + r.declErr(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl)) + } + } + } +} + +func (r *resolver) shortVarDecl(decl *ast.AssignStmt) { + // Go spec: A short variable declaration may redeclare variables + // provided they were originally declared in the same block with + // the same type, and at least one of the non-blank variables is new. + n := 0 // number of new variables + for _, x := range decl.Lhs { + if ident, isIdent := x.(*ast.Ident); isIdent { + assert(ident.Obj == nil, "identifier already declared or resolved") + obj := ast.NewObj(ast.Var, ident.Name) + // remember corresponding assignment for other tools + obj.Decl = decl + ident.Obj = obj + if ident.Name != "_" { + if debugResolve { + r.dump("declaring %s@%v", ident.Name, ident.Pos()) + } + if alt := r.topScope.Insert(obj); alt != nil { + ident.Obj = alt // redeclaration + } else { + n++ // new declaration + } + } + } + } + if n == 0 && r.declErr != nil { + r.declErr(decl.Lhs[0].Pos(), "no new variables on left side of :=") + } +} + +// The unresolved object is a sentinel to mark identifiers that have been added +// to the list of unresolved identifiers. The sentinel is only used for verifying +// internal consistency. +var unresolved = new(ast.Object) + +// If x is an identifier, resolve attempts to resolve x by looking up +// the object it denotes. If no object is found and collectUnresolved is +// set, x is marked as unresolved and collected in the list of unresolved +// identifiers. +// +func (r *resolver) resolve(ident *ast.Ident, collectUnresolved bool) { + if ident.Obj != nil { + panic(fmt.Sprintf("%s: identifier %s already declared or resolved", r.handle.Position(ident.Pos()), ident.Name)) + } + // '_' and 'type' should never refer to existing declarations: '_' because it + // has special handling in the spec, and 'type' because it is a keyword, and + // only valid in an interface type list. + if ident.Name == "_" || ident.Name == "type" { + return + } + for s := r.topScope; s != nil; s = s.Outer { + if obj := s.Lookup(ident.Name); obj != nil { + assert(obj.Name != "", "obj with no name") + ident.Obj = obj + return + } + } + // all local scopes are known, so any unresolved identifier + // must be found either in the file scope, package scope + // (perhaps in another file), or universe scope --- collect + // them so that they can be resolved later + if collectUnresolved { + ident.Obj = unresolved + r.unresolved = append(r.unresolved, ident) + } +} + +func (r *resolver) walkExprs(list []ast.Expr) { + for _, node := range list { + ast.Walk(r, node) + } +} + +func (r *resolver) walkLHS(list []ast.Expr) { + for _, expr := range list { + expr := unparen(expr) + if _, ok := expr.(*ast.Ident); !ok && expr != nil { + ast.Walk(r, expr) + } + } +} + +func (r *resolver) walkStmts(list []ast.Stmt) { + for _, stmt := range list { + ast.Walk(r, stmt) + } +} + +func (r *resolver) Visit(node ast.Node) ast.Visitor { + if debugResolve && node != nil { + r.dump("node %T@%v", node, node.Pos()) + } + + switch n := node.(type) { + + // Expressions. + case *ast.Ident: + r.resolve(n, true) + + case *ast.FuncLit: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFuncType(n.Type) + r.walkBody(n.Body) + + case *ast.SelectorExpr: + ast.Walk(r, n.X) + // Note: don't try to resolve n.Sel, as we don't support qualified + // resolution. + + case *ast.StructType: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFieldList(n.Fields, ast.Var) + + case *ast.FuncType: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFuncType(n) + + case *ast.CompositeLit: + if n.Type != nil { + ast.Walk(r, n.Type) + } + for _, e := range n.Elts { + if kv, _ := e.(*ast.KeyValueExpr); kv != nil { + // See issue #45160: try to resolve composite lit keys, but don't + // collect them as unresolved if resolution failed. This replicates + // existing behavior when resolving during parsing. + if ident, _ := kv.Key.(*ast.Ident); ident != nil { + r.resolve(ident, false) + } else { + ast.Walk(r, kv.Key) + } + ast.Walk(r, kv.Value) + } else { + ast.Walk(r, e) + } + } + + case *ast.InterfaceType: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFieldList(n.Methods, ast.Fun) + + // Statements + case *ast.LabeledStmt: + r.declare(n, nil, r.labelScope, ast.Lbl, n.Label) + ast.Walk(r, n.Stmt) + + case *ast.AssignStmt: + r.walkExprs(n.Rhs) + if n.Tok == token.DEFINE { + r.shortVarDecl(n) + } else { + r.walkExprs(n.Lhs) + } + + case *ast.BranchStmt: + // add to list of unresolved targets + if n.Tok != token.FALLTHROUGH && n.Label != nil { + depth := len(r.targetStack) - 1 + r.targetStack[depth] = append(r.targetStack[depth], n.Label) + } + + case *ast.BlockStmt: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkStmts(n.List) + + case *ast.IfStmt: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Init != nil { + ast.Walk(r, n.Init) + } + ast.Walk(r, n.Cond) + ast.Walk(r, n.Body) + if n.Else != nil { + ast.Walk(r, n.Else) + } + + case *ast.CaseClause: + r.walkExprs(n.List) + r.openScope(n.Pos()) + defer r.closeScope() + r.walkStmts(n.Body) + + case *ast.SwitchStmt: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Init != nil { + ast.Walk(r, n.Init) + } + if n.Tag != nil { + // The scope below reproduces some unnecessary behavior of the parser, + // opening an extra scope in case this is a type switch. It's not needed + // for expression switches. + // TODO: remove this once we've matched the parser resolution exactly. + if n.Init != nil { + r.openScope(n.Tag.Pos()) + defer r.closeScope() + } + ast.Walk(r, n.Tag) + } + if n.Body != nil { + r.walkStmts(n.Body.List) + } + + case *ast.TypeSwitchStmt: + if n.Init != nil { + r.openScope(n.Pos()) + defer r.closeScope() + ast.Walk(r, n.Init) + } + r.openScope(n.Assign.Pos()) + defer r.closeScope() + ast.Walk(r, n.Assign) + // s.Body consists only of case clauses, so does not get its own + // scope. + if n.Body != nil { + r.walkStmts(n.Body.List) + } + + case *ast.CommClause: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Comm != nil { + ast.Walk(r, n.Comm) + } + r.walkStmts(n.Body) + + case *ast.SelectStmt: + // as for switch statements, select statement bodies don't get their own + // scope. + if n.Body != nil { + r.walkStmts(n.Body.List) + } + + case *ast.ForStmt: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Init != nil { + ast.Walk(r, n.Init) + } + if n.Cond != nil { + ast.Walk(r, n.Cond) + } + if n.Post != nil { + ast.Walk(r, n.Post) + } + ast.Walk(r, n.Body) + + case *ast.RangeStmt: + r.openScope(n.Pos()) + defer r.closeScope() + ast.Walk(r, n.X) + var lhs []ast.Expr + if n.Key != nil { + lhs = append(lhs, n.Key) + } + if n.Value != nil { + lhs = append(lhs, n.Value) + } + if len(lhs) > 0 { + if n.Tok == token.DEFINE { + // Note: we can't exactly match the behavior of object resolution + // during the parsing pass here, as it uses the position of the RANGE + // token for the RHS OpPos. That information is not contained within + // the AST. + as := &ast.AssignStmt{ + Lhs: lhs, + Tok: token.DEFINE, + TokPos: n.TokPos, + Rhs: []ast.Expr{&ast.UnaryExpr{Op: token.RANGE, X: n.X}}, + } + // TODO(rFindley): this walkLHS reproduced the parser resolution, but + // is it necessary? By comparison, for a normal AssignStmt we don't + // walk the LHS in case there is an invalid identifier list. + r.walkLHS(lhs) + r.shortVarDecl(as) + } else { + r.walkExprs(lhs) + } + } + ast.Walk(r, n.Body) + + // Declarations + case *ast.GenDecl: + switch n.Tok { + case token.CONST, token.VAR: + for i, spec := range n.Specs { + spec := spec.(*ast.ValueSpec) + kind := ast.Con + if n.Tok == token.VAR { + kind = ast.Var + } + r.walkExprs(spec.Values) + if spec.Type != nil { + ast.Walk(r, spec.Type) + } + r.declare(spec, i, r.topScope, kind, spec.Names...) + } + case token.TYPE: + for _, spec := range n.Specs { + spec := spec.(*ast.TypeSpec) + // Go spec: The scope of a type identifier declared inside a function begins + // at the identifier in the TypeSpec and ends at the end of the innermost + // containing block. + r.declare(spec, nil, r.topScope, ast.Typ, spec.Name) + if tparams := typeparams.Get(spec); tparams != nil { + r.openScope(spec.Pos()) + defer r.closeScope() + r.walkTParams(tparams) + } + ast.Walk(r, spec.Type) + } + } + + case *ast.FuncDecl: + // Open the function scope. + r.openScope(n.Pos()) + defer r.closeScope() + + // Resolve the receiver first, without declaring. + r.resolveList(n.Recv) + + // Type parameters are walked normally: they can reference each other, and + // can be referenced by normal parameters. + if tparams := typeparams.Get(n.Type); tparams != nil { + r.walkTParams(tparams) + // TODO(rFindley): need to address receiver type parameters. + } + + // Resolve and declare parameters in a specific order to get duplicate + // declaration errors in the correct location. + r.resolveList(n.Type.Params) + r.resolveList(n.Type.Results) + r.declareList(n.Recv, ast.Var) + r.declareList(n.Type.Params, ast.Var) + r.declareList(n.Type.Results, ast.Var) + + r.walkBody(n.Body) + if n.Recv == nil && n.Name.Name != "init" { + r.declare(n, nil, r.pkgScope, ast.Fun, n.Name) + } + + default: + return r + } + + return nil +} + +func (r *resolver) walkFuncType(typ *ast.FuncType) { + // typ.TParams must be walked separately for FuncDecls. + r.resolveList(typ.Params) + r.resolveList(typ.Results) + r.declareList(typ.Params, ast.Var) + r.declareList(typ.Results, ast.Var) +} + +func (r *resolver) resolveList(list *ast.FieldList) { + if list == nil { + return + } + for _, f := range list.List { + if f.Type != nil { + ast.Walk(r, f.Type) + } + } +} + +func (r *resolver) declareList(list *ast.FieldList, kind ast.ObjKind) { + if list == nil { + return + } + for _, f := range list.List { + r.declare(f, nil, r.topScope, kind, f.Names...) + } +} + +func (r *resolver) walkFieldList(list *ast.FieldList, kind ast.ObjKind) { + if list == nil { + return + } + r.resolveList(list) + r.declareList(list, kind) +} + +// walkTParams is like walkFieldList, but declares type parameters eagerly so +// that they may be resolved in the constraint expressions held in the field +// Type. +func (r *resolver) walkTParams(list *ast.FieldList) { + if list == nil { + return + } + r.declareList(list, ast.Typ) + r.resolveList(list) +} + +func (r *resolver) walkBody(body *ast.BlockStmt) { + if body == nil { + return + } + r.openLabelScope() + defer r.closeLabelScope() + r.walkStmts(body.List) +} diff --git a/src/go/parser/resolver_test.go b/src/go/parser/resolver_test.go new file mode 100644 index 0000000..625c009 --- /dev/null +++ b/src/go/parser/resolver_test.go @@ -0,0 +1,180 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "fmt" + "go/ast" + "go/internal/typeparams" + "go/scanner" + "go/token" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestResolution checks that identifiers are resolved to the declarations +// annotated in the source, by comparing the positions of the resulting +// Ident.Obj.Decl to positions marked in the source via special comments. +// +// In the test source, any comment prefixed with '=' or '@' (or both) marks the +// previous token position as the declaration ('=') or a use ('@') of an +// identifier. The text following '=' and '@' in the comment string is the +// label to use for the location. Declaration labels must be unique within the +// file, and use labels must refer to an existing declaration label. It's OK +// for a comment to denote both the declaration and use of a label (e.g. +// '=@foo'). Leading and trailing whitespace is ignored. Any comment not +// beginning with '=' or '@' is ignored. +func TestResolution(t *testing.T) { + dir := filepath.Join("testdata", "resolution") + fis, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + for _, fi := range fis { + t.Run(fi.Name(), func(t *testing.T) { + fset := token.NewFileSet() + path := filepath.Join(dir, fi.Name()) + src := readFile(path) // panics on failure + var mode Mode + if strings.HasSuffix(path, ".go2") { + if !typeparams.Enabled { + t.Skip("type params are not enabled") + } + } else { + mode |= typeparams.DisallowParsing + } + file, err := ParseFile(fset, path, src, mode) + if err != nil { + t.Fatal(err) + } + + // Compare the positions of objects resolved during parsing (fromParser) + // to those annotated in source comments (fromComments). + + handle := fset.File(file.Package) + fromParser := declsFromParser(file) + fromComments := declsFromComments(handle, src) + + pos := func(pos token.Pos) token.Position { + p := handle.Position(pos) + // The file name is implied by the subtest, so remove it to avoid + // clutter in error messages. + p.Filename = "" + return p + } + for k, want := range fromComments { + if got := fromParser[k]; got != want { + t.Errorf("%s resolved to %s, want %s", pos(k), pos(got), pos(want)) + } + delete(fromParser, k) + } + // What remains in fromParser are unexpected resolutions. + for k, got := range fromParser { + t.Errorf("%s resolved to %s, want no object", pos(k), pos(got)) + } + }) + } +} + +// declsFromParser walks the file and collects the map associating an +// identifier position with its declaration position. +func declsFromParser(file *ast.File) map[token.Pos]token.Pos { + objmap := map[token.Pos]token.Pos{} + ast.Inspect(file, func(node ast.Node) bool { + // Ignore blank identifiers to reduce noise. + if ident, _ := node.(*ast.Ident); ident != nil && ident.Obj != nil && ident.Name != "_" { + objmap[ident.Pos()] = ident.Obj.Pos() + } + return true + }) + return objmap +} + +// declsFromComments looks at comments annotating uses and declarations, and +// maps each identifier use to its corresponding declaration. See the +// description of these annotations in the documentation for TestResolution. +func declsFromComments(handle *token.File, src []byte) map[token.Pos]token.Pos { + decls, uses := positionMarkers(handle, src) + + objmap := make(map[token.Pos]token.Pos) + // Join decls and uses on name, to build the map of use->decl. + for name, posns := range uses { + declpos, ok := decls[name] + if !ok { + panic(fmt.Sprintf("missing declaration for %s", name)) + } + for _, pos := range posns { + objmap[pos] = declpos + } + } + return objmap +} + +// positionMarkers extracts named positions from the source denoted by comments +// prefixed with '=' (declarations) and '@' (uses): for example '@foo' or +// '=@bar'. It returns a map of name->position for declarations, and +// name->position(s) for uses. +func positionMarkers(handle *token.File, src []byte) (decls map[string]token.Pos, uses map[string][]token.Pos) { + var s scanner.Scanner + s.Init(handle, src, nil, scanner.ScanComments) + decls = make(map[string]token.Pos) + uses = make(map[string][]token.Pos) + var prev token.Pos // position of last non-comment, non-semicolon token + +scanFile: + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + break scanFile + case token.COMMENT: + name, decl, use := annotatedObj(lit) + if len(name) > 0 { + if decl { + if _, ok := decls[name]; ok { + panic(fmt.Sprintf("duplicate declaration markers for %s", name)) + } + decls[name] = prev + } + if use { + uses[name] = append(uses[name], prev) + } + } + case token.SEMICOLON: + // ignore automatically inserted semicolon + if lit == "\n" { + continue scanFile + } + fallthrough + default: + prev = pos + } + } + return decls, uses +} + +func annotatedObj(lit string) (name string, decl, use bool) { + if lit[1] == '*' { + lit = lit[:len(lit)-2] // strip trailing */ + } + lit = strings.TrimSpace(lit[2:]) + +scanLit: + for idx, r := range lit { + switch r { + case '=': + decl = true + case '@': + use = true + default: + name = lit[idx:] + break scanLit + } + } + return +} diff --git a/src/go/parser/short_test.go b/src/go/parser/short_test.go new file mode 100644 index 0000000..67fef15 --- /dev/null +++ b/src/go/parser/short_test.go @@ -0,0 +1,279 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for short valid and invalid programs. + +package parser + +import ( + "go/internal/typeparams" + "testing" +) + +var valids = []string{ + "package p\n", + `package p;`, + `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, + `package p; func f() { if f(T{}) {} };`, + `package p; func f() { _ = <-chan int(nil) };`, + `package p; func f() { _ = (<-chan int)(nil) };`, + `package p; func f() { _ = (<-chan <-chan int)(nil) };`, + `package p; func f() { _ = <-chan <-chan <-chan <-chan <-int(nil) };`, + `package p; func f(func() func() func());`, + `package p; func f(...T);`, + `package p; func f(float, ...int);`, + `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, + `package p; func f(int,) {};`, + `package p; func f(...int,) {};`, + `package p; func f(x ...int,) {};`, + `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, + `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, + `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, + `package p; var a = T{{1, 2}, {3, 4}}`, + `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + `package p; func f() { select { case x := (<-c): } };`, + `package p; func f() { if ; true {} };`, + `package p; func f() { switch ; {} };`, + `package p; func f() { for _ = range "foo" + "bar" {} };`, + `package p; func f() { var s []int; g(s[:], s[i:], s[:j], s[i:j], s[i:j:k], s[:j:k]) };`, + `package p; var ( _ = (struct {*T}).m; _ = (interface {T}).m )`, + `package p; func ((T),) m() {}`, + `package p; func ((*T),) m() {}`, + `package p; func (*(T),) m() {}`, + `package p; func _(x []int) { for range x {} }`, + `package p; func _() { if [T{}.n]int{} {} }`, + `package p; func _() { map[int]int{}[0]++; map[int]int{}[0] += 1 }`, + `package p; func _(x interface{f()}) { interface{f()}(x).f() }`, + `package p; func _(x chan int) { chan int(x) <- 0 }`, + `package p; const (x = 0; y; z)`, // issue 9639 + `package p; var _ = map[P]int{P{}:0, {}:1}`, + `package p; var _ = map[*P]int{&P{}:0, {}:1}`, + `package p; type T = int`, + `package p; type (T = p.T; _ = struct{}; x = *T)`, + `package p; type T (*int)`, + `package p; type _ struct{ ((int)) }`, + `package p; type _ struct{ (*(int)) }`, + `package p; type _ struct{ ([]byte) }`, // disallowed by type-checker + `package p; var _ = func()T(nil)`, + `package p; func _(T (P))`, + `package p; func _(T []E)`, + `package p; func _(T [P]E)`, + `package p; type _ [A+B]struct{}`, + `package p; func (R) _()`, + `package p; type _ struct{ f [n]E }`, + `package p; type _ struct{ f [a+b+c+d]E }`, + `package p; type I1 interface{}; type I2 interface{ I1 }`, +} + +// validWithTParamsOnly holds source code examples that are valid if +// parseTypeParams is set, but invalid if not. When checking with the +// parseTypeParams set, errors are ignored. +var validWithTParamsOnly = []string{ + `package p; type _ []T[ /* ERROR "expected ';', found '\['" */ int]`, + `package p; type T[P any /* ERROR "expected ']', found any" */ ] struct { P }`, + `package p; type T[P comparable /* ERROR "expected ']', found comparable" */ ] struct { P }`, + `package p; type T[P comparable /* ERROR "expected ']', found comparable" */ [P]] struct { P }`, + `package p; type T[P1, /* ERROR "expected ']', found ','" */ P2 any] struct { P1; f []P2 }`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ T any]()()`, + `package p; func _(T (P))`, + `package p; func f[ /* ERROR "expected '\(', found '\['" */ A, B any](); func _() { _ = f[int, int] }`, + `package p; func _(x /* ERROR "mixed named and unnamed parameters" */ T[P1, P2, P3])`, + `package p; func _(x /* ERROR "mixed named and unnamed parameters" */ p.T[Q])`, + `package p; func _(p.T[ /* ERROR "missing ',' in parameter list" */ Q])`, + `package p; type _[A interface /* ERROR "expected ']', found 'interface'" */ {},] struct{}`, + `package p; type _[A interface /* ERROR "expected ']', found 'interface'" */ {}] struct{}`, + `package p; type _[A, /* ERROR "expected ']', found ','" */ B any,] struct{}`, + `package p; type _[A, /* ERROR "expected ']', found ','" */ B any] struct{}`, + `package p; type _[A any /* ERROR "expected ']', found any" */,] struct{}`, + `package p; type _[A any /* ERROR "expected ']', found any" */ ]struct{}`, + `package p; type _[A any /* ERROR "expected ']', found any" */ ] struct{ A }`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ T any]()`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ T any](x T)`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ T1, T2 any](x T)`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ A, B any](a A) B`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ A, B C](a A) B`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ A, B C[A, B]](a A) B`, + `package p; func (T) _[ /* ERROR "expected '\(', found '\['" */ A, B any](a A) B`, + `package p; func (T) _[ /* ERROR "expected '\(', found '\['" */ A, B C](a A) B`, + `package p; func (T) _[ /* ERROR "expected '\(', found '\['" */ A, B C[A, B]](a A) B`, + `package p; type _[A, /* ERROR "expected ']', found ','" */ B any] interface { _(a A) B }`, + `package p; type _[A, /* ERROR "expected ']', found ','" */ B C[A, B]] interface { _(a A) B }`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ T1, T2 interface{}](x T1) T2`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ T1 interface{ m() }, T2, T3 interface{}](x T1, y T3) T2`, + `package p; var _ = [ /* ERROR "expected expression" */ ]T[int]{}`, + `package p; var _ = [ /* ERROR "expected expression" */ 10]T[int]{}`, + `package p; var _ = func /* ERROR "expected expression" */ ()T[int]{}`, + `package p; var _ = map /* ERROR "expected expression" */ [T[int]]T[int]{}`, + `package p; var _ = chan /* ERROR "expected expression" */ T[int](x)`, + `package p; func _(_ T[ /* ERROR "missing ',' in parameter list" */ P], T P) T[P]`, + `package p; var _ T[ /* ERROR "expected ';', found '\['" */ chan int]`, + + // TODO(rfindley) this error message could be improved. + `package p; func (_ /* ERROR "mixed named and unnamed parameters" */ R[P]) _[T any](x T)`, + `package p; func (_ /* ERROR "mixed named and unnamed parameters" */ R[ P, Q]) _[T1, T2 any](x T)`, + + `package p; func (R[P] /* ERROR "missing element type" */ ) _[T any]()`, + `package p; func _(T[P] /* ERROR "missing element type" */ )`, + `package p; func _(T[P1, /* ERROR "expected ']', found ','" */ P2, P3 ])`, + `package p; func _(T[P] /* ERROR "missing element type" */ ) T[P]`, + `package p; type _ struct{ T[P] /* ERROR "missing element type" */ }`, + `package p; type _ struct{ T[struct /* ERROR "expected expression" */ {a, b, c int}] }`, + `package p; type _ interface{type /* ERROR "expected '}', found 'type'" */ int}`, + `package p; type _ interface{type /* ERROR "expected '}', found 'type'" */ int, float32; type bool; m(); type string;}`, + `package p; type I1[T any /* ERROR "expected ']', found any" */ ] interface{}; type I2 interface{ I1[int] }`, + `package p; type I1[T any /* ERROR "expected ']', found any" */ ] interface{}; type I2[T any] interface{ I1[T] }`, + `package p; type _ interface { f[ /* ERROR "expected ';', found '\['" */ T any]() }`, +} + +func TestValid(t *testing.T) { + t.Run("no tparams", func(t *testing.T) { + for _, src := range valids { + checkErrors(t, src, src, DeclarationErrors|AllErrors, false) + } + }) + t.Run("tparams", func(t *testing.T) { + if !typeparams.Enabled { + t.Skip("type params are not enabled") + } + for _, src := range valids { + checkErrors(t, src, src, DeclarationErrors|AllErrors, false) + } + for _, src := range validWithTParamsOnly { + checkErrors(t, src, src, DeclarationErrors|AllErrors, false) + } + }) +} + +// TestSingle is useful to track down a problem with a single short test program. +func TestSingle(t *testing.T) { + const src = `package p; var _ = T{}` + checkErrors(t, src, src, DeclarationErrors|AllErrors, true) +} + +var invalids = []string{ + `foo /* ERROR "expected 'package'" */ !`, + `package p; func f() { if { /* ERROR "missing condition" */ } };`, + `package p; func f() { if ; /* ERROR "missing condition" */ {} };`, + `package p; func f() { if f(); /* ERROR "missing condition" */ {} };`, + `package p; func f() { if _ = range /* ERROR "expected operand" */ x; true {} };`, + `package p; func f() { switch _ /* ERROR "expected switch expression" */ = range x; true {} };`, + `package p; func f() { for _ = range x ; /* ERROR "expected '{'" */ ; {} };`, + `package p; func f() { for ; ; _ = range /* ERROR "expected operand" */ x {} };`, + `package p; func f() { for ; _ /* ERROR "expected boolean or range expression" */ = range x ; {} };`, + `package p; func f() { switch t = /* ERROR "expected ':=', found '='" */ t.(type) {} };`, + `package p; func f() { switch t /* ERROR "expected switch expression" */ , t = t.(type) {} };`, + `package p; func f() { switch t /* ERROR "expected switch expression" */ = t.(type), t {} };`, + `package p; var a = [ /* ERROR "expected expression" */ 1]int;`, + `package p; var a = [ /* ERROR "expected expression" */ ...]int;`, + `package p; var a = struct /* ERROR "expected expression" */ {}`, + `package p; var a = func /* ERROR "expected expression" */ ();`, + `package p; var a = interface /* ERROR "expected expression" */ {}`, + `package p; var a = [ /* ERROR "expected expression" */ ]int`, + `package p; var a = map /* ERROR "expected expression" */ [int]int`, + `package p; var a = chan /* ERROR "expected expression" */ int;`, + `package p; var a = []int{[ /* ERROR "expected expression" */ ]int};`, + `package p; var a = ( /* ERROR "expected expression" */ []int);`, + `package p; var a = <- /* ERROR "expected expression" */ chan int;`, + `package p; func f() { select { case _ <- chan /* ERROR "expected expression" */ int: } };`, + `package p; func f() { _ = (<-<- /* ERROR "expected 'chan'" */ chan int)(nil) };`, + `package p; func f() { _ = (<-chan<-chan<-chan<-chan<-chan<- /* ERROR "expected channel type" */ int)(nil) };`, + `package p; func f() { var t []int; t /* ERROR "expected identifier on left side of :=" */ [0] := 0 };`, + `package p; func f() { if x := g(); x /* ERROR "expected boolean expression" */ = 0 {}};`, + `package p; func f() { _ = x = /* ERROR "expected '=='" */ 0 {}};`, + `package p; func f() { _ = 1 == func()int { var x bool; x = x = /* ERROR "expected '=='" */ true; return x }() };`, + `package p; func f() { var s []int; _ = s[] /* ERROR "expected operand" */ };`, + `package p; func f() { var s []int; _ = s[i:j: /* ERROR "3rd index required" */ ] };`, + `package p; func f() { var s []int; _ = s[i: /* ERROR "2nd index required" */ :k] };`, + `package p; func f() { var s []int; _ = s[i: /* ERROR "2nd index required" */ :] };`, + `package p; func f() { var s []int; _ = s[: /* ERROR "2nd index required" */ :] };`, + `package p; func f() { var s []int; _ = s[: /* ERROR "2nd index required" */ ::] };`, + `package p; func f() { var s []int; _ = s[i:j:k: /* ERROR "expected ']'" */ l] };`, + `package p; func f() { for x /* ERROR "boolean or range expression" */ = []string {} }`, + `package p; func f() { for x /* ERROR "boolean or range expression" */ := []string {} }`, + `package p; func f() { for i /* ERROR "boolean or range expression" */ , x = []string {} }`, + `package p; func f() { for i /* ERROR "boolean or range expression" */ , x := []string {} }`, + `package p; func f() { go f /* ERROR HERE "function must be invoked" */ }`, + `package p; func f() { defer func() {} /* ERROR HERE "function must be invoked" */ }`, + `package p; func f() { go func() { func() { f(x func /* ERROR "missing ','" */ (){}) } } }`, + `package p; func _() (type /* ERROR "found 'type'" */ T)(T)`, + `package p; func (type /* ERROR "found 'type'" */ T)(T) _()`, + `package p; type _[A+B, /* ERROR "expected ']'" */ ] int`, + + // TODO: this error should be positioned on the ':' + `package p; var a = a[[]int:[ /* ERROR "expected expression" */ ]int];`, + // TODO: the compiler error is better here: "cannot parenthesize embedded type" + `package p; type I1 interface{}; type I2 interface{ (/* ERROR "expected '}', found '\('" */ I1) }`, + + // issue 8656 + `package p; func f() (a b string /* ERROR "missing ','" */ , ok bool)`, + + // issue 9639 + `package p; var x /* ERROR "missing variable type or initialization" */ , y, z;`, + `package p; const x /* ERROR "missing constant value" */ ;`, + `package p; const x /* ERROR "missing constant value" */ int;`, + `package p; const (x = 0; y; z /* ERROR "missing constant value" */ int);`, + + // issue 12437 + `package p; var _ = struct { x int, /* ERROR "expected ';', found ','" */ }{};`, + `package p; var _ = struct { x int, /* ERROR "expected ';', found ','" */ y float }{};`, + + // issue 11611 + `package p; type _ struct { int, } /* ERROR "expected 'IDENT', found '}'" */ ;`, + `package p; type _ struct { int, float } /* ERROR "expected type, found '}'" */ ;`, + + // issue 13475 + `package p; func f() { if true {} else ; /* ERROR "expected if statement or block" */ }`, + `package p; func f() { if true {} else defer /* ERROR "expected if statement or block" */ f() }`, +} + +// invalidNoTParamErrs holds invalid source code examples annotated with the +// error messages produced when ParseTypeParams is not set. +var invalidNoTParamErrs = []string{ + `package p; type _[_ any /* ERROR "expected ']', found any" */ ] int; var _ = T[]{}`, + `package p; type T[P any /* ERROR "expected ']', found any" */ ] = T0`, + `package p; var _ func[ /* ERROR "expected '\(', found '\['" */ T any](T)`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ ]()`, + `package p; type _[A, /* ERROR "expected ']', found ','" */] struct{ A }`, + `package p; func _[ /* ERROR "expected '\(', found '\['" */ type P, *Q interface{}]()`, +} + +// invalidTParamErrs holds invalid source code examples annotated with the +// error messages produced when ParseTypeParams is set. +var invalidTParamErrs = []string{ + `package p; type _[_ any] int; var _ = T[] /* ERROR "expected operand" */ {}`, + `package p; type T[P any] = /* ERROR "cannot be alias" */ T0`, + `package p; var _ func[ /* ERROR "cannot have type parameters" */ T any](T)`, + `package p; func _[]/* ERROR "empty type parameter list" */()`, + + // TODO(rfindley) a better location would be after the ']' + `package p; type _[A/* ERROR "all type parameters must be named" */,] struct{ A }`, + + // TODO(rfindley) this error is confusing. + `package p; func _[type /* ERROR "all type parameters must be named" */P, *Q interface{}]()`, +} + +func TestInvalid(t *testing.T) { + t.Run("no tparams", func(t *testing.T) { + for _, src := range invalids { + checkErrors(t, src, src, DeclarationErrors|AllErrors|typeparams.DisallowParsing, true) + } + for _, src := range validWithTParamsOnly { + checkErrors(t, src, src, DeclarationErrors|AllErrors|typeparams.DisallowParsing, true) + } + for _, src := range invalidNoTParamErrs { + checkErrors(t, src, src, DeclarationErrors|AllErrors|typeparams.DisallowParsing, true) + } + }) + t.Run("tparams", func(t *testing.T) { + if !typeparams.Enabled { + t.Skip("type params are not enabled") + } + for _, src := range invalids { + checkErrors(t, src, src, DeclarationErrors|AllErrors, true) + } + for _, src := range invalidTParamErrs { + checkErrors(t, src, src, DeclarationErrors|AllErrors, true) + } + }) +} diff --git a/src/go/parser/testdata/chans.go2 b/src/go/parser/testdata/chans.go2 new file mode 100644 index 0000000..fad2bce --- /dev/null +++ b/src/go/parser/testdata/chans.go2 @@ -0,0 +1,62 @@ +package chans + +import "runtime" + +// Ranger returns a Sender and a Receiver. The Receiver provides a +// Next method to retrieve values. The Sender provides a Send method +// to send values and a Close method to stop sending values. The Next +// method indicates when the Sender has been closed, and the Send +// method indicates when the Receiver has been freed. +// +// This is a convenient way to exit a goroutine sending values when +// the receiver stops reading them. +func Ranger[T any]() (*Sender[T], *Receiver[T]) { + c := make(chan T) + d := make(chan bool) + s := &Sender[T]{values: c, done: d} + r := &Receiver[T]{values: c, done: d} + runtime.SetFinalizer(r, r.finalize) + return s, r +} + +// A sender is used to send values to a Receiver. +type Sender[T any] struct { + values chan<- T + done <-chan bool +} + +// Send sends a value to the receiver. It returns whether any more +// values may be sent; if it returns false the value was not sent. +func (s *Sender[T]) Send(v T) bool { + select { + case s.values <- v: + return true + case <-s.done: + return false + } +} + +// Close tells the receiver that no more values will arrive. +// After Close is called, the Sender may no longer be used. +func (s *Sender[T]) Close() { + close(s.values) +} + +// A Receiver receives values from a Sender. +type Receiver[T any] struct { + values <-chan T + done chan<- bool +} + +// Next returns the next value from the channel. The bool result +// indicates whether the value is valid, or whether the Sender has +// been closed and no more values will be received. +func (r *Receiver[T]) Next() (T, bool) { + v, ok := <-r.values + return v, ok +} + +// finalize is a finalizer for the receiver. +func (r *Receiver[T]) finalize() { + close(r.done) +} diff --git a/src/go/parser/testdata/commas.src b/src/go/parser/testdata/commas.src new file mode 100644 index 0000000..e0603cf --- /dev/null +++ b/src/go/parser/testdata/commas.src @@ -0,0 +1,19 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for error messages/parser synchronization +// after missing commas. + +package p + +var _ = []int{ + 0/* ERROR HERE "missing ','" */ +} + +var _ = []int{ + 0, + 1, + 2, + 3/* ERROR HERE "missing ','" */ +} diff --git a/src/go/parser/testdata/issue11377.src b/src/go/parser/testdata/issue11377.src new file mode 100644 index 0000000..1c43800 --- /dev/null +++ b/src/go/parser/testdata/issue11377.src @@ -0,0 +1,27 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 11377: Better synchronization of +// parser after certain syntax errors. + +package p + +func bad1() { + if f()) /* ERROR "expected ';', found '\)'" */ { + return + } +} + +// There shouldn't be any errors down below. + +func F1() {} +func F2() {} +func F3() {} +func F4() {} +func F5() {} +func F6() {} +func F7() {} +func F8() {} +func F9() {} +func F10() {} diff --git a/src/go/parser/testdata/issue23434.src b/src/go/parser/testdata/issue23434.src new file mode 100644 index 0000000..24a0832 --- /dev/null +++ b/src/go/parser/testdata/issue23434.src @@ -0,0 +1,25 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 23434: Better synchronization of +// parser after missing type. There should be exactly +// one error each time, with now follow errors. + +package p + +func g() { + m := make(map[string]! /* ERROR "expected type, found '!'" */ ) + for { + x := 1 + print(x) + } +} + +func f() { + m := make(map[string]) /* ERROR "expected type, found '\)'" */ + for { + x := 1 + print(x) + } +} diff --git a/src/go/parser/testdata/issue3106.src b/src/go/parser/testdata/issue3106.src new file mode 100644 index 0000000..2db10be --- /dev/null +++ b/src/go/parser/testdata/issue3106.src @@ -0,0 +1,46 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 3106: Better synchronization of +// parser after certain syntax errors. + +package main + +func f() { + var m Mutex + c := MakeCond(&m) + percent := 0 + const step = 10 + for i := 0; i < 5; i++ { + go func() { + for { + // Emulates some useful work. + time.Sleep(1e8) + m.Lock() + defer + if /* ERROR "expected ';', found 'if'" */ percent == 100 { + m.Unlock() + break + } + percent++ + if percent % step == 0 { + //c.Signal() + } + m.Unlock() + } + }() + } + for { + m.Lock() + if percent == 0 || percent % step != 0 { + c.Wait() + } + fmt.Print(",") + if percent == 100 { + m.Unlock() + break + } + m.Unlock() + } +} diff --git a/src/go/parser/testdata/issue34946.src b/src/go/parser/testdata/issue34946.src new file mode 100644 index 0000000..6bb15e1 --- /dev/null +++ b/src/go/parser/testdata/issue34946.src @@ -0,0 +1,22 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 34946: Better synchronization of +// parser for function declarations that start their +// body's opening { on a new line. + +package p + +// accept Allman/BSD-style declaration but complain +// (implicit semicolon between signature and body) +func _() int +{ /* ERROR "unexpected semicolon or newline before {" */ + { return 0 } +} + +func _() {} + +func _(); { /* ERROR "unexpected semicolon or newline before {" */ } + +func _() {} diff --git a/src/go/parser/testdata/issue42951/not_a_file.go/invalid.go b/src/go/parser/testdata/issue42951/not_a_file.go/invalid.go new file mode 100644 index 0000000..bb698be --- /dev/null +++ b/src/go/parser/testdata/issue42951/not_a_file.go/invalid.go @@ -0,0 +1 @@ +This file should not be parsed by ParseDir. diff --git a/src/go/parser/testdata/issue44504.src b/src/go/parser/testdata/issue44504.src new file mode 100644 index 0000000..7791f4a --- /dev/null +++ b/src/go/parser/testdata/issue44504.src @@ -0,0 +1,13 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 44504: panic due to duplicate resolution of slice/index +// operands. We should not try to resolve a LHS expression with invalid syntax. + +package p + +func _() { + var items []bool + items[] /* ERROR "operand" */ = false +} diff --git a/src/go/parser/testdata/linalg.go2 b/src/go/parser/testdata/linalg.go2 new file mode 100644 index 0000000..fba0d02 --- /dev/null +++ b/src/go/parser/testdata/linalg.go2 @@ -0,0 +1,83 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linalg + +import "math" + +// Numeric is type bound that matches any numeric type. +// It would likely be in a constraints package in the standard library. +type Numeric interface { + type int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, uintptr, + float32, float64, + complex64, complex128 +} + +func DotProduct[T Numeric](s1, s2 []T) T { + if len(s1) != len(s2) { + panic("DotProduct: slices of unequal length") + } + var r T + for i := range s1 { + r += s1[i] * s2[i] + } + return r +} + +// NumericAbs matches numeric types with an Abs method. +type NumericAbs[T any] interface { + Numeric + + Abs() T +} + +// AbsDifference computes the absolute value of the difference of +// a and b, where the absolute value is determined by the Abs method. +func AbsDifference[T NumericAbs](a, b T) T { + d := a - b + return d.Abs() +} + +// OrderedNumeric is a type bound that matches numeric types that support the < operator. +type OrderedNumeric interface { + type int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, uintptr, + float32, float64 +} + +// Complex is a type bound that matches the two complex types, which do not have a < operator. +type Complex interface { + type complex64, complex128 +} + +// OrderedAbs is a helper type that defines an Abs method for +// ordered numeric types. +type OrderedAbs[T OrderedNumeric] T + +func (a OrderedAbs[T]) Abs() OrderedAbs[T] { + if a < 0 { + return -a + } + return a +} + +// ComplexAbs is a helper type that defines an Abs method for +// complex types. +type ComplexAbs[T Complex] T + +func (a ComplexAbs[T]) Abs() ComplexAbs[T] { + r := float64(real(a)) + i := float64(imag(a)) + d := math.Sqrt(r * r + i * i) + return ComplexAbs[T](complex(d, 0)) +} + +func OrderedAbsDifference[T OrderedNumeric](a, b T) T { + return T(AbsDifference(OrderedAbs[T](a), OrderedAbs[T](b))) +} + +func ComplexAbsDifference[T Complex](a, b T) T { + return T(AbsDifference(ComplexAbs[T](a), ComplexAbs[T](b))) +} diff --git a/src/go/parser/testdata/map.go2 b/src/go/parser/testdata/map.go2 new file mode 100644 index 0000000..74c79ae --- /dev/null +++ b/src/go/parser/testdata/map.go2 @@ -0,0 +1,109 @@ +// Package orderedmap provides an ordered map, implemented as a binary tree. +package orderedmap + +import "chans" + +// Map is an ordered map. +type Map[K, V any] struct { + root *node[K, V] + compare func(K, K) int +} + +// node is the type of a node in the binary tree. +type node[K, V any] struct { + key K + val V + left, right *node[K, V] +} + +// New returns a new map. +func New[K, V any](compare func(K, K) int) *Map[K, V] { + return &Map[K, V]{compare: compare} +} + +// find looks up key in the map, and returns either a pointer +// to the node holding key, or a pointer to the location where +// such a node would go. +func (m *Map[K, V]) find(key K) **node[K, V] { + pn := &m.root + for *pn != nil { + switch cmp := m.compare(key, (*pn).key); { + case cmp < 0: + pn = &(*pn).left + case cmp > 0: + pn = &(*pn).right + default: + return pn + } + } + return pn +} + +// Insert inserts a new key/value into the map. +// If the key is already present, the value is replaced. +// Returns true if this is a new key, false if already present. +func (m *Map[K, V]) Insert(key K, val V) bool { + pn := m.find(key) + if *pn != nil { + (*pn).val = val + return false + } + *pn = &node[K, V]{key: key, val: val} + return true +} + +// Find returns the value associated with a key, or zero if not present. +// The found result reports whether the key was found. +func (m *Map[K, V]) Find(key K) (V, bool) { + pn := m.find(key) + if *pn == nil { + var zero V // see the discussion of zero values, above + return zero, false + } + return (*pn).val, true +} + +// keyValue is a pair of key and value used when iterating. +type keyValue[K, V any] struct { + key K + val V +} + +// InOrder returns an iterator that does an in-order traversal of the map. +func (m *Map[K, V]) InOrder() *Iterator[K, V] { + sender, receiver := chans.Ranger[keyValue[K, V]]() + var f func(*node[K, V]) bool + f = func(n *node[K, V]) bool { + if n == nil { + return true + } + // Stop sending values if sender.Send returns false, + // meaning that nothing is listening at the receiver end. + return f(n.left) && + // TODO + // sender.Send(keyValue[K, V]{n.key, n.val}) && + f(n.right) + } + go func() { + f(m.root) + sender.Close() + }() + return &Iterator{receiver} +} + +// Iterator is used to iterate over the map. +type Iterator[K, V any] struct { + r *chans.Receiver[keyValue[K, V]] +} + +// Next returns the next key and value pair, and a boolean indicating +// whether they are valid or whether we have reached the end. +func (it *Iterator[K, V]) Next() (K, V, bool) { + keyval, ok := it.r.Next() + if !ok { + var zerok K + var zerov V + return zerok, zerov, false + } + return keyval.key, keyval.val, true +} diff --git a/src/go/parser/testdata/metrics.go2 b/src/go/parser/testdata/metrics.go2 new file mode 100644 index 0000000..ef1c66b --- /dev/null +++ b/src/go/parser/testdata/metrics.go2 @@ -0,0 +1,58 @@ +package metrics + +import "sync" + +type Metric1[T comparable] struct { + mu sync.Mutex + m map[T]int +} + +func (m *Metric1[T]) Add(v T) { + m.mu.Lock() + defer m.mu.Unlock() + if m.m == nil { + m.m = make(map[T]int) + } + m[v]++ +} + +type key2[T1, T2 comparable] struct { + f1 T1 + f2 T2 +} + +type Metric2[T1, T2 cmp2] struct { + mu sync.Mutex + m map[key2[T1, T2]]int +} + +func (m *Metric2[T1, T2]) Add(v1 T1, v2 T2) { + m.mu.Lock() + defer m.mu.Unlock() + if m.m == nil { + m.m = make(map[key2[T1, T2]]int) + } + m[key[T1, T2]{v1, v2}]++ +} + +type key3[T1, T2, T3 comparable] struct { + f1 T1 + f2 T2 + f3 T3 +} + +type Metric3[T1, T2, T3 comparable] struct { + mu sync.Mutex + m map[key3[T1, T2, T3]]int +} + +func (m *Metric3[T1, T2, T3]) Add(v1 T1, v2 T2, v3 T3) { + m.mu.Lock() + defer m.mu.Unlock() + if m.m == nil { + m.m = make(map[key3]int) + } + m[key[T1, T2, T3]{v1, v2, v3}]++ +} + +// Repeat for the maximum number of permitted arguments. diff --git a/src/go/parser/testdata/resolution/issue45136.src b/src/go/parser/testdata/resolution/issue45136.src new file mode 100644 index 0000000..e1d63d8 --- /dev/null +++ b/src/go/parser/testdata/resolution/issue45136.src @@ -0,0 +1,27 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue45136 + +type obj /* =@obj */ struct { + name /*=@name */ string +} + +func _() { + var foo /* =@foo */ = "foo" + obj /* @obj */ ["foo"] + obj /* @obj */ .run() + obj /* @obj */ { + name: foo /* @foo */, + } + obj /* @obj */ { + name: "bar", + }.run() + + var _ = File{key: obj /* @obj */ {}} + var _ = File{obj /* @obj */ {}} + + []obj /* @obj */ {foo /* @foo */} + x /* =@x1 */ := obj /* @obj */{} +} diff --git a/src/go/parser/testdata/resolution/issue45160.src b/src/go/parser/testdata/resolution/issue45160.src new file mode 100644 index 0000000..6be933b --- /dev/null +++ b/src/go/parser/testdata/resolution/issue45160.src @@ -0,0 +1,25 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue45160 + +func mklink1 /* =@mklink1func */() {} + +func _() { + var tests /* =@tests */ = []dirLinkTest /* @dirLinkTest */ { + { + mklink1 /* @mklink1func */: func() {}, + mklink2: func(link /* =@link */, target /* =@target */ string) error { + return nil + }, + }, + } +} + +type dirLinkTest /* =@dirLinkTest */ struct { + mklink1 /* =@mklink1field */ func(string, string) error + mklink2 /* =@mklink2field */ func(string, string) error +} + +func mklink2 /* =@mklink2func */() {} diff --git a/src/go/parser/testdata/resolution/resolution.src b/src/go/parser/testdata/resolution/resolution.src new file mode 100644 index 0000000..a880dd1 --- /dev/null +++ b/src/go/parser/testdata/resolution/resolution.src @@ -0,0 +1,63 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package resolution + +func f /* =@fdecl */(n /* =@narg */ ast.Node) bool { + if n /* =@ninit */, ok /* =@ok */ := n /* @narg */ .(*ast.SelectorExpr); ok /* @ok */ { + sel = n /* @ninit */ + } +} + +type c /* =@cdecl */ map[token.Pos]resolvedObj + +func (v /* =@vdecl */ c /* @cdecl */) Visit(node /* =@nodearg */ ast.Node) (w /* =@w */ ast.Visitor) {} + +const ( + basic /* =@basic */ = iota + labelOk // =@labelOk +) + +type T /* =@T */ int + +func _(count /* =@count */ T /* @T */) { + x /* =@x1 */ := c /* @cdecl */{} + switch x /* =@x2 */ := x /* @x1 */; x /* =@x3 */ := x /* @x2 */.(type) { + case c /* @cdecl */: + default: + } +loop /* =@loop */: + for { + if true { + break loop /* @loop */ + } + } + select { + case err /* =@err1 */ := <-_: + return err /* @err1 */ + case err /* =@err2 */ := <-_: + return err /* @err2 */ + } + + _ = func(p1 /* =@p1 */ int, p2 /* =@p2 */ p1) { + closed /* =@closed */ := p1 // @p1 + shadowed /* =@shadowed1 */ := p2 // @p2 + _ = func(shadowed /* =@shadowed2 */ p2 /* @p2 */) { + closed /* @closed */ = 1 + shadowed /* @shadowed2 */ = 2 + } + } +} + +func (r /* =@r */ c /* @cdecl */) m(_ r) c /* @cdecl */ { return r /* @r */ } + +var cycle /* =@cycle */ = cycle /* @cycle */ + 1 + +type chain /* =@chain */ struct { + next /* =@next */ *chain /* @chain */ +} + +func recursive /* =@recursive */() { + recursive /* @recursive */ () +} diff --git a/src/go/parser/testdata/resolution/typeparams.go2 b/src/go/parser/testdata/resolution/typeparams.go2 new file mode 100644 index 0000000..0ffecd6 --- /dev/null +++ b/src/go/parser/testdata/resolution/typeparams.go2 @@ -0,0 +1,43 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package resolution + +type List /* =@List */ [E /* =@E */ any] []E // @E + +type Pair /* =@Pair */ [L /* =@L */, R /* =@R */ any] struct { + Left /* =@Left */ L // @L + Right /* =@Right */ R // @R + L /* =@Lfield */ int +} + +var _ = Pair /* @Pair */ [int, string]{} + +type Addable /* =@Addable */ interface { + type int64, float64 +} + +func Add /* =@AddDecl */[T /* =@T */ Addable /* @Addable */](l /* =@l */, r /* =@r */ T /* @T */) T /* @T */ { + var t /* =@t */ T /* @T */ + return l /* @l */ + r /* @r */ + t /* @t */ +} + +type Receiver /* =@Receiver */[P /* =@P */ any] struct {} + +// TODO(rFindley): make a decision on how/whether to resolve identifiers that +// refer to receiver type parameters, as is the case for the 'P' result +// parameter below. +func (r /* =@recv */ Receiver /* @Receiver */ [P]) m() P {} + +func f /* =@f */[T1 /* =@T1 */ interface{type []T2 /* @T2 */}, T2 /* =@T2 */ any]( + x /* =@x */ T1 /* @T1 */, T1 /* =@T1_duplicate */ y, // Note that this is a bug: + // the duplicate T1 should + // not be allowed. + ){ + // Note that duplicate short var declarations resolve to their alt declaration. + x /* @x */ := 0 + y /* =@y */ := 0 + T1 /* @T1 */ := 0 + var t1var /* =@t1var */ T1 /* @T1 */ +} diff --git a/src/go/parser/testdata/set.go2 b/src/go/parser/testdata/set.go2 new file mode 100644 index 0000000..0da6377 --- /dev/null +++ b/src/go/parser/testdata/set.go2 @@ -0,0 +1,31 @@ +// Package set implements sets of any type. +package set + +type Set[Elem comparable] map[Elem]struct{} + +func Make[Elem comparable]() Set[Elem] { + return make(Set(Elem)) +} + +func (s Set[Elem]) Add(v Elem) { + s[v] = struct{}{} +} + +func (s Set[Elem]) Delete(v Elem) { + delete(s, v) +} + +func (s Set[Elem]) Contains(v Elem) bool { + _, ok := s[v] + return ok +} + +func (s Set[Elem]) Len() int { + return len(s) +} + +func (s Set[Elem]) Iterate(f func(Elem)) { + for v := range s { + f(v) + } +} diff --git a/src/go/parser/testdata/slices.go2 b/src/go/parser/testdata/slices.go2 new file mode 100644 index 0000000..e060212 --- /dev/null +++ b/src/go/parser/testdata/slices.go2 @@ -0,0 +1,31 @@ +// Package slices implements various slice algorithms. +package slices + +// Map turns a []T1 to a []T2 using a mapping function. +func Map[T1, T2 any](s []T1, f func(T1) T2) []T2 { + r := make([]T2, len(s)) + for i, v := range s { + r[i] = f(v) + } + return r +} + +// Reduce reduces a []T1 to a single value using a reduction function. +func Reduce[T1, T2 any](s []T1, initializer T2, f func(T2, T1) T2) T2 { + r := initializer + for _, v := range s { + r = f(r, v) + } + return r +} + +// Filter filters values from a slice using a filter function. +func Filter[T any](s []T, f func(T) bool) []T { + var r []T + for _, v := range s { + if f(v) { + r = append(r, v) + } + } + return r +} diff --git a/src/go/parser/testdata/sort.go2 b/src/go/parser/testdata/sort.go2 new file mode 100644 index 0000000..88be79f --- /dev/null +++ b/src/go/parser/testdata/sort.go2 @@ -0,0 +1,27 @@ +package sort + +type orderedSlice[Elem comparable] []Elem + +func (s orderedSlice[Elem]) Len() int { return len(s) } +func (s orderedSlice[Elem]) Less(i, j int) bool { return s[i] < s[j] } +func (s orderedSlice[Elem]) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// OrderedSlice sorts the slice s in ascending order. +// The elements of s must be ordered using the < operator. +func OrderedSlice[Elem comparable](s []Elem) { + sort.Sort(orderedSlice[Elem](s)) +} + +type sliceFn[Elem any] struct { + s []Elem + f func(Elem, Elem) bool +} + +func (s sliceFn[Elem]) Len() int { return len(s.s) } +func (s sliceFn[Elem]) Less(i, j int) bool { return s.f(s.s[i], s.s[j]) } +func (s sliceFn[Elem]) Swap(i, j int) { s.s[i], s.s[j] = s.s[j], s.s[i] } + +// SliceFn sorts the slice s according to the function f. +func SliceFn[Elem any](s []Elem, f func(Elem, Elem) bool) { + Sort(sliceFn[Elem]{s, f}) +} diff --git a/src/go/parser/testdata/typeparams.src b/src/go/parser/testdata/typeparams.src new file mode 100644 index 0000000..1fea23f --- /dev/null +++ b/src/go/parser/testdata/typeparams.src @@ -0,0 +1,17 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test cases for error messages produced while parsing code that uses type +// parameters, without ParseTypeParams being enabled. + +package p + +type List[E any /* ERROR "expected ']', found any" */ ] []E + +type Pair[L, /* ERROR "expected ']', found ','" */ R any] struct { + Left L + Right R +} + +var _ = Pair[int, /* ERROR "expected ']' or ':', found ','" */ string]{} |