Adding upstream version 1.18.10.upstream/1.18.10upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 881 insertions, 0 deletions

diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go
new file mode 100644
index 0000000..218bc24
--- /dev/null
+++ b/src/cmd/compile/internal/syntax/scanner.go
@@ -0,0 +1,881 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements scanner, a lexical tokenizer for
+// Go source. After initialization, consecutive calls of
+// next advance the scanner one token at a time.
+//
+// This file, source.go, tokens.go, and token_string.go are self-contained
+// (`go tool compile scanner.go source.go tokens.go token_string.go` compiles)
+// and thus could be made into their own package.
+
+package syntax
+
+import (
+	"fmt"
+	"io"
+	"unicode"
+	"unicode/utf8"
+)
+
+// The mode flags below control which comments are reported
+// by calling the error handler. If no flag is set, comments
+// are ignored.
+const (
+	comments   uint = 1 << iota // call handler for all comments
+	directives                  // call handler for directives only
+)
+
+type scanner struct {
+	source
+	mode   uint
+	nlsemi bool // if set '\n' and EOF translate to ';'
+
+	// current token, valid after calling next()
+	line, col uint
+	blank     bool // line is blank up to col
+	tok       token
+	lit       string   // valid if tok is _Name, _Literal, or _Semi ("semicolon", "newline", or "EOF"); may be malformed if bad is true
+	bad       bool     // valid if tok is _Literal, true if a syntax error occurred, lit may be malformed
+	kind      LitKind  // valid if tok is _Literal
+	op        Operator // valid if tok is _Operator, _AssignOp, or _IncOp
+	prec      int      // valid if tok is _Operator, _AssignOp, or _IncOp
+}
+
+func (s *scanner) init(src io.Reader, errh func(line, col uint, msg string), mode uint) {
+	s.source.init(src, errh)
+	s.mode = mode
+	s.nlsemi = false
+}
+
+// errorf reports an error at the most recently read character position.
+func (s *scanner) errorf(format string, args ...interface{}) {
+	s.error(fmt.Sprintf(format, args...))
+}
+
+// errorAtf reports an error at a byte column offset relative to the current token start.
+func (s *scanner) errorAtf(offset int, format string, args ...interface{}) {
+	s.errh(s.line, s.col+uint(offset), fmt.Sprintf(format, args...))
+}
+
+// setLit sets the scanner state for a recognized _Literal token.
+func (s *scanner) setLit(kind LitKind, ok bool) {
+	s.nlsemi = true
+	s.tok = _Literal
+	s.lit = string(s.segment())
+	s.bad = !ok
+	s.kind = kind
+}
+
+// next advances the scanner by reading the next token.
+//
+// If a read, source encoding, or lexical error occurs, next calls
+// the installed error handler with the respective error position
+// and message. The error message is guaranteed to be non-empty and
+// never starts with a '/'. The error handler must exist.
+//
+// If the scanner mode includes the comments flag and a comment
+// (including comments containing directives) is encountered, the
+// error handler is also called with each comment position and text
+// (including opening /* or // and closing */, but without a newline
+// at the end of line comments). Comment text always starts with a /
+// which can be used to distinguish these handler calls from errors.
+//
+// If the scanner mode includes the directives (but not the comments)
+// flag, only comments containing a //line, /*line, or //go: directive
+// are reported, in the same way as regular comments.
+func (s *scanner) next() {
+	nlsemi := s.nlsemi
+	s.nlsemi = false
+
+redo:
+	// skip white space
+	s.stop()
+	startLine, startCol := s.pos()
+	for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !nlsemi || s.ch == '\r' {
+		s.nextch()
+	}
+
+	// token start
+	s.line, s.col = s.pos()
+	s.blank = s.line > startLine || startCol == colbase
+	s.start()
+	if isLetter(s.ch) || s.ch >= utf8.RuneSelf && s.atIdentChar(true) {
+		s.nextch()
+		s.ident()
+		return
+	}
+
+	switch s.ch {
+	case -1:
+		if nlsemi {
+			s.lit = "EOF"
+			s.tok = _Semi
+			break
+		}
+		s.tok = _EOF
+
+	case '\n':
+		s.nextch()
+		s.lit = "newline"
+		s.tok = _Semi
+
+	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		s.number(false)
+
+	case '"':
+		s.stdString()
+
+	case '`':
+		s.rawString()
+
+	case '\'':
+		s.rune()
+
+	case '(':
+		s.nextch()
+		s.tok = _Lparen
+
+	case '[':
+		s.nextch()
+		s.tok = _Lbrack
+
+	case '{':
+		s.nextch()
+		s.tok = _Lbrace
+
+	case ',':
+		s.nextch()
+		s.tok = _Comma
+
+	case ';':
+		s.nextch()
+		s.lit = "semicolon"
+		s.tok = _Semi
+
+	case ')':
+		s.nextch()
+		s.nlsemi = true
+		s.tok = _Rparen
+
+	case ']':
+		s.nextch()
+		s.nlsemi = true
+		s.tok = _Rbrack
+
+	case '}':
+		s.nextch()
+		s.nlsemi = true
+		s.tok = _Rbrace
+
+	case ':':
+		s.nextch()
+		if s.ch == '=' {
+			s.nextch()
+			s.tok = _Define
+			break
+		}
+		s.tok = _Colon
+
+	case '.':
+		s.nextch()
+		if isDecimal(s.ch) {
+			s.number(true)
+			break
+		}
+		if s.ch == '.' {
+			s.nextch()
+			if s.ch == '.' {
+				s.nextch()
+				s.tok = _DotDotDot
+				break
+			}
+			s.rewind() // now s.ch holds 1st '.'
+			s.nextch() // consume 1st '.' again
+		}
+		s.tok = _Dot
+
+	case '+':
+		s.nextch()
+		s.op, s.prec = Add, precAdd
+		if s.ch != '+' {
+			goto assignop
+		}
+		s.nextch()
+		s.nlsemi = true
+		s.tok = _IncOp
+
+	case '-':
+		s.nextch()
+		s.op, s.prec = Sub, precAdd
+		if s.ch != '-' {
+			goto assignop
+		}
+		s.nextch()
+		s.nlsemi = true
+		s.tok = _IncOp
+
+	case '*':
+		s.nextch()
+		s.op, s.prec = Mul, precMul
+		// don't goto assignop - want _Star token
+		if s.ch == '=' {
+			s.nextch()
+			s.tok = _AssignOp
+			break
+		}
+		s.tok = _Star
+
+	case '/':
+		s.nextch()
+		if s.ch == '/' {
+			s.nextch()
+			s.lineComment()
+			goto redo
+		}
+		if s.ch == '*' {
+			s.nextch()
+			s.fullComment()
+			if line, _ := s.pos(); line > s.line && nlsemi {
+				// A multi-line comment acts like a newline;
+				// it translates to a ';' if nlsemi is set.
+				s.lit = "newline"
+				s.tok = _Semi
+				break
+			}
+			goto redo
+		}
+		s.op, s.prec = Div, precMul
+		goto assignop
+
+	case '%':
+		s.nextch()
+		s.op, s.prec = Rem, precMul
+		goto assignop
+
+	case '&':
+		s.nextch()
+		if s.ch == '&' {
+			s.nextch()
+			s.op, s.prec = AndAnd, precAndAnd
+			s.tok = _Operator
+			break
+		}
+		s.op, s.prec = And, precMul
+		if s.ch == '^' {
+			s.nextch()
+			s.op = AndNot
+		}
+		goto assignop
+
+	case '|':
+		s.nextch()
+		if s.ch == '|' {
+			s.nextch()
+			s.op, s.prec = OrOr, precOrOr
+			s.tok = _Operator
+			break
+		}
+		s.op, s.prec = Or, precAdd
+		goto assignop
+
+	case '^':
+		s.nextch()
+		s.op, s.prec = Xor, precAdd
+		goto assignop
+
+	case '<':
+		s.nextch()
+		if s.ch == '=' {
+			s.nextch()
+			s.op, s.prec = Leq, precCmp
+			s.tok = _Operator
+			break
+		}
+		if s.ch == '<' {
+			s.nextch()
+			s.op, s.prec = Shl, precMul
+			goto assignop
+		}
+		if s.ch == '-' {
+			s.nextch()
+			s.tok = _Arrow
+			break
+		}
+		s.op, s.prec = Lss, precCmp
+		s.tok = _Operator
+
+	case '>':
+		s.nextch()
+		if s.ch == '=' {
+			s.nextch()
+			s.op, s.prec = Geq, precCmp
+			s.tok = _Operator
+			break
+		}
+		if s.ch == '>' {
+			s.nextch()
+			s.op, s.prec = Shr, precMul
+			goto assignop
+		}
+		s.op, s.prec = Gtr, precCmp
+		s.tok = _Operator
+
+	case '=':
+		s.nextch()
+		if s.ch == '=' {
+			s.nextch()
+			s.op, s.prec = Eql, precCmp
+			s.tok = _Operator
+			break
+		}
+		s.tok = _Assign
+
+	case '!':
+		s.nextch()
+		if s.ch == '=' {
+			s.nextch()
+			s.op, s.prec = Neq, precCmp
+			s.tok = _Operator
+			break
+		}
+		s.op, s.prec = Not, 0
+		s.tok = _Operator
+
+	case '~':
+		s.nextch()
+		s.op, s.prec = Tilde, 0
+		s.tok = _Operator
+
+	default:
+		s.errorf("invalid character %#U", s.ch)
+		s.nextch()
+		goto redo
+	}
+
+	return
+
+assignop:
+	if s.ch == '=' {
+		s.nextch()
+		s.tok = _AssignOp
+		return
+	}
+	s.tok = _Operator
+}
+
+func (s *scanner) ident() {
+	// accelerate common case (7bit ASCII)
+	for isLetter(s.ch) || isDecimal(s.ch) {
+		s.nextch()
+	}
+
+	// general case
+	if s.ch >= utf8.RuneSelf {
+		for s.atIdentChar(false) {
+			s.nextch()
+		}
+	}
+
+	// possibly a keyword
+	lit := s.segment()
+	if len(lit) >= 2 {
+		if tok := keywordMap[hash(lit)]; tok != 0 && tokStrFast(tok) == string(lit) {
+			s.nlsemi = contains(1<<_Break|1<<_Continue|1<<_Fallthrough|1<<_Return, tok)
+			s.tok = tok
+			return
+		}
+	}
+
+	s.nlsemi = true
+	s.lit = string(lit)
+	s.tok = _Name
+}
+
+// tokStrFast is a faster version of token.String, which assumes that tok
+// is one of the valid tokens - and can thus skip bounds checks.
+func tokStrFast(tok token) string {
+	return _token_name[_token_index[tok-1]:_token_index[tok]]
+}
+
+func (s *scanner) atIdentChar(first bool) bool {
+	switch {
+	case unicode.IsLetter(s.ch) || s.ch == '_':
+		// ok
+	case unicode.IsDigit(s.ch):
+		if first {
+			s.errorf("identifier cannot begin with digit %#U", s.ch)
+		}
+	case s.ch >= utf8.RuneSelf:
+		s.errorf("invalid character %#U in identifier", s.ch)
+	default:
+		return false
+	}
+	return true
+}
+
+// hash is a perfect hash function for keywords.
+// It assumes that s has at least length 2.
+func hash(s []byte) uint {
+	return (uint(s[0])<<4 ^ uint(s[1]) + uint(len(s))) & uint(len(keywordMap)-1)
+}
+
+var keywordMap [1 << 6]token // size must be power of two
+
+func init() {
+	// populate keywordMap
+	for tok := _Break; tok <= _Var; tok++ {
+		h := hash([]byte(tok.String()))
+		if keywordMap[h] != 0 {
+			panic("imperfect hash")
+		}
+		keywordMap[h] = tok
+	}
+}
+
+func lower(ch rune) rune     { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter
+func isLetter(ch rune) bool  { return 'a' <= lower(ch) && lower(ch) <= 'z' || ch == '_' }
+func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
+func isHex(ch rune) bool     { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' }
+
+// digits accepts the sequence { digit | '_' }.
+// If base <= 10, digits accepts any decimal digit but records
+// the index (relative to the literal start) of a digit >= base
+// in *invalid, if *invalid < 0.
+// digits returns a bitset describing whether the sequence contained
+// digits (bit 0 is set), or separators '_' (bit 1 is set).
+func (s *scanner) digits(base int, invalid *int) (digsep int) {
+	if base <= 10 {
+		max := rune('0' + base)
+		for isDecimal(s.ch) || s.ch == '_' {
+			ds := 1
+			if s.ch == '_' {
+				ds = 2
+			} else if s.ch >= max && *invalid < 0 {
+				_, col := s.pos()
+				*invalid = int(col - s.col) // record invalid rune index
+			}
+			digsep |= ds
+			s.nextch()
+		}
+	} else {
+		for isHex(s.ch) || s.ch == '_' {
+			ds := 1
+			if s.ch == '_' {
+				ds = 2
+			}
+			digsep |= ds
+			s.nextch()
+		}
+	}
+	return
+}
+
+func (s *scanner) number(seenPoint bool) {
+	ok := true
+	kind := IntLit
+	base := 10        // number base
+	prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b'
+	digsep := 0       // bit 0: digit present, bit 1: '_' present
+	invalid := -1     // index of invalid digit in literal, or < 0
+
+	// integer part
+	if !seenPoint {
+		if s.ch == '0' {
+			s.nextch()
+			switch lower(s.ch) {
+			case 'x':
+				s.nextch()
+				base, prefix = 16, 'x'
+			case 'o':
+				s.nextch()
+				base, prefix = 8, 'o'
+			case 'b':
+				s.nextch()
+				base, prefix = 2, 'b'
+			default:
+				base, prefix = 8, '0'
+				digsep = 1 // leading 0
+			}
+		}
+		digsep |= s.digits(base, &invalid)
+		if s.ch == '.' {
+			if prefix == 'o' || prefix == 'b' {
+				s.errorf("invalid radix point in %s literal", baseName(base))
+				ok = false
+			}
+			s.nextch()
+			seenPoint = true
+		}
+	}
+
+	// fractional part
+	if seenPoint {
+		kind = FloatLit
+		digsep |= s.digits(base, &invalid)
+	}
+
+	if digsep&1 == 0 && ok {
+		s.errorf("%s literal has no digits", baseName(base))
+		ok = false
+	}
+
+	// exponent
+	if e := lower(s.ch); e == 'e' || e == 'p' {
+		if ok {
+			switch {
+			case e == 'e' && prefix != 0 && prefix != '0':
+				s.errorf("%q exponent requires decimal mantissa", s.ch)
+				ok = false
+			case e == 'p' && prefix != 'x':
+				s.errorf("%q exponent requires hexadecimal mantissa", s.ch)
+				ok = false
+			}
+		}
+		s.nextch()
+		kind = FloatLit
+		if s.ch == '+' || s.ch == '-' {
+			s.nextch()
+		}
+		digsep = s.digits(10, nil) | digsep&2 // don't lose sep bit
+		if digsep&1 == 0 && ok {
+			s.errorf("exponent has no digits")
+			ok = false
+		}
+	} else if prefix == 'x' && kind == FloatLit && ok {
+		s.errorf("hexadecimal mantissa requires a 'p' exponent")
+		ok = false
+	}
+
+	// suffix 'i'
+	if s.ch == 'i' {
+		kind = ImagLit
+		s.nextch()
+	}
+
+	s.setLit(kind, ok) // do this now so we can use s.lit below
+
+	if kind == IntLit && invalid >= 0 && ok {
+		s.errorAtf(invalid, "invalid digit %q in %s literal", s.lit[invalid], baseName(base))
+		ok = false
+	}
+
+	if digsep&2 != 0 && ok {
+		if i := invalidSep(s.lit); i >= 0 {
+			s.errorAtf(i, "'_' must separate successive digits")
+			ok = false
+		}
+	}
+
+	s.bad = !ok // correct s.bad
+}
+
+func baseName(base int) string {
+	switch base {
+	case 2:
+		return "binary"
+	case 8:
+		return "octal"
+	case 10:
+		return "decimal"
+	case 16:
+		return "hexadecimal"
+	}
+	panic("invalid base")
+}
+
+// invalidSep returns the index of the first invalid separator in x, or -1.
+func invalidSep(x string) int {
+	x1 := ' ' // prefix char, we only care if it's 'x'
+	d := '.'  // digit, one of '_', '0' (a digit), or '.' (anything else)
+	i := 0
+
+	// a prefix counts as a digit
+	if len(x) >= 2 && x[0] == '0' {
+		x1 = lower(rune(x[1]))
+		if x1 == 'x' || x1 == 'o' || x1 == 'b' {
+			d = '0'
+			i = 2
+		}
+	}
+
+	// mantissa and exponent
+	for ; i < len(x); i++ {
+		p := d // previous digit
+		d = rune(x[i])
+		switch {
+		case d == '_':
+			if p != '0' {
+				return i
+			}
+		case isDecimal(d) || x1 == 'x' && isHex(d):
+			d = '0'
+		default:
+			if p == '_' {
+				return i - 1
+			}
+			d = '.'
+		}
+	}
+	if d == '_' {
+		return len(x) - 1
+	}
+
+	return -1
+}
+
+func (s *scanner) rune() {
+	ok := true
+	s.nextch()
+
+	n := 0
+	for ; ; n++ {
+		if s.ch == '\'' {
+			if ok {
+				if n == 0 {
+					s.errorf("empty rune literal or unescaped '")
+					ok = false
+				} else if n != 1 {
+					s.errorAtf(0, "more than one character in rune literal")
+					ok = false
+				}
+			}
+			s.nextch()
+			break
+		}
+		if s.ch == '\\' {
+			s.nextch()
+			if !s.escape('\'') {
+				ok = false
+			}
+			continue
+		}
+		if s.ch == '\n' {
+			if ok {
+				s.errorf("newline in rune literal")
+				ok = false
+			}
+			break
+		}
+		if s.ch < 0 {
+			if ok {
+				s.errorAtf(0, "rune literal not terminated")
+				ok = false
+			}
+			break
+		}
+		s.nextch()
+	}
+
+	s.setLit(RuneLit, ok)
+}
+
+func (s *scanner) stdString() {
+	ok := true
+	s.nextch()
+
+	for {
+		if s.ch == '"' {
+			s.nextch()
+			break
+		}
+		if s.ch == '\\' {
+			s.nextch()
+			if !s.escape('"') {
+				ok = false
+			}
+			continue
+		}
+		if s.ch == '\n' {
+			s.errorf("newline in string")
+			ok = false
+			break
+		}
+		if s.ch < 0 {
+			s.errorAtf(0, "string not terminated")
+			ok = false
+			break
+		}
+		s.nextch()
+	}
+
+	s.setLit(StringLit, ok)
+}
+
+func (s *scanner) rawString() {
+	ok := true
+	s.nextch()
+
+	for {
+		if s.ch == '`' {
+			s.nextch()
+			break
+		}
+		if s.ch < 0 {
+			s.errorAtf(0, "string not terminated")
+			ok = false
+			break
+		}
+		s.nextch()
+	}
+	// We leave CRs in the string since they are part of the
+	// literal (even though they are not part of the literal
+	// value).
+
+	s.setLit(StringLit, ok)
+}
+
+func (s *scanner) comment(text string) {
+	s.errorAtf(0, "%s", text)
+}
+
+func (s *scanner) skipLine() {
+	// don't consume '\n' - needed for nlsemi logic
+	for s.ch >= 0 && s.ch != '\n' {
+		s.nextch()
+	}
+}
+
+func (s *scanner) lineComment() {
+	// opening has already been consumed
+
+	if s.mode&comments != 0 {
+		s.skipLine()
+		s.comment(string(s.segment()))
+		return
+	}
+
+	// are we saving directives? or is this definitely not a directive?
+	if s.mode&directives == 0 || (s.ch != 'g' && s.ch != 'l') {
+		s.stop()
+		s.skipLine()
+		return
+	}
+
+	// recognize go: or line directives
+	prefix := "go:"
+	if s.ch == 'l' {
+		prefix = "line "
+	}
+	for _, m := range prefix {
+		if s.ch != m {
+			s.stop()
+			s.skipLine()
+			return
+		}
+		s.nextch()
+	}
+
+	// directive text
+	s.skipLine()
+	s.comment(string(s.segment()))
+}
+
+func (s *scanner) skipComment() bool {
+	for s.ch >= 0 {
+		for s.ch == '*' {
+			s.nextch()
+			if s.ch == '/' {
+				s.nextch()
+				return true
+			}
+		}
+		s.nextch()
+	}
+	s.errorAtf(0, "comment not terminated")
+	return false
+}
+
+func (s *scanner) fullComment() {
+	/* opening has already been consumed */
+
+	if s.mode&comments != 0 {
+		if s.skipComment() {
+			s.comment(string(s.segment()))
+		}
+		return
+	}
+
+	if s.mode&directives == 0 || s.ch != 'l' {
+		s.stop()
+		s.skipComment()
+		return
+	}
+
+	// recognize line directive
+	const prefix = "line "
+	for _, m := range prefix {
+		if s.ch != m {
+			s.stop()
+			s.skipComment()
+			return
+		}
+		s.nextch()
+	}
+
+	// directive text
+	if s.skipComment() {
+		s.comment(string(s.segment()))
+	}
+}
+
+func (s *scanner) escape(quote rune) bool {
+	var n int
+	var base, max uint32
+
+	switch s.ch {
+	case quote, 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\':
+		s.nextch()
+		return true
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		n, base, max = 3, 8, 255
+	case 'x':
+		s.nextch()
+		n, base, max = 2, 16, 255
+	case 'u':
+		s.nextch()
+		n, base, max = 4, 16, unicode.MaxRune
+	case 'U':
+		s.nextch()
+		n, base, max = 8, 16, unicode.MaxRune
+	default:
+		if s.ch < 0 {
+			return true // complain in caller about EOF
+		}
+		s.errorf("unknown escape")
+		return false
+	}
+
+	var x uint32
+	for i := n; i > 0; i-- {
+		if s.ch < 0 {
+			return true // complain in caller about EOF
+		}
+		d := base
+		if isDecimal(s.ch) {
+			d = uint32(s.ch) - '0'
+		} else if 'a' <= lower(s.ch) && lower(s.ch) <= 'f' {
+			d = uint32(lower(s.ch)) - 'a' + 10
+		}
+		if d >= base {
+			s.errorf("invalid character %q in %s escape", s.ch, baseName(int(base)))
+			return false
+		}
+		// d < base
+		x = x*base + d
+		s.nextch()
+	}
+
+	if x > max && base == 8 {
+		s.errorf("octal escape value %d > 255", x)
+		return false
+	}
+
+	if x > max || 0xD800 <= x && x < 0xE000 /* surrogate range */ {
+		s.errorf("escape is invalid Unicode code point %#U", x)
+		return false
+	}
+
+	return true
+}