URL https://opencores.org/ocsvn/openrisc/openrisc/trunk

Subversion Repositories openrisc

[/] [openrisc/] [trunk/] [gnu-dev/] [or1k-gcc/] [libgo/] [go/] [text/] [scanner/] [scanner_test.go] - Blame information for rev 775

Go to most recent revision | Details | Compare with Previous | View Log


// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
 
package scanner
 
import (
        "bytes"
        "fmt"
        "io"
        "strings"
        "testing"
        "unicode/utf8"
)
 
// A StringReader delivers its data one string segment at a time via Read.
type StringReader struct {
        data []string
        step int
}
 
func (r *StringReader) Read(p []byte) (n int, err error) {
        if r.step < len(r.data) {
                s := r.data[r.step]
                n = copy(p, s)
                r.step++
        } else {
                err = io.EOF
        }
        return
}
 
func readRuneSegments(t *testing.T, segments []string) {
        got := ""
        want := strings.Join(segments, "")
        s := new(Scanner).Init(&StringReader{data: segments})
        for {
                ch := s.Next()
                if ch == EOF {
                        break
                }
                got += string(ch)
        }
        if got != want {
                t.Errorf("segments=%v got=%s want=%s", segments, got, want)
        }
}
 
var segmentList = [][]string{
        {},
        {""},
        {"日", "本語"},
        {"\u65e5", "\u672c", "\u8a9e"},
        {"\U000065e5", " ", "\U0000672c", "\U00008a9e"},
        {"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"},
        {"Hello", ", ", "World", "!"},
        {"Hello", ", ", "", "World", "!"},
}
 
func TestNext(t *testing.T) {
        for _, s := range segmentList {
                readRuneSegments(t, s)
        }
}
 
type token struct {
        tok  rune
        text string
}
 
var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
 
var tokenList = []token{
        {Comment, "// line comments"},
        {Comment, "//"},
        {Comment, "////"},
        {Comment, "// comment"},
        {Comment, "// /* comment */"},
        {Comment, "// // comment //"},
        {Comment, "//" + f100},
 
        {Comment, "// general comments"},
        {Comment, "/**/"},
        {Comment, "/***/"},
        {Comment, "/* comment */"},
        {Comment, "/* // comment */"},
        {Comment, "/* /* comment */"},
        {Comment, "/*\n comment\n*/"},
        {Comment, "/*" + f100 + "*/"},
 
        {Comment, "// identifiers"},
        {Ident, "a"},
        {Ident, "a0"},
        {Ident, "foobar"},
        {Ident, "abc123"},
        {Ident, "LGTM"},
        {Ident, "_"},
        {Ident, "_abc123"},
        {Ident, "abc123_"},
        {Ident, "_abc_123_"},
        {Ident, "_äöü"},
        {Ident, "_本"},
        {Ident, "äöü"},
        {Ident, "本"},
        {Ident, "a۰۱۸"},
        {Ident, "foo६४"},
        {Ident, "bar９８７６"},
        {Ident, f100},
 
        {Comment, "// decimal ints"},
        {Int, "0"},
        {Int, "1"},
        {Int, "9"},
        {Int, "42"},
        {Int, "1234567890"},
 
        {Comment, "// octal ints"},
        {Int, "00"},
        {Int, "01"},
        {Int, "07"},
        {Int, "042"},
        {Int, "01234567"},
 
        {Comment, "// hexadecimal ints"},
        {Int, "0x0"},
        {Int, "0x1"},
        {Int, "0xf"},
        {Int, "0x42"},
        {Int, "0x123456789abcDEF"},
        {Int, "0x" + f100},
        {Int, "0X0"},
        {Int, "0X1"},
        {Int, "0XF"},
        {Int, "0X42"},
        {Int, "0X123456789abcDEF"},
        {Int, "0X" + f100},
 
        {Comment, "// floats"},
        {Float, "0."},
        {Float, "1."},
        {Float, "42."},
        {Float, "01234567890."},
        {Float, ".0"},
        {Float, ".1"},
        {Float, ".42"},
        {Float, ".0123456789"},
        {Float, "0.0"},
        {Float, "1.0"},
        {Float, "42.0"},
        {Float, "01234567890.0"},
        {Float, "0e0"},
        {Float, "1e0"},
        {Float, "42e0"},
        {Float, "01234567890e0"},
        {Float, "0E0"},
        {Float, "1E0"},
        {Float, "42E0"},
        {Float, "01234567890E0"},
        {Float, "0e+10"},
        {Float, "1e-10"},
        {Float, "42e+10"},
        {Float, "01234567890e-10"},
        {Float, "0E+10"},
        {Float, "1E-10"},
        {Float, "42E+10"},
        {Float, "01234567890E-10"},
 
        {Comment, "// chars"},
        {Char, `' '`},
        {Char, `'a'`},
        {Char, `'本'`},
        {Char, `'\a'`},
        {Char, `'\b'`},
        {Char, `'\f'`},
        {Char, `'\n'`},
        {Char, `'\r'`},
        {Char, `'\t'`},
        {Char, `'\v'`},
        {Char, `'\''`},
        {Char, `'\000'`},
        {Char, `'\777'`},
        {Char, `'\x00'`},
        {Char, `'\xff'`},
        {Char, `'\u0000'`},
        {Char, `'\ufA16'`},
        {Char, `'\U00000000'`},
        {Char, `'\U0000ffAB'`},
 
        {Comment, "// strings"},
        {String, `" "`},
        {String, `"a"`},
        {String, `"本"`},
        {String, `"\a"`},
        {String, `"\b"`},
        {String, `"\f"`},
        {String, `"\n"`},
        {String, `"\r"`},
        {String, `"\t"`},
        {String, `"\v"`},
        {String, `"\""`},
        {String, `"\000"`},
        {String, `"\777"`},
        {String, `"\x00"`},
        {String, `"\xff"`},
        {String, `"\u0000"`},
        {String, `"\ufA16"`},
        {String, `"\U00000000"`},
        {String, `"\U0000ffAB"`},
        {String, `"` + f100 + `"`},
 
        {Comment, "// raw strings"},
        {String, "``"},
        {String, "`\\`"},
        {String, "`" + "\n\n/* foobar */\n\n" + "`"},
        {String, "`" + f100 + "`"},
 
        {Comment, "// individual characters"},
        // NUL character is not allowed
        {'\x01', "\x01"},
        {' ' - 1, string(' ' - 1)},
        {'+', "+"},
        {'/', "/"},
        {'.', "."},
        {'~', "~"},
        {'(', "("},
}
 
func makeSource(pattern string) *bytes.Buffer {
        var buf bytes.Buffer
        for _, k := range tokenList {
                fmt.Fprintf(&buf, pattern, k.text)
        }
        return &buf
}
 
func checkTok(t *testing.T, s *Scanner, line int, got, want rune, text string) {
        if got != want {
                t.Fatalf("tok = %s, want %s for %q", TokenString(got), TokenString(want), text)
        }
        if s.Line != line {
                t.Errorf("line = %d, want %d for %q", s.Line, line, text)
        }
        stext := s.TokenText()
        if stext != text {
                t.Errorf("text = %q, want %q", stext, text)
        } else {
                // check idempotency of TokenText() call
                stext = s.TokenText()
                if stext != text {
                        t.Errorf("text = %q, want %q (idempotency check)", stext, text)
                }
        }
}
 
func countNewlines(s string) int {
        n := 0
        for _, ch := range s {
                if ch == '\n' {
                        n++
                }
        }
        return n
}
 
func testScan(t *testing.T, mode uint) {
        s := new(Scanner).Init(makeSource(" \t%s\n"))
        s.Mode = mode
        tok := s.Scan()
        line := 1
        for _, k := range tokenList {
                if mode&SkipComments == 0 || k.tok != Comment {
                        checkTok(t, s, line, tok, k.tok, k.text)
                        tok = s.Scan()
                }
                line += countNewlines(k.text) + 1 // each token is on a new line
        }
        checkTok(t, s, line, tok, EOF, "")
}
 
func TestScan(t *testing.T) {
        testScan(t, GoTokens)
        testScan(t, GoTokens&^SkipComments)
}
 
func TestPosition(t *testing.T) {
        src := makeSource("\t\t\t\t%s\n")
        s := new(Scanner).Init(src)
        s.Mode = GoTokens &^ SkipComments
        s.Scan()
        pos := Position{"", 4, 1, 5}
        for _, k := range tokenList {
                if s.Offset != pos.Offset {
                        t.Errorf("offset = %d, want %d for %q", s.Offset, pos.Offset, k.text)
                }
                if s.Line != pos.Line {
                        t.Errorf("line = %d, want %d for %q", s.Line, pos.Line, k.text)
                }
                if s.Column != pos.Column {
                        t.Errorf("column = %d, want %d for %q", s.Column, pos.Column, k.text)
                }
                pos.Offset += 4 + len(k.text) + 1     // 4 tabs + token bytes + newline
                pos.Line += countNewlines(k.text) + 1 // each token is on a new line
                s.Scan()
        }
        // make sure there were no token-internal errors reported by scanner
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
}
 
func TestScanZeroMode(t *testing.T) {
        src := makeSource("%s\n")
        str := src.String()
        s := new(Scanner).Init(src)
        s.Mode = 0       // don't recognize any token classes
        s.Whitespace = 0 // don't skip any whitespace
        tok := s.Scan()
        for i, ch := range str {
                if tok != ch {
                        t.Fatalf("%d. tok = %s, want %s", i, TokenString(tok), TokenString(ch))
                }
                tok = s.Scan()
        }
        if tok != EOF {
                t.Fatalf("tok = %s, want EOF", TokenString(tok))
        }
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
}
 
func testScanSelectedMode(t *testing.T, mode uint, class rune) {
        src := makeSource("%s\n")
        s := new(Scanner).Init(src)
        s.Mode = mode
        tok := s.Scan()
        for tok != EOF {
                if tok < 0 && tok != class {
                        t.Fatalf("tok = %s, want %s", TokenString(tok), TokenString(class))
                }
                tok = s.Scan()
        }
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
}
 
func TestScanSelectedMask(t *testing.T) {
        testScanSelectedMode(t, 0, 0)
        testScanSelectedMode(t, ScanIdents, Ident)
        // Don't test ScanInts and ScanNumbers since some parts of
        // the floats in the source look like (illegal) octal ints
        // and ScanNumbers may return either Int or Float.
        testScanSelectedMode(t, ScanChars, Char)
        testScanSelectedMode(t, ScanStrings, String)
        testScanSelectedMode(t, SkipComments, 0)
        testScanSelectedMode(t, ScanComments, Comment)
}
 
func TestScanNext(t *testing.T) {
        s := new(Scanner).Init(bytes.NewBufferString("if a == bcd /* comment */ {\n\ta += c\n} // line comment ending in eof"))
        checkTok(t, s, 1, s.Scan(), Ident, "if")
        checkTok(t, s, 1, s.Scan(), Ident, "a")
        checkTok(t, s, 1, s.Scan(), '=', "=")
        checkTok(t, s, 0, s.Next(), '=', "")
        checkTok(t, s, 0, s.Next(), ' ', "")
        checkTok(t, s, 0, s.Next(), 'b', "")
        checkTok(t, s, 1, s.Scan(), Ident, "cd")
        checkTok(t, s, 1, s.Scan(), '{', "{")
        checkTok(t, s, 2, s.Scan(), Ident, "a")
        checkTok(t, s, 2, s.Scan(), '+', "+")
        checkTok(t, s, 0, s.Next(), '=', "")
        checkTok(t, s, 2, s.Scan(), Ident, "c")
        checkTok(t, s, 3, s.Scan(), '}', "}")
        checkTok(t, s, 3, s.Scan(), -1, "")
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
}
 
func TestScanWhitespace(t *testing.T) {
        var buf bytes.Buffer
        var ws uint64
        // start at 1, NUL character is not allowed
        for ch := byte(1); ch < ' '; ch++ {
                buf.WriteByte(ch)
                ws |= 1 << ch
        }
        const orig = 'x'
        buf.WriteByte(orig)
 
        s := new(Scanner).Init(&buf)
        s.Mode = 0
        s.Whitespace = ws
        tok := s.Scan()
        if tok != orig {
                t.Errorf("tok = %s, want %s", TokenString(tok), TokenString(orig))
        }
}
 
func testError(t *testing.T, src, pos, msg string, tok rune) {
        s := new(Scanner).Init(bytes.NewBufferString(src))
        errorCalled := false
        s.Error = func(s *Scanner, m string) {
                if !errorCalled {
                        // only look at first error
                        if p := s.Pos().String(); p != pos {
                                t.Errorf("pos = %q, want %q for %q", p, pos, src)
                        }
                        if m != msg {
                                t.Errorf("msg = %q, want %q for %q", m, msg, src)
                        }
                        errorCalled = true
                }
        }
        tk := s.Scan()
        if tk != tok {
                t.Errorf("tok = %s, want %s for %q", TokenString(tk), TokenString(tok), src)
        }
        if !errorCalled {
                t.Errorf("error handler not called for %q", src)
        }
        if s.ErrorCount == 0 {
                t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src)
        }
}
 
func TestError(t *testing.T) {
        testError(t, "\x00", "1:1", "illegal character NUL", 0)
        testError(t, "\x80", "1:1", "illegal UTF-8 encoding", utf8.RuneError)
        testError(t, "\xff", "1:1", "illegal UTF-8 encoding", utf8.RuneError)
 
        testError(t, "a\x00", "1:2", "illegal character NUL", Ident)
        testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", Ident)
        testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", Ident)
 
        testError(t, `"a`+"\x00", "1:3", "illegal character NUL", String)
        testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", String)
        testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", String)
 
        testError(t, "`a"+"\x00", "1:3", "illegal character NUL", String)
        testError(t, "`ab"+"\x80", "1:4", "illegal UTF-8 encoding", String)
        testError(t, "`abc"+"\xff", "1:5", "illegal UTF-8 encoding", String)
 
        testError(t, `'\"'`, "1:3", "illegal char escape", Char)
        testError(t, `"\'"`, "1:3", "illegal char escape", String)
 
        testError(t, `01238`, "1:6", "illegal octal number", Int)
        testError(t, `'aa'`, "1:4", "illegal char literal", Char)
 
        testError(t, `'`, "1:2", "literal not terminated", Char)
        testError(t, `'`+"\n", "1:2", "literal not terminated", Char)
        testError(t, `"abc`, "1:5", "literal not terminated", String)
        testError(t, `"abc`+"\n", "1:5", "literal not terminated", String)
        testError(t, "`abc\n", "2:1", "literal not terminated", String)
        testError(t, `/*/`, "1:4", "comment not terminated", EOF)
}
 
func checkPos(t *testing.T, got, want Position) {
        if got.Offset != want.Offset || got.Line != want.Line || got.Column != want.Column {
                t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d",
                        got.Offset, got.Line, got.Column, want.Offset, want.Line, want.Column)
        }
}
 
func checkNextPos(t *testing.T, s *Scanner, offset, line, column int, char rune) {
        if ch := s.Next(); ch != char {
                t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
        }
        want := Position{Offset: offset, Line: line, Column: column}
        checkPos(t, s.Pos(), want)
}
 
func checkScanPos(t *testing.T, s *Scanner, offset, line, column int, char rune) {
        want := Position{Offset: offset, Line: line, Column: column}
        checkPos(t, s.Pos(), want)
        if ch := s.Scan(); ch != char {
                t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
                if string(ch) != s.TokenText() {
                        t.Errorf("tok = %q, want %q", s.TokenText(), string(ch))
                }
        }
        checkPos(t, s.Position, want)
}
 
func TestPos(t *testing.T) {
        // corner case: empty source
        s := new(Scanner).Init(bytes.NewBufferString(""))
        checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
        s.Peek() // peek doesn't affect the position
        checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
 
        // corner case: source with only a newline
        s = new(Scanner).Init(bytes.NewBufferString("\n"))
        checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
        checkNextPos(t, s, 1, 2, 1, '\n')
        // after EOF position doesn't change
        for i := 10; i > 0; i-- {
                checkScanPos(t, s, 1, 2, 1, EOF)
        }
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
 
        // corner case: source with only a single character
        s = new(Scanner).Init(bytes.NewBufferString("本"))
        checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
        checkNextPos(t, s, 3, 1, 2, '本')
        // after EOF position doesn't change
        for i := 10; i > 0; i-- {
                checkScanPos(t, s, 3, 1, 2, EOF)
        }
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
 
        // positions after calling Next
        s = new(Scanner).Init(bytes.NewBufferString("  foo६४  \n\n本語\n"))
        checkNextPos(t, s, 1, 1, 2, ' ')
        s.Peek() // peek doesn't affect the position
        checkNextPos(t, s, 2, 1, 3, ' ')
        checkNextPos(t, s, 3, 1, 4, 'f')
        checkNextPos(t, s, 4, 1, 5, 'o')
        checkNextPos(t, s, 5, 1, 6, 'o')
        checkNextPos(t, s, 8, 1, 7, '६')
        checkNextPos(t, s, 11, 1, 8, '४')
        checkNextPos(t, s, 12, 1, 9, ' ')
        checkNextPos(t, s, 13, 1, 10, ' ')
        checkNextPos(t, s, 14, 2, 1, '\n')
        checkNextPos(t, s, 15, 3, 1, '\n')
        checkNextPos(t, s, 18, 3, 2, '本')
        checkNextPos(t, s, 21, 3, 3, '語')
        checkNextPos(t, s, 22, 4, 1, '\n')
        // after EOF position doesn't change
        for i := 10; i > 0; i-- {
                checkScanPos(t, s, 22, 4, 1, EOF)
        }
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
 
        // positions after calling Scan
        s = new(Scanner).Init(bytes.NewBufferString("abc\n本語\n\nx"))
        s.Mode = 0
        s.Whitespace = 0
        checkScanPos(t, s, 0, 1, 1, 'a')
        s.Peek() // peek doesn't affect the position
        checkScanPos(t, s, 1, 1, 2, 'b')
        checkScanPos(t, s, 2, 1, 3, 'c')
        checkScanPos(t, s, 3, 1, 4, '\n')
        checkScanPos(t, s, 4, 2, 1, '本')
        checkScanPos(t, s, 7, 2, 2, '語')
        checkScanPos(t, s, 10, 2, 3, '\n')
        checkScanPos(t, s, 11, 3, 1, '\n')
        checkScanPos(t, s, 12, 4, 1, 'x')
        // after EOF position doesn't change
        for i := 10; i > 0; i-- {
                checkScanPos(t, s, 13, 4, 2, EOF)
        }
        if s.ErrorCount != 0 {
                t.Errorf("%d errors", s.ErrorCount)
        }
}

Browse

Tools

Subversion Repositories openrisc

[/] [openrisc/] [trunk/] [gnu-dev/] [or1k-gcc/] [libgo/] [go/] [text/] [scanner/] [scanner_test.go] - Blame information for rev 775

Line No.	Rev	Author	Line
1	747	jeremybenn	`// Copyright 2009 The Go Authors. All rights reserved.`
2			`// Use of this source code is governed by a BSD-style`
3			`// license that can be found in the LICENSE file.`
4
5			`package scanner`
6
7			`import (`
8			`"bytes"`
9			`"fmt"`
10			`"io"`
11			`"strings"`
12			`"testing"`
13			`"unicode/utf8"`
14			`)`
15
16			`// A StringReader delivers its data one string segment at a time via Read.`
17			`type StringReader struct {`
18			`data []string`
19			`step int`
20			`}`
21
22			`func (r *StringReader) Read(p []byte) (n int, err error) {`
23			`if r.step < len(r.data) {`
24			`s := r.data[r.step]`
25			`n = copy(p, s)`
26			`r.step++`
27			`} else {`
28			`err = io.EOF`
29			`}`
30			`return`
31			`}`
32
33			`func readRuneSegments(t *testing.T, segments []string) {`
34			`got := ""`
35			`want := strings.Join(segments, "")`
36			`s := new(Scanner).Init(&StringReader{data: segments})`
37			`for {`
38			`ch := s.Next()`
39			`if ch == EOF {`
40			`break`
41			`}`
42			`got += string(ch)`
43			`}`
44			`if got != want {`
45			`t.Errorf("segments=%v got=%s want=%s", segments, got, want)`
46			`}`
47			`}`
48
49			`var segmentList = [][]string{`
50			`{},`
51			`{""},`
52			`{"日", "本語"},`
53			`{"\u65e5", "\u672c", "\u8a9e"},`
54			`{"\U000065e5", " ", "\U0000672c", "\U00008a9e"},`
55			`{"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"},`
56			`{"Hello", ", ", "World", "!"},`
57			`{"Hello", ", ", "", "World", "!"},`
58			`}`
59
60			`func TestNext(t *testing.T) {`
61			`for _, s := range segmentList {`
62			`readRuneSegments(t, s)`
63			`}`
64			`}`
65
66			`type token struct {`
67			`tok rune`
68			`text string`
69			`}`
70
71			`var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"`
72
73			`var tokenList = []token{`
74			`{Comment, "// line comments"},`
75			`{Comment, "//"},`
76			`{Comment, "////"},`
77			`{Comment, "// comment"},`
78			`{Comment, "// /* comment */"},`
79			`{Comment, "// // comment //"},`
80			`{Comment, "//" + f100},`
81
82			`{Comment, "// general comments"},`
83			`{Comment, "/**/"},`
84			`{Comment, "/***/"},`
85			`{Comment, "/* comment */"},`
86			`{Comment, "/* // comment */"},`
87			`{Comment, "/* /* comment */"},`
88			`{Comment, "/\n comment\n/"},`
89			`{Comment, "/" + f100 + "/"},`
90
91			`{Comment, "// identifiers"},`
92			`{Ident, "a"},`
93			`{Ident, "a0"},`
94			`{Ident, "foobar"},`
95			`{Ident, "abc123"},`
96			`{Ident, "LGTM"},`
97			`{Ident, "_"},`
98			`{Ident, "_abc123"},`
99			`{Ident, "abc123_"},`
100			`{Ident, "_abc_123_"},`
101			`{Ident, "_äöü"},`
102			`{Ident, "_本"},`
103			`{Ident, "äöü"},`
104			`{Ident, "本"},`
105			`{Ident, "a۰۱۸"},`
106			`{Ident, "foo६४"},`
107			`{Ident, "bar９８７６"},`
108			`{Ident, f100},`
109
110			`{Comment, "// decimal ints"},`
111			`{Int, "0"},`
112			`{Int, "1"},`
113			`{Int, "9"},`
114			`{Int, "42"},`
115			`{Int, "1234567890"},`
116
117			`{Comment, "// octal ints"},`
118			`{Int, "00"},`
119			`{Int, "01"},`
120			`{Int, "07"},`
121			`{Int, "042"},`
122			`{Int, "01234567"},`
123
124			`{Comment, "// hexadecimal ints"},`
125			`{Int, "0x0"},`
126			`{Int, "0x1"},`
127			`{Int, "0xf"},`
128			`{Int, "0x42"},`
129			`{Int, "0x123456789abcDEF"},`
130			`{Int, "0x" + f100},`
131			`{Int, "0X0"},`
132			`{Int, "0X1"},`
133			`{Int, "0XF"},`
134			`{Int, "0X42"},`
135			`{Int, "0X123456789abcDEF"},`
136			`{Int, "0X" + f100},`
137
138			`{Comment, "// floats"},`
139			`{Float, "0."},`
140			`{Float, "1."},`
141			`{Float, "42."},`
142			`{Float, "01234567890."},`
143			`{Float, ".0"},`
144			`{Float, ".1"},`
145			`{Float, ".42"},`
146			`{Float, ".0123456789"},`
147			`{Float, "0.0"},`
148			`{Float, "1.0"},`
149			`{Float, "42.0"},`
150			`{Float, "01234567890.0"},`
151			`{Float, "0e0"},`
152			`{Float, "1e0"},`
153			`{Float, "42e0"},`
154			`{Float, "01234567890e0"},`
155			`{Float, "0E0"},`
156			`{Float, "1E0"},`
157			`{Float, "42E0"},`
158			`{Float, "01234567890E0"},`
159			`{Float, "0e+10"},`
160			`{Float, "1e-10"},`
161			`{Float, "42e+10"},`
162			`{Float, "01234567890e-10"},`
163			`{Float, "0E+10"},`
164			`{Float, "1E-10"},`
165			`{Float, "42E+10"},`
166			`{Float, "01234567890E-10"},`
167
168			`{Comment, "// chars"},`
169			{Char, `' '`},
170			{Char, `'a'`},
171			{Char, `'本'`},
172			{Char, `'\a'`},
173			{Char, `'\b'`},
174			{Char, `'\f'`},
175			{Char, `'\n'`},
176			{Char, `'\r'`},
177			{Char, `'\t'`},
178			{Char, `'\v'`},
179			{Char, `'\''`},
180			{Char, `'\000'`},
181			{Char, `'\777'`},
182			{Char, `'\x00'`},
183			{Char, `'\xff'`},
184			{Char, `'\u0000'`},
185			{Char, `'\ufA16'`},
186			{Char, `'\U00000000'`},
187			{Char, `'\U0000ffAB'`},
188
189			`{Comment, "// strings"},`
190			{String, `" "`},
191			{String, `"a"`},
192			{String, `"本"`},
193			{String, `"\a"`},
194			{String, `"\b"`},
195			{String, `"\f"`},
196			{String, `"\n"`},
197			{String, `"\r"`},
198			{String, `"\t"`},
199			{String, `"\v"`},
200			{String, `"\""`},
201			{String, `"\000"`},
202			{String, `"\777"`},
203			{String, `"\x00"`},
204			{String, `"\xff"`},
205			{String, `"\u0000"`},
206			{String, `"\ufA16"`},
207			{String, `"\U00000000"`},
208			{String, `"\U0000ffAB"`},
209			{String, `"` + f100 + `"`},
210
211			`{Comment, "// raw strings"},`
212			{String, "``"},
213			{String, "`\\`"},
214			{String, "`" + "\n\n/* foobar */\n\n" + "`"},
215			{String, "`" + f100 + "`"},
216
217			`{Comment, "// individual characters"},`
218			`// NUL character is not allowed`
219			`{'\x01', "\x01"},`
220			`{' ' - 1, string(' ' - 1)},`
221			`{'+', "+"},`
222			`{'/', "/"},`
223			`{'.', "."},`
224			`{'~', "~"},`
225			`{'(', "("},`
226			`}`
227
228			`func makeSource(pattern string) *bytes.Buffer {`
229			`var buf bytes.Buffer`
230			`for _, k := range tokenList {`
231			`fmt.Fprintf(&buf, pattern, k.text)`
232			`}`
233			`return &buf`
234			`}`
235
236			`func checkTok(t testing.T, s Scanner, line int, got, want rune, text string) {`
237			`if got != want {`
238			`t.Fatalf("tok = %s, want %s for %q", TokenString(got), TokenString(want), text)`
239			`}`
240			`if s.Line != line {`
241			`t.Errorf("line = %d, want %d for %q", s.Line, line, text)`
242			`}`
243			`stext := s.TokenText()`
244			`if stext != text {`
245			`t.Errorf("text = %q, want %q", stext, text)`
246			`} else {`
247			`// check idempotency of TokenText() call`
248			`stext = s.TokenText()`
249			`if stext != text {`
250			`t.Errorf("text = %q, want %q (idempotency check)", stext, text)`
251			`}`
252			`}`
253			`}`
254
255			`func countNewlines(s string) int {`
256			`n := 0`
257			`for _, ch := range s {`
258			`if ch == '\n' {`
259			`n++`
260			`}`
261			`}`
262			`return n`
263			`}`
264
265			`func testScan(t *testing.T, mode uint) {`
266			`s := new(Scanner).Init(makeSource(" \t%s\n"))`
267			`s.Mode = mode`
268			`tok := s.Scan()`
269			`line := 1`
270			`for _, k := range tokenList {`
271			`if mode&SkipComments == 0 \|\| k.tok != Comment {`
272			`checkTok(t, s, line, tok, k.tok, k.text)`
273			`tok = s.Scan()`
274			`}`
275			`line += countNewlines(k.text) + 1 // each token is on a new line`
276			`}`
277			`checkTok(t, s, line, tok, EOF, "")`
278			`}`
279
280			`func TestScan(t *testing.T) {`
281			`testScan(t, GoTokens)`
282			`testScan(t, GoTokens&^SkipComments)`
283			`}`
284
285			`func TestPosition(t *testing.T) {`
286			`src := makeSource("\t\t\t\t%s\n")`
287			`s := new(Scanner).Init(src)`
288			`s.Mode = GoTokens &^ SkipComments`
289			`s.Scan()`
290			`pos := Position{"", 4, 1, 5}`
291			`for _, k := range tokenList {`
292			`if s.Offset != pos.Offset {`
293			`t.Errorf("offset = %d, want %d for %q", s.Offset, pos.Offset, k.text)`
294			`}`
295			`if s.Line != pos.Line {`
296			`t.Errorf("line = %d, want %d for %q", s.Line, pos.Line, k.text)`
297			`}`
298			`if s.Column != pos.Column {`
299			`t.Errorf("column = %d, want %d for %q", s.Column, pos.Column, k.text)`
300			`}`
301			`pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline`
302			`pos.Line += countNewlines(k.text) + 1 // each token is on a new line`
303			`s.Scan()`
304			`}`
305			`// make sure there were no token-internal errors reported by scanner`
306			`if s.ErrorCount != 0 {`
307			`t.Errorf("%d errors", s.ErrorCount)`
308			`}`
309			`}`
310
311			`func TestScanZeroMode(t *testing.T) {`
312			`src := makeSource("%s\n")`
313			`str := src.String()`
314			`s := new(Scanner).Init(src)`
315			`s.Mode = 0 // don't recognize any token classes`
316			`s.Whitespace = 0 // don't skip any whitespace`
317			`tok := s.Scan()`
318			`for i, ch := range str {`
319			`if tok != ch {`
320			`t.Fatalf("%d. tok = %s, want %s", i, TokenString(tok), TokenString(ch))`
321			`}`
322			`tok = s.Scan()`
323			`}`
324			`if tok != EOF {`
325			`t.Fatalf("tok = %s, want EOF", TokenString(tok))`
326			`}`
327			`if s.ErrorCount != 0 {`
328			`t.Errorf("%d errors", s.ErrorCount)`
329			`}`
330			`}`
331
332			`func testScanSelectedMode(t *testing.T, mode uint, class rune) {`
333			`src := makeSource("%s\n")`
334			`s := new(Scanner).Init(src)`
335			`s.Mode = mode`
336			`tok := s.Scan()`
337			`for tok != EOF {`
338			`if tok < 0 && tok != class {`
339			`t.Fatalf("tok = %s, want %s", TokenString(tok), TokenString(class))`
340			`}`
341			`tok = s.Scan()`
342			`}`
343			`if s.ErrorCount != 0 {`
344			`t.Errorf("%d errors", s.ErrorCount)`
345			`}`
346			`}`
347
348			`func TestScanSelectedMask(t *testing.T) {`
349			`testScanSelectedMode(t, 0, 0)`
350			`testScanSelectedMode(t, ScanIdents, Ident)`
351			`// Don't test ScanInts and ScanNumbers since some parts of`
352			`// the floats in the source look like (illegal) octal ints`
353			`// and ScanNumbers may return either Int or Float.`
354			`testScanSelectedMode(t, ScanChars, Char)`
355			`testScanSelectedMode(t, ScanStrings, String)`
356			`testScanSelectedMode(t, SkipComments, 0)`
357			`testScanSelectedMode(t, ScanComments, Comment)`
358			`}`
359
360			`func TestScanNext(t *testing.T) {`
361			`s := new(Scanner).Init(bytes.NewBufferString("if a == bcd /* comment */ {\n\ta += c\n} // line comment ending in eof"))`
362			`checkTok(t, s, 1, s.Scan(), Ident, "if")`
363			`checkTok(t, s, 1, s.Scan(), Ident, "a")`
364			`checkTok(t, s, 1, s.Scan(), '=', "=")`
365			`checkTok(t, s, 0, s.Next(), '=', "")`
366			`checkTok(t, s, 0, s.Next(), ' ', "")`
367			`checkTok(t, s, 0, s.Next(), 'b', "")`
368			`checkTok(t, s, 1, s.Scan(), Ident, "cd")`
369			`checkTok(t, s, 1, s.Scan(), '{', "{")`
370			`checkTok(t, s, 2, s.Scan(), Ident, "a")`
371			`checkTok(t, s, 2, s.Scan(), '+', "+")`
372			`checkTok(t, s, 0, s.Next(), '=', "")`
373			`checkTok(t, s, 2, s.Scan(), Ident, "c")`
374			`checkTok(t, s, 3, s.Scan(), '}', "}")`
375			`checkTok(t, s, 3, s.Scan(), -1, "")`
376			`if s.ErrorCount != 0 {`
377			`t.Errorf("%d errors", s.ErrorCount)`
378			`}`
379			`}`
380
381			`func TestScanWhitespace(t *testing.T) {`
382			`var buf bytes.Buffer`
383			`var ws uint64`
384			`// start at 1, NUL character is not allowed`
385			`for ch := byte(1); ch < ' '; ch++ {`
386			`buf.WriteByte(ch)`
387			`ws \|= 1 << ch`
388			`}`
389			`const orig = 'x'`
390			`buf.WriteByte(orig)`
391
392			`s := new(Scanner).Init(&buf)`
393			`s.Mode = 0`
394			`s.Whitespace = ws`
395			`tok := s.Scan()`
396			`if tok != orig {`
397			`t.Errorf("tok = %s, want %s", TokenString(tok), TokenString(orig))`
398			`}`
399			`}`
400
401			`func testError(t *testing.T, src, pos, msg string, tok rune) {`
402			`s := new(Scanner).Init(bytes.NewBufferString(src))`
403			`errorCalled := false`
404			`s.Error = func(s *Scanner, m string) {`
405			`if !errorCalled {`
406			`// only look at first error`
407			`if p := s.Pos().String(); p != pos {`
408			`t.Errorf("pos = %q, want %q for %q", p, pos, src)`
409			`}`
410			`if m != msg {`
411			`t.Errorf("msg = %q, want %q for %q", m, msg, src)`
412			`}`
413			`errorCalled = true`
414			`}`
415			`}`
416			`tk := s.Scan()`
417			`if tk != tok {`
418			`t.Errorf("tok = %s, want %s for %q", TokenString(tk), TokenString(tok), src)`
419			`}`
420			`if !errorCalled {`
421			`t.Errorf("error handler not called for %q", src)`
422			`}`
423			`if s.ErrorCount == 0 {`
424			`t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src)`
425			`}`
426			`}`
427
428			`func TestError(t *testing.T) {`
429			`testError(t, "\x00", "1:1", "illegal character NUL", 0)`
430			`testError(t, "\x80", "1:1", "illegal UTF-8 encoding", utf8.RuneError)`
431			`testError(t, "\xff", "1:1", "illegal UTF-8 encoding", utf8.RuneError)`
432
433			`testError(t, "a\x00", "1:2", "illegal character NUL", Ident)`
434			`testError(t, "ab\x80", "1:3", "illegal UTF-8 encoding", Ident)`
435			`testError(t, "abc\xff", "1:4", "illegal UTF-8 encoding", Ident)`
436
437			testError(t, `"a`+"\x00", "1:3", "illegal character NUL", String)
438			testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", String)
439			testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", String)
440
441			testError(t, "`a"+"\x00", "1:3", "illegal character NUL", String)
442			testError(t, "`ab"+"\x80", "1:4", "illegal UTF-8 encoding", String)
443			testError(t, "`abc"+"\xff", "1:5", "illegal UTF-8 encoding", String)
444
445			testError(t, `'\"'`, "1:3", "illegal char escape", Char)
446			testError(t, `"\'"`, "1:3", "illegal char escape", String)
447
448			testError(t, `01238`, "1:6", "illegal octal number", Int)
449			testError(t, `'aa'`, "1:4", "illegal char literal", Char)
450
451			testError(t, `'`, "1:2", "literal not terminated", Char)
452			testError(t, `'`+"\n", "1:2", "literal not terminated", Char)
453			testError(t, `"abc`, "1:5", "literal not terminated", String)
454			testError(t, `"abc`+"\n", "1:5", "literal not terminated", String)
455			testError(t, "`abc\n", "2:1", "literal not terminated", String)
456			testError(t, `/*/`, "1:4", "comment not terminated", EOF)
457			`}`
458
459			`func checkPos(t *testing.T, got, want Position) {`
460			`if got.Offset != want.Offset \|\| got.Line != want.Line \|\| got.Column != want.Column {`
461			`t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d",`
462			`got.Offset, got.Line, got.Column, want.Offset, want.Line, want.Column)`
463			`}`
464			`}`
465
466			`func checkNextPos(t testing.T, s Scanner, offset, line, column int, char rune) {`
467			`if ch := s.Next(); ch != char {`
468			`t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))`
469			`}`
470			`want := Position{Offset: offset, Line: line, Column: column}`
471			`checkPos(t, s.Pos(), want)`
472			`}`
473
474			`func checkScanPos(t testing.T, s Scanner, offset, line, column int, char rune) {`
475			`want := Position{Offset: offset, Line: line, Column: column}`
476			`checkPos(t, s.Pos(), want)`
477			`if ch := s.Scan(); ch != char {`
478			`t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))`
479			`if string(ch) != s.TokenText() {`
480			`t.Errorf("tok = %q, want %q", s.TokenText(), string(ch))`
481			`}`
482			`}`
483			`checkPos(t, s.Position, want)`
484			`}`
485
486			`func TestPos(t *testing.T) {`
487			`// corner case: empty source`
488			`s := new(Scanner).Init(bytes.NewBufferString(""))`
489			`checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})`
490			`s.Peek() // peek doesn't affect the position`
491			`checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})`
492
493			`// corner case: source with only a newline`
494			`s = new(Scanner).Init(bytes.NewBufferString("\n"))`
495			`checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})`
496			`checkNextPos(t, s, 1, 2, 1, '\n')`
497			`// after EOF position doesn't change`
498			`for i := 10; i > 0; i-- {`
499			`checkScanPos(t, s, 1, 2, 1, EOF)`
500			`}`
501			`if s.ErrorCount != 0 {`
502			`t.Errorf("%d errors", s.ErrorCount)`
503			`}`
504
505			`// corner case: source with only a single character`
506			`s = new(Scanner).Init(bytes.NewBufferString("本"))`
507			`checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})`
508			`checkNextPos(t, s, 3, 1, 2, '本')`
509			`// after EOF position doesn't change`
510			`for i := 10; i > 0; i-- {`
511			`checkScanPos(t, s, 3, 1, 2, EOF)`
512			`}`
513			`if s.ErrorCount != 0 {`
514			`t.Errorf("%d errors", s.ErrorCount)`
515			`}`
516
517			`// positions after calling Next`
518			`s = new(Scanner).Init(bytes.NewBufferString(" foo६४ \n\n本語\n"))`
519			`checkNextPos(t, s, 1, 1, 2, ' ')`
520			`s.Peek() // peek doesn't affect the position`
521			`checkNextPos(t, s, 2, 1, 3, ' ')`
522			`checkNextPos(t, s, 3, 1, 4, 'f')`
523			`checkNextPos(t, s, 4, 1, 5, 'o')`
524			`checkNextPos(t, s, 5, 1, 6, 'o')`
525			`checkNextPos(t, s, 8, 1, 7, '६')`
526			`checkNextPos(t, s, 11, 1, 8, '४')`
527			`checkNextPos(t, s, 12, 1, 9, ' ')`
528			`checkNextPos(t, s, 13, 1, 10, ' ')`
529			`checkNextPos(t, s, 14, 2, 1, '\n')`
530			`checkNextPos(t, s, 15, 3, 1, '\n')`
531			`checkNextPos(t, s, 18, 3, 2, '本')`
532			`checkNextPos(t, s, 21, 3, 3, '語')`
533			`checkNextPos(t, s, 22, 4, 1, '\n')`
534			`// after EOF position doesn't change`
535			`for i := 10; i > 0; i-- {`
536			`checkScanPos(t, s, 22, 4, 1, EOF)`
537			`}`
538			`if s.ErrorCount != 0 {`
539			`t.Errorf("%d errors", s.ErrorCount)`
540			`}`
541
542			`// positions after calling Scan`
543			`s = new(Scanner).Init(bytes.NewBufferString("abc\n本語\n\nx"))`
544			`s.Mode = 0`
545			`s.Whitespace = 0`
546			`checkScanPos(t, s, 0, 1, 1, 'a')`
547			`s.Peek() // peek doesn't affect the position`
548			`checkScanPos(t, s, 1, 1, 2, 'b')`
549			`checkScanPos(t, s, 2, 1, 3, 'c')`
550			`checkScanPos(t, s, 3, 1, 4, '\n')`
551			`checkScanPos(t, s, 4, 2, 1, '本')`
552			`checkScanPos(t, s, 7, 2, 2, '語')`
553			`checkScanPos(t, s, 10, 2, 3, '\n')`
554			`checkScanPos(t, s, 11, 3, 1, '\n')`
555			`checkScanPos(t, s, 12, 4, 1, 'x')`
556			`// after EOF position doesn't change`
557			`for i := 10; i > 0; i-- {`
558			`checkScanPos(t, s, 13, 4, 2, EOF)`
559			`}`
560			`if s.ErrorCount != 0 {`
561			`t.Errorf("%d errors", s.ErrorCount)`
562			`}`
563			`}`