URL https://opencores.org/ocsvn/openrisc/openrisc/trunk

Subversion Repositories openrisc

[/] [openrisc/] [trunk/] [gnu-dev/] [or1k-gcc/] [libgo/] [go/] [text/] [template/] [parse/] [lex.go] - Blame information for rev 747

Details | Compare with Previous | View Log


// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
 
package parse
 
import (
        "fmt"
        "strings"
        "unicode"
        "unicode/utf8"
)
 
// item represents a token or text string returned from the scanner.
type item struct {
        typ itemType
        val string
}
 
func (i item) String() string {
        switch {
        case i.typ == itemEOF:
                return "EOF"
        case i.typ == itemError:
                return i.val
        case i.typ > itemKeyword:
                return fmt.Sprintf("<%s>", i.val)
        case len(i.val) > 10:
                return fmt.Sprintf("%.10q...", i.val)
        }
        return fmt.Sprintf("%q", i.val)
}
 
// itemType identifies the type of lex items.
type itemType int
 
const (
        itemError        itemType = iota // error occurred; value is text of error
        itemBool                         // boolean constant
        itemChar                         // printable ASCII character; grab bag for comma etc.
        itemCharConstant                 // character constant
        itemComplex                      // complex constant (1+2i); imaginary is just a number
        itemColonEquals                  // colon-equals (':=') introducing a declaration
        itemEOF
        itemField      // alphanumeric identifier, starting with '.', possibly chained ('.x.y')
        itemIdentifier // alphanumeric identifier
        itemLeftDelim  // left action delimiter
        itemNumber     // simple number, including imaginary
        itemPipe       // pipe symbol
        itemRawString  // raw quoted string (includes quotes)
        itemRightDelim // right action delimiter
        itemString     // quoted string (includes quotes)
        itemText       // plain text
        itemVariable   // variable starting with '$', such as '$' or  '$1' or '$hello'.
        // Keywords appear after all the rest.
        itemKeyword  // used only to delimit the keywords
        itemDot      // the cursor, spelled '.'.
        itemDefine   // define keyword
        itemElse     // else keyword
        itemEnd      // end keyword
        itemIf       // if keyword
        itemRange    // range keyword
        itemTemplate // template keyword
        itemWith     // with keyword
)
 
// Make the types prettyprint.
var itemName = map[itemType]string{
        itemError:        "error",
        itemBool:         "bool",
        itemChar:         "char",
        itemCharConstant: "charconst",
        itemComplex:      "complex",
        itemColonEquals:  ":=",
        itemEOF:          "EOF",
        itemField:        "field",
        itemIdentifier:   "identifier",
        itemLeftDelim:    "left delim",
        itemNumber:       "number",
        itemPipe:         "pipe",
        itemRawString:    "raw string",
        itemRightDelim:   "right delim",
        itemString:       "string",
        itemVariable:     "variable",
        // keywords
        itemDot:      ".",
        itemDefine:   "define",
        itemElse:     "else",
        itemIf:       "if",
        itemEnd:      "end",
        itemRange:    "range",
        itemTemplate: "template",
        itemWith:     "with",
}
 
func (i itemType) String() string {
        s := itemName[i]
        if s == "" {
                return fmt.Sprintf("item%d", int(i))
        }
        return s
}
 
var key = map[string]itemType{
        ".":        itemDot,
        "define":   itemDefine,
        "else":     itemElse,
        "end":      itemEnd,
        "if":       itemIf,
        "range":    itemRange,
        "template": itemTemplate,
        "with":     itemWith,
}
 
const eof = -1
 
// stateFn represents the state of the scanner as a function that returns the next state.
type stateFn func(*lexer) stateFn
 
// lexer holds the state of the scanner.
type lexer struct {
        name       string    // the name of the input; used only for error reports.
        input      string    // the string being scanned.
        leftDelim  string    // start of action.
        rightDelim string    // end of action.
        state      stateFn   // the next lexing function to enter.
        pos        int       // current position in the input.
        start      int       // start position of this item.
        width      int       // width of last rune read from input.
        items      chan item // channel of scanned items.
}
 
// next returns the next rune in the input.
func (l *lexer) next() (r rune) {
        if l.pos >= len(l.input) {
                l.width = 0
                return eof
        }
        r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
        l.pos += l.width
        return r
}
 
// peek returns but does not consume the next rune in the input.
func (l *lexer) peek() rune {
        r := l.next()
        l.backup()
        return r
}
 
// backup steps back one rune. Can only be called once per call of next.
func (l *lexer) backup() {
        l.pos -= l.width
}
 
// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
        l.items <- item{t, l.input[l.start:l.pos]}
        l.start = l.pos
}
 
// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
        l.start = l.pos
}
 
// accept consumes the next rune if it's from the valid set.
func (l *lexer) accept(valid string) bool {
        if strings.IndexRune(valid, l.next()) >= 0 {
                return true
        }
        l.backup()
        return false
}
 
// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
        for strings.IndexRune(valid, l.next()) >= 0 {
        }
        l.backup()
}
 
// lineNumber reports which line we're on. Doing it this way
// means we don't have to worry about peek double counting.
func (l *lexer) lineNumber() int {
        return 1 + strings.Count(l.input[:l.pos], "\n")
}
 
// error returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.run.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
        l.items <- item{itemError, fmt.Sprintf(format, args...)}
        return nil
}
 
// nextItem returns the next item from the input.
func (l *lexer) nextItem() item {
        for {
                select {
                case item := <-l.items:
                        return item
                default:
                        l.state = l.state(l)
                }
        }
        panic("not reached")
}
 
// lex creates a new scanner for the input string.
func lex(name, input, left, right string) *lexer {
        if left == "" {
                left = leftDelim
        }
        if right == "" {
                right = rightDelim
        }
        l := &lexer{
                name:       name,
                input:      input,
                leftDelim:  left,
                rightDelim: right,
                state:      lexText,
                items:      make(chan item, 2), // Two items of buffering is sufficient for all state functions
        }
        return l
}
 
// state functions
 
const (
        leftDelim    = "{{"
        rightDelim   = "}}"
        leftComment  = "/*"
        rightComment = "*/"
)
 
// lexText scans until an opening action delimiter, "{{".
func lexText(l *lexer) stateFn {
        for {
                if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
                        if l.pos > l.start {
                                l.emit(itemText)
                        }
                        return lexLeftDelim
                }
                if l.next() == eof {
                        break
                }
        }
        // Correctly reached EOF.
        if l.pos > l.start {
                l.emit(itemText)
        }
        l.emit(itemEOF)
        return nil
}
 
// lexLeftDelim scans the left delimiter, which is known to be present.
func lexLeftDelim(l *lexer) stateFn {
        if strings.HasPrefix(l.input[l.pos:], l.leftDelim+leftComment) {
                return lexComment
        }
        l.pos += len(l.leftDelim)
        l.emit(itemLeftDelim)
        return lexInsideAction
}
 
// lexComment scans a comment. The left comment marker is known to be present.
func lexComment(l *lexer) stateFn {
        i := strings.Index(l.input[l.pos:], rightComment+l.rightDelim)
        if i < 0 {
                return l.errorf("unclosed comment")
        }
        l.pos += i + len(rightComment) + len(l.rightDelim)
        l.ignore()
        return lexText
}
 
// lexRightDelim scans the right delimiter, which is known to be present.
func lexRightDelim(l *lexer) stateFn {
        l.pos += len(l.rightDelim)
        l.emit(itemRightDelim)
        return lexText
}
 
// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
        // Either number, quoted string, or identifier.
        // Spaces separate and are ignored.
        // Pipe symbols separate and are emitted.
        if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
                return lexRightDelim
        }
        switch r := l.next(); {
        case r == eof || r == '\n':
                return l.errorf("unclosed action")
        case isSpace(r):
                l.ignore()
        case r == ':':
                if l.next() != '=' {
                        return l.errorf("expected :=")
                }
                l.emit(itemColonEquals)
        case r == '|':
                l.emit(itemPipe)
        case r == '"':
                return lexQuote
        case r == '`':
                return lexRawQuote
        case r == '$':
                return lexIdentifier
        case r == '\'':
                return lexChar
        case r == '.':
                // special look-ahead for ".field" so we don't break l.backup().
                if l.pos < len(l.input) {
                        r := l.input[l.pos]
                        if r < '0' || '9' < r {
                                return lexIdentifier // itemDot comes from the keyword table.
                        }
                }
                fallthrough // '.' can start a number.
        case r == '+' || r == '-' || ('0' <= r && r <= '9'):
                l.backup()
                return lexNumber
        case isAlphaNumeric(r):
                l.backup()
                return lexIdentifier
        case r <= unicode.MaxASCII && unicode.IsPrint(r):
                l.emit(itemChar)
                return lexInsideAction
        default:
                return l.errorf("unrecognized character in action: %#U", r)
        }
        return lexInsideAction
}
 
// lexIdentifier scans an alphanumeric or field.
func lexIdentifier(l *lexer) stateFn {
Loop:
        for {
                switch r := l.next(); {
                case isAlphaNumeric(r):
                        // absorb.
                case r == '.' && (l.input[l.start] == '.' || l.input[l.start] == '$'):
                        // field chaining; absorb into one token.
                default:
                        l.backup()
                        word := l.input[l.start:l.pos]
                        switch {
                        case key[word] > itemKeyword:
                                l.emit(key[word])
                        case word[0] == '.':
                                l.emit(itemField)
                        case word[0] == '$':
                                l.emit(itemVariable)
                        case word == "true", word == "false":
                                l.emit(itemBool)
                        default:
                                l.emit(itemIdentifier)
                        }
                        break Loop
                }
        }
        return lexInsideAction
}
 
// lexChar scans a character constant. The initial quote is already
// scanned.  Syntax checking is done by the parse.
func lexChar(l *lexer) stateFn {
Loop:
        for {
                switch l.next() {
                case '\\':
                        if r := l.next(); r != eof && r != '\n' {
                                break
                        }
                        fallthrough
                case eof, '\n':
                        return l.errorf("unterminated character constant")
                case '\'':
                        break Loop
                }
        }
        l.emit(itemCharConstant)
        return lexInsideAction
}
 
// lexNumber scans a number: decimal, octal, hex, float, or imaginary.  This
// isn't a perfect number scanner - for instance it accepts "." and "0x0.2"
// and "089" - but when it's wrong the input is invalid and the parser (via
// strconv) will notice.
func lexNumber(l *lexer) stateFn {
        if !l.scanNumber() {
                return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
        }
        if sign := l.peek(); sign == '+' || sign == '-' {
                // Complex: 1+2i.  No spaces, must end in 'i'.
                if !l.scanNumber() || l.input[l.pos-1] != 'i' {
                        return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
                }
                l.emit(itemComplex)
        } else {
                l.emit(itemNumber)
        }
        return lexInsideAction
}
 
func (l *lexer) scanNumber() bool {
        // Optional leading sign.
        l.accept("+-")
        // Is it hex?
        digits := "0123456789"
        if l.accept("0") && l.accept("xX") {
                digits = "0123456789abcdefABCDEF"
        }
        l.acceptRun(digits)
        if l.accept(".") {
                l.acceptRun(digits)
        }
        if l.accept("eE") {
                l.accept("+-")
                l.acceptRun("0123456789")
        }
        // Is it imaginary?
        l.accept("i")
        // Next thing mustn't be alphanumeric.
        if isAlphaNumeric(l.peek()) {
                l.next()
                return false
        }
        return true
}
 
// lexQuote scans a quoted string.
func lexQuote(l *lexer) stateFn {
Loop:
        for {
                switch l.next() {
                case '\\':
                        if r := l.next(); r != eof && r != '\n' {
                                break
                        }
                        fallthrough
                case eof, '\n':
                        return l.errorf("unterminated quoted string")
                case '"':
                        break Loop
                }
        }
        l.emit(itemString)
        return lexInsideAction
}
 
// lexRawQuote scans a raw quoted string.
func lexRawQuote(l *lexer) stateFn {
Loop:
        for {
                switch l.next() {
                case eof, '\n':
                        return l.errorf("unterminated raw quoted string")
                case '`':
                        break Loop
                }
        }
        l.emit(itemRawString)
        return lexInsideAction
}
 
// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
        switch r {
        case ' ', '\t', '\n', '\r':
                return true
        }
        return false
}
 
// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool {
        return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

Browse

Tools

Subversion Repositories openrisc

[/] [openrisc/] [trunk/] [gnu-dev/] [or1k-gcc/] [libgo/] [go/] [text/] [template/] [parse/] [lex.go] - Blame information for rev 747

Line No.	Rev	Author	Line
1	747	jeremybenn	`// Copyright 2011 The Go Authors. All rights reserved.`
2			`// Use of this source code is governed by a BSD-style`
3			`// license that can be found in the LICENSE file.`
4
5			`package parse`
6
7			`import (`
8			`"fmt"`
9			`"strings"`
10			`"unicode"`
11			`"unicode/utf8"`
12			`)`
13
14			`// item represents a token or text string returned from the scanner.`
15			`type item struct {`
16			`typ itemType`
17			`val string`
18			`}`
19
20			`func (i item) String() string {`
21			`switch {`
22			`case i.typ == itemEOF:`
23			`return "EOF"`
24			`case i.typ == itemError:`
25			`return i.val`
26			`case i.typ > itemKeyword:`
27			`return fmt.Sprintf("<%s>", i.val)`
28			`case len(i.val) > 10:`
29			`return fmt.Sprintf("%.10q...", i.val)`
30			`}`
31			`return fmt.Sprintf("%q", i.val)`
32			`}`
33
34			`// itemType identifies the type of lex items.`
35			`type itemType int`
36
37			`const (`
38			`itemError itemType = iota // error occurred; value is text of error`
39			`itemBool // boolean constant`
40			`itemChar // printable ASCII character; grab bag for comma etc.`
41			`itemCharConstant // character constant`
42			`itemComplex // complex constant (1+2i); imaginary is just a number`
43			`itemColonEquals // colon-equals (':=') introducing a declaration`
44			`itemEOF`
45			`itemField // alphanumeric identifier, starting with '.', possibly chained ('.x.y')`
46			`itemIdentifier // alphanumeric identifier`
47			`itemLeftDelim // left action delimiter`
48			`itemNumber // simple number, including imaginary`
49			`itemPipe // pipe symbol`
50			`itemRawString // raw quoted string (includes quotes)`
51			`itemRightDelim // right action delimiter`
52			`itemString // quoted string (includes quotes)`
53			`itemText // plain text`
54			`itemVariable // variable starting with '$', such as '$' or '$1' or '$hello'.`
55			`// Keywords appear after all the rest.`
56			`itemKeyword // used only to delimit the keywords`
57			`itemDot // the cursor, spelled '.'.`
58			`itemDefine // define keyword`
59			`itemElse // else keyword`
60			`itemEnd // end keyword`
61			`itemIf // if keyword`
62			`itemRange // range keyword`
63			`itemTemplate // template keyword`
64			`itemWith // with keyword`
65			`)`
66
67			`// Make the types prettyprint.`
68			`var itemName = map[itemType]string{`
69			`itemError: "error",`
70			`itemBool: "bool",`
71			`itemChar: "char",`
72			`itemCharConstant: "charconst",`
73			`itemComplex: "complex",`
74			`itemColonEquals: ":=",`
75			`itemEOF: "EOF",`
76			`itemField: "field",`
77			`itemIdentifier: "identifier",`
78			`itemLeftDelim: "left delim",`
79			`itemNumber: "number",`
80			`itemPipe: "pipe",`
81			`itemRawString: "raw string",`
82			`itemRightDelim: "right delim",`
83			`itemString: "string",`
84			`itemVariable: "variable",`
85			`// keywords`
86			`itemDot: ".",`
87			`itemDefine: "define",`
88			`itemElse: "else",`
89			`itemIf: "if",`
90			`itemEnd: "end",`
91			`itemRange: "range",`
92			`itemTemplate: "template",`
93			`itemWith: "with",`
94			`}`
95
96			`func (i itemType) String() string {`
97			`s := itemName[i]`
98			`if s == "" {`
99			`return fmt.Sprintf("item%d", int(i))`
100			`}`
101			`return s`
102			`}`
103
104			`var key = map[string]itemType{`
105			`".": itemDot,`
106			`"define": itemDefine,`
107			`"else": itemElse,`
108			`"end": itemEnd,`
109			`"if": itemIf,`
110			`"range": itemRange,`
111			`"template": itemTemplate,`
112			`"with": itemWith,`
113			`}`
114
115			`const eof = -1`
116
117			`// stateFn represents the state of the scanner as a function that returns the next state.`
118			`type stateFn func(*lexer) stateFn`
119
120			`// lexer holds the state of the scanner.`
121			`type lexer struct {`
122			`name string // the name of the input; used only for error reports.`
123			`input string // the string being scanned.`
124			`leftDelim string // start of action.`
125			`rightDelim string // end of action.`
126			`state stateFn // the next lexing function to enter.`
127			`pos int // current position in the input.`
128			`start int // start position of this item.`
129			`width int // width of last rune read from input.`
130			`items chan item // channel of scanned items.`
131			`}`
132
133			`// next returns the next rune in the input.`
134			`func (l *lexer) next() (r rune) {`
135			`if l.pos >= len(l.input) {`
136			`l.width = 0`
137			`return eof`
138			`}`
139			`r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])`
140			`l.pos += l.width`
141			`return r`
142			`}`
143
144			`// peek returns but does not consume the next rune in the input.`
145			`func (l *lexer) peek() rune {`
146			`r := l.next()`
147			`l.backup()`
148			`return r`
149			`}`
150
151			`// backup steps back one rune. Can only be called once per call of next.`
152			`func (l *lexer) backup() {`
153			`l.pos -= l.width`
154			`}`
155
156			`// emit passes an item back to the client.`
157			`func (l *lexer) emit(t itemType) {`
158			`l.items <- item{t, l.input[l.start:l.pos]}`
159			`l.start = l.pos`
160			`}`
161
162			`// ignore skips over the pending input before this point.`
163			`func (l *lexer) ignore() {`
164			`l.start = l.pos`
165			`}`
166
167			`// accept consumes the next rune if it's from the valid set.`
168			`func (l *lexer) accept(valid string) bool {`
169			`if strings.IndexRune(valid, l.next()) >= 0 {`
170			`return true`
171			`}`
172			`l.backup()`
173			`return false`
174			`}`
175
176			`// acceptRun consumes a run of runes from the valid set.`
177			`func (l *lexer) acceptRun(valid string) {`
178			`for strings.IndexRune(valid, l.next()) >= 0 {`
179			`}`
180			`l.backup()`
181			`}`
182
183			`// lineNumber reports which line we're on. Doing it this way`
184			`// means we don't have to worry about peek double counting.`
185			`func (l *lexer) lineNumber() int {`
186			`return 1 + strings.Count(l.input[:l.pos], "\n")`
187			`}`
188
189			`// error returns an error token and terminates the scan by passing`
190			`// back a nil pointer that will be the next state, terminating l.run.`
191			`func (l *lexer) errorf(format string, args ...interface{}) stateFn {`
192			`l.items <- item{itemError, fmt.Sprintf(format, args...)}`
193			`return nil`
194			`}`
195
196			`// nextItem returns the next item from the input.`
197			`func (l *lexer) nextItem() item {`
198			`for {`
199			`select {`
200			`case item := <-l.items:`
201			`return item`
202			`default:`
203			`l.state = l.state(l)`
204			`}`
205			`}`
206			`panic("not reached")`
207			`}`
208
209			`// lex creates a new scanner for the input string.`
210			`func lex(name, input, left, right string) *lexer {`
211			`if left == "" {`
212			`left = leftDelim`
213			`}`
214			`if right == "" {`
215			`right = rightDelim`
216			`}`
217			`l := &lexer{`
218			`name: name,`
219			`input: input,`
220			`leftDelim: left,`
221			`rightDelim: right,`
222			`state: lexText,`
223			`items: make(chan item, 2), // Two items of buffering is sufficient for all state functions`
224			`}`
225			`return l`
226			`}`
227
228			`// state functions`
229
230			`const (`
231			`leftDelim = "{{"`
232			`rightDelim = "}}"`
233			`leftComment = "/*"`
234			`rightComment = "*/"`
235			`)`
236
237			`// lexText scans until an opening action delimiter, "{{".`
238			`func lexText(l *lexer) stateFn {`
239			`for {`
240			`if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {`
241			`if l.pos > l.start {`
242			`l.emit(itemText)`
243			`}`
244			`return lexLeftDelim`
245			`}`
246			`if l.next() == eof {`
247			`break`
248			`}`
249			`}`
250			`// Correctly reached EOF.`
251			`if l.pos > l.start {`
252			`l.emit(itemText)`
253			`}`
254			`l.emit(itemEOF)`
255			`return nil`
256			`}`
257
258			`// lexLeftDelim scans the left delimiter, which is known to be present.`
259			`func lexLeftDelim(l *lexer) stateFn {`
260			`if strings.HasPrefix(l.input[l.pos:], l.leftDelim+leftComment) {`
261			`return lexComment`
262			`}`
263			`l.pos += len(l.leftDelim)`
264			`l.emit(itemLeftDelim)`
265			`return lexInsideAction`
266			`}`
267
268			`// lexComment scans a comment. The left comment marker is known to be present.`
269			`func lexComment(l *lexer) stateFn {`
270			`i := strings.Index(l.input[l.pos:], rightComment+l.rightDelim)`
271			`if i < 0 {`
272			`return l.errorf("unclosed comment")`
273			`}`
274			`l.pos += i + len(rightComment) + len(l.rightDelim)`
275			`l.ignore()`
276			`return lexText`
277			`}`
278
279			`// lexRightDelim scans the right delimiter, which is known to be present.`
280			`func lexRightDelim(l *lexer) stateFn {`
281			`l.pos += len(l.rightDelim)`
282			`l.emit(itemRightDelim)`
283			`return lexText`
284			`}`
285
286			`// lexInsideAction scans the elements inside action delimiters.`
287			`func lexInsideAction(l *lexer) stateFn {`
288			`// Either number, quoted string, or identifier.`
289			`// Spaces separate and are ignored.`
290			`// Pipe symbols separate and are emitted.`
291			`if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {`
292			`return lexRightDelim`
293			`}`
294			`switch r := l.next(); {`
295			`case r == eof \|\| r == '\n':`
296			`return l.errorf("unclosed action")`
297			`case isSpace(r):`
298			`l.ignore()`
299			`case r == ':':`
300			`if l.next() != '=' {`
301			`return l.errorf("expected :=")`
302			`}`
303			`l.emit(itemColonEquals)`
304			`case r == '\|':`
305			`l.emit(itemPipe)`
306			`case r == '"':`
307			`return lexQuote`
308			case r == '`':
309			`return lexRawQuote`
310			`case r == '$':`
311			`return lexIdentifier`
312			`case r == '\'':`
313			`return lexChar`
314			`case r == '.':`
315			`// special look-ahead for ".field" so we don't break l.backup().`
316			`if l.pos < len(l.input) {`
317			`r := l.input[l.pos]`
318			`if r < '0' \|\| '9' < r {`
319			`return lexIdentifier // itemDot comes from the keyword table.`
320			`}`
321			`}`
322			`fallthrough // '.' can start a number.`
323			`case r == '+' \|\| r == '-' \|\| ('0' <= r && r <= '9'):`
324			`l.backup()`
325			`return lexNumber`
326			`case isAlphaNumeric(r):`
327			`l.backup()`
328			`return lexIdentifier`
329			`case r <= unicode.MaxASCII && unicode.IsPrint(r):`
330			`l.emit(itemChar)`
331			`return lexInsideAction`
332			`default:`
333			`return l.errorf("unrecognized character in action: %#U", r)`
334			`}`
335			`return lexInsideAction`
336			`}`
337
338			`// lexIdentifier scans an alphanumeric or field.`
339			`func lexIdentifier(l *lexer) stateFn {`
340			`Loop:`
341			`for {`
342			`switch r := l.next(); {`
343			`case isAlphaNumeric(r):`
344			`// absorb.`
345			`case r == '.' && (l.input[l.start] == '.' \|\| l.input[l.start] == '$'):`
346			`// field chaining; absorb into one token.`
347			`default:`
348			`l.backup()`
349			`word := l.input[l.start:l.pos]`
350			`switch {`
351			`case key[word] > itemKeyword:`
352			`l.emit(key[word])`
353			`case word[0] == '.':`
354			`l.emit(itemField)`
355			`case word[0] == '$':`
356			`l.emit(itemVariable)`
357			`case word == "true", word == "false":`
358			`l.emit(itemBool)`
359			`default:`
360			`l.emit(itemIdentifier)`
361			`}`
362			`break Loop`
363			`}`
364			`}`
365			`return lexInsideAction`
366			`}`
367
368			`// lexChar scans a character constant. The initial quote is already`
369			`// scanned. Syntax checking is done by the parse.`
370			`func lexChar(l *lexer) stateFn {`
371			`Loop:`
372			`for {`
373			`switch l.next() {`
374			`case '\\':`
375			`if r := l.next(); r != eof && r != '\n' {`
376			`break`
377			`}`
378			`fallthrough`
379			`case eof, '\n':`
380			`return l.errorf("unterminated character constant")`
381			`case '\'':`
382			`break Loop`
383			`}`
384			`}`
385			`l.emit(itemCharConstant)`
386			`return lexInsideAction`
387			`}`
388
389			`// lexNumber scans a number: decimal, octal, hex, float, or imaginary. This`
390			`// isn't a perfect number scanner - for instance it accepts "." and "0x0.2"`
391			`// and "089" - but when it's wrong the input is invalid and the parser (via`
392			`// strconv) will notice.`
393			`func lexNumber(l *lexer) stateFn {`
394			`if !l.scanNumber() {`
395			`return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])`
396			`}`
397			`if sign := l.peek(); sign == '+' \|\| sign == '-' {`
398			`// Complex: 1+2i. No spaces, must end in 'i'.`
399			`if !l.scanNumber() \|\| l.input[l.pos-1] != 'i' {`
400			`return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])`
401			`}`
402			`l.emit(itemComplex)`
403			`} else {`
404			`l.emit(itemNumber)`
405			`}`
406			`return lexInsideAction`
407			`}`
408
409			`func (l *lexer) scanNumber() bool {`
410			`// Optional leading sign.`
411			`l.accept("+-")`
412			`// Is it hex?`
413			`digits := "0123456789"`
414			`if l.accept("0") && l.accept("xX") {`
415			`digits = "0123456789abcdefABCDEF"`
416			`}`
417			`l.acceptRun(digits)`
418			`if l.accept(".") {`
419			`l.acceptRun(digits)`
420			`}`
421			`if l.accept("eE") {`
422			`l.accept("+-")`
423			`l.acceptRun("0123456789")`
424			`}`
425			`// Is it imaginary?`
426			`l.accept("i")`
427			`// Next thing mustn't be alphanumeric.`
428			`if isAlphaNumeric(l.peek()) {`
429			`l.next()`
430			`return false`
431			`}`
432			`return true`
433			`}`
434
435			`// lexQuote scans a quoted string.`
436			`func lexQuote(l *lexer) stateFn {`
437			`Loop:`
438			`for {`
439			`switch l.next() {`
440			`case '\\':`
441			`if r := l.next(); r != eof && r != '\n' {`
442			`break`
443			`}`
444			`fallthrough`
445			`case eof, '\n':`
446			`return l.errorf("unterminated quoted string")`
447			`case '"':`
448			`break Loop`
449			`}`
450			`}`
451			`l.emit(itemString)`
452			`return lexInsideAction`
453			`}`
454
455			`// lexRawQuote scans a raw quoted string.`
456			`func lexRawQuote(l *lexer) stateFn {`
457			`Loop:`
458			`for {`
459			`switch l.next() {`
460			`case eof, '\n':`
461			`return l.errorf("unterminated raw quoted string")`
462			case '`':
463			`break Loop`
464			`}`
465			`}`
466			`l.emit(itemRawString)`
467			`return lexInsideAction`
468			`}`
469
470			`// isSpace reports whether r is a space character.`
471			`func isSpace(r rune) bool {`
472			`switch r {`
473			`case ' ', '\t', '\n', '\r':`
474			`return true`
475			`}`
476			`return false`
477			`}`
478
479			`// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.`
480			`func isAlphaNumeric(r rune) bool {`
481			`return r == '_' \|\| unicode.IsLetter(r) \|\| unicode.IsDigit(r)`
482			`}`