Subversion Repositories openrisc

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parse

import (
        "fmt"
        "strings"
        "unicode"
        "unicode/utf8"
)

// item represents a token or text string returned from the scanner.
type item struct {
        typ itemType
        val string
}

func (i item) String() string {
        switch {
        case i.typ == itemEOF:
                return "EOF"
        case i.typ == itemError:
                return i.val
        case i.typ > itemKeyword:
                return fmt.Sprintf("<%s>", i.val)
        case len(i.val) > 10:
                return fmt.Sprintf("%.10q...", i.val)
        }
        return fmt.Sprintf("%q", i.val)
}

// itemType identifies the type of lex items.
type itemType int

const (
        itemError        itemType = iota // error occurred; value is text of error
        itemBool                         // boolean constant
        itemChar                         // printable ASCII character; grab bag for comma etc.
        itemCharConstant                 // character constant
        itemComplex                      // complex constant (1+2i); imaginary is just a number
        itemColonEquals                  // colon-equals (':=') introducing a declaration
        itemEOF
        itemField      // alphanumeric identifier, starting with '.', possibly chained ('.x.y')
        itemIdentifier // alphanumeric identifier
        itemLeftDelim  // left action delimiter
        itemNumber     // simple number, including imaginary
        itemPipe       // pipe symbol
        itemRawString  // raw quoted string (includes quotes)
        itemRightDelim // right action delimiter
        itemString     // quoted string (includes quotes)
        itemText       // plain text
        itemVariable   // variable starting with '$', such as '$' or  '$1' or '$hello'.
        // Keywords appear after all the rest.
        itemKeyword  // used only to delimit the keywords
        itemDot      // the cursor, spelled '.'.
        itemDefine   // define keyword
        itemElse     // else keyword
        itemEnd      // end keyword
        itemIf       // if keyword
        itemRange    // range keyword
        itemTemplate // template keyword
        itemWith     // with keyword
)

// Make the types prettyprint.
var itemName = map[itemType]string{
        itemError:        "error",
        itemBool:         "bool",
        itemChar:         "char",
        itemCharConstant: "charconst",
        itemComplex:      "complex",
        itemColonEquals:  ":=",
        itemEOF:          "EOF",
        itemField:        "field",
        itemIdentifier:   "identifier",
        itemLeftDelim:    "left delim",
        itemNumber:       "number",
        itemPipe:         "pipe",
        itemRawString:    "raw string",
        itemRightDelim:   "right delim",
        itemString:       "string",
        itemVariable:     "variable",
        // keywords
        itemDot:      ".",
        itemDefine:   "define",
        itemElse:     "else",
        itemIf:       "if",
        itemEnd:      "end",
        itemRange:    "range",
        itemTemplate: "template",
        itemWith:     "with",
}

func (i itemType) String() string {
        s := itemName[i]
        if s == "" {
                return fmt.Sprintf("item%d", int(i))
        }
        return s
}

var key = map[string]itemType{
        ".":        itemDot,
        "define":   itemDefine,
        "else":     itemElse,
        "end":      itemEnd,
        "if":       itemIf,
        "range":    itemRange,
        "template": itemTemplate,
        "with":     itemWith,
}

const eof = -1

// stateFn represents the state of the scanner as a function that returns the next state.
type stateFn func(*lexer) stateFn

// lexer holds the state of the scanner.
type lexer struct {
        name       string    // the name of the input; used only for error reports.
        input      string    // the string being scanned.
        leftDelim  string    // start of action.
        rightDelim string    // end of action.
        state      stateFn   // the next lexing function to enter.
        pos        int       // current position in the input.
        start      int       // start position of this item.
        width      int       // width of last rune read from input.
        items      chan item // channel of scanned items.
}

// next returns the next rune in the input.
func (l *lexer) next() (r rune) {
        if l.pos >= len(l.input) {
                l.width = 0
                return eof
        }
        r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
        l.pos += l.width
        return r
}

// peek returns but does not consume the next rune in the input.
func (l *lexer) peek() rune {
        r := l.next()
        l.backup()
        return r
}

// backup steps back one rune. Can only be called once per call of next.
func (l *lexer) backup() {
        l.pos -= l.width
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
        l.items <- item{t, l.input[l.start:l.pos]}
        l.start = l.pos
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
        l.start = l.pos
}

// accept consumes the next rune if it's from the valid set.
func (l *lexer) accept(valid string) bool {
        if strings.IndexRune(valid, l.next()) >= 0 {
                return true
        }
        l.backup()
        return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
        for strings.IndexRune(valid, l.next()) >= 0 {
        }
        l.backup()
}

// lineNumber reports which line we're on. Doing it this way
// means we don't have to worry about peek double counting.
func (l *lexer) lineNumber() int {
        return 1 + strings.Count(l.input[:l.pos], "\n")
}

// error returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.run.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
        l.items <- item{itemError, fmt.Sprintf(format, args...)}
        return nil
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() item {
        for {
                select {
                case item := <-l.items:
                        return item
                default:
                        l.state = l.state(l)
                }
        }
        panic("not reached")
}

// lex creates a new scanner for the input string.
func lex(name, input, left, right string) *lexer {
        if left == "" {
                left = leftDelim
        }
        if right == "" {
                right = rightDelim
        }
        l := &lexer{
                name:       name,
                input:      input,
                leftDelim:  left,
                rightDelim: right,
                state:      lexText,
                items:      make(chan item, 2), // Two items of buffering is sufficient for all state functions
        }
        return l
}

// state functions

const (
        leftDelim    = "{{"
        rightDelim   = "}}"
        leftComment  = "/*"
        rightComment = "*/"
)

// lexText scans until an opening action delimiter, "{{".
func lexText(l *lexer) stateFn {
        for {
                if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
                        if l.pos > l.start {
                                l.emit(itemText)
                        }
                        return lexLeftDelim
                }
                if l.next() == eof {
                        break
                }
        }
        // Correctly reached EOF.
        if l.pos > l.start {
                l.emit(itemText)
        }
        l.emit(itemEOF)
        return nil
}

// lexLeftDelim scans the left delimiter, which is known to be present.
func lexLeftDelim(l *lexer) stateFn {
        if strings.HasPrefix(l.input[l.pos:], l.leftDelim+leftComment) {
                return lexComment
        }
        l.pos += len(l.leftDelim)
        l.emit(itemLeftDelim)
        return lexInsideAction
}

// lexComment scans a comment. The left comment marker is known to be present.
func lexComment(l *lexer) stateFn {
        i := strings.Index(l.input[l.pos:], rightComment+l.rightDelim)
        if i < 0 {
                return l.errorf("unclosed comment")
        }
        l.pos += i + len(rightComment) + len(l.rightDelim)
        l.ignore()
        return lexText
}

// lexRightDelim scans the right delimiter, which is known to be present.
func lexRightDelim(l *lexer) stateFn {
        l.pos += len(l.rightDelim)
        l.emit(itemRightDelim)
        return lexText
}

// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
        // Either number, quoted string, or identifier.
        // Spaces separate and are ignored.
        // Pipe symbols separate and are emitted.
        if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
                return lexRightDelim
        }
        switch r := l.next(); {
        case r == eof || r == '\n':
                return l.errorf("unclosed action")
        case isSpace(r):
                l.ignore()
        case r == ':':
                if l.next() != '=' {
                        return l.errorf("expected :=")
                }
                l.emit(itemColonEquals)
        case r == '|':
                l.emit(itemPipe)
        case r == '"':
                return lexQuote
        case r == '`':
                return lexRawQuote
        case r == '$':
                return lexIdentifier
        case r == '\'':
                return lexChar
        case r == '.':
                // special look-ahead for ".field" so we don't break l.backup().
                if l.pos < len(l.input) {
                        r := l.input[l.pos]
                        if r < '0' || '9' < r {
                                return lexIdentifier // itemDot comes from the keyword table.
                        }
                }
                fallthrough // '.' can start a number.
        case r == '+' || r == '-' || ('0' <= r && r <= '9'):
                l.backup()
                return lexNumber
        case isAlphaNumeric(r):
                l.backup()
                return lexIdentifier
        case r <= unicode.MaxASCII && unicode.IsPrint(r):
                l.emit(itemChar)
                return lexInsideAction
        default:
                return l.errorf("unrecognized character in action: %#U", r)
        }
        return lexInsideAction
}

// lexIdentifier scans an alphanumeric or field.
func lexIdentifier(l *lexer) stateFn {
Loop:
        for {
                switch r := l.next(); {
                case isAlphaNumeric(r):
                        // absorb.
                case r == '.' && (l.input[l.start] == '.' || l.input[l.start] == '$'):
                        // field chaining; absorb into one token.
                default:
                        l.backup()
                        word := l.input[l.start:l.pos]
                        switch {
                        case key[word] > itemKeyword:
                                l.emit(key[word])
                        case word[0] == '.':
                                l.emit(itemField)
                        case word[0] == '$':
                                l.emit(itemVariable)
                        case word == "true", word == "false":
                                l.emit(itemBool)
                        default:
                                l.emit(itemIdentifier)
                        }
                        break Loop
                }
        }
        return lexInsideAction
}

// lexChar scans a character constant. The initial quote is already
// scanned.  Syntax checking is done by the parse.
func lexChar(l *lexer) stateFn {
Loop:
        for {
                switch l.next() {
                case '\\':
                        if r := l.next(); r != eof && r != '\n' {
                                break
                        }
                        fallthrough
                case eof, '\n':
                        return l.errorf("unterminated character constant")
                case '\'':
                        break Loop
                }
        }
        l.emit(itemCharConstant)
        return lexInsideAction
}

// lexNumber scans a number: decimal, octal, hex, float, or imaginary.  This
// isn't a perfect number scanner - for instance it accepts "." and "0x0.2"
// and "089" - but when it's wrong the input is invalid and the parser (via
// strconv) will notice.
func lexNumber(l *lexer) stateFn {
        if !l.scanNumber() {
                return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
        }
        if sign := l.peek(); sign == '+' || sign == '-' {
                // Complex: 1+2i.  No spaces, must end in 'i'.
                if !l.scanNumber() || l.input[l.pos-1] != 'i' {
                        return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
                }
                l.emit(itemComplex)
        } else {
                l.emit(itemNumber)
        }
        return lexInsideAction
}

func (l *lexer) scanNumber() bool {
        // Optional leading sign.
        l.accept("+-")
        // Is it hex?
        digits := "0123456789"
        if l.accept("0") && l.accept("xX") {
                digits = "0123456789abcdefABCDEF"
        }
        l.acceptRun(digits)
        if l.accept(".") {
                l.acceptRun(digits)
        }
        if l.accept("eE") {
                l.accept("+-")
                l.acceptRun("0123456789")
        }
        // Is it imaginary?
        l.accept("i")
        // Next thing mustn't be alphanumeric.
        if isAlphaNumeric(l.peek()) {
                l.next()
                return false
        }
        return true
}

// lexQuote scans a quoted string.
func lexQuote(l *lexer) stateFn {
Loop:
        for {
                switch l.next() {
                case '\\':
                        if r := l.next(); r != eof && r != '\n' {
                                break
                        }
                        fallthrough
                case eof, '\n':
                        return l.errorf("unterminated quoted string")
                case '"':
                        break Loop
                }
        }
        l.emit(itemString)
        return lexInsideAction
}

// lexRawQuote scans a raw quoted string.
func lexRawQuote(l *lexer) stateFn {
Loop:
        for {
                switch l.next() {
                case eof, '\n':
                        return l.errorf("unterminated raw quoted string")
                case '`':
                        break Loop
                }
        }
        l.emit(itemRawString)
        return lexInsideAction
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
        switch r {
        case ' ', '\t', '\n', '\r':
                return true
        }
        return false
}

// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool {
        return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}