URL https://opencores.org/ocsvn/tcp_socket/tcp_socket/trunk

Subversion Repositories tcp_socket

[/] [tcp_socket/] [trunk/] [chips2/] [chips/] [compiler/] [tokens.py] - Blame information for rev 2

Go to most recent revision | Details | Compare with Previous | View Log


__author__ = "Jon Dawson"
__copyright__ = "Copyright (C) 2012, Jonathan P Dawson"
__version__ = "0.1"
 
import os.path
import StringIO
 
from chips.compiler.exceptions import C2CHIPError
from chips.compiler.builtins import builtins
 
operators = [
  "!", "~", "+", "-", "*", "/", "//", "%", "=", "==", "<", ">", "<=", ">=",
  "!=", "|", "&", "^", "||", "&&", "(", ")", "{", "}", "[", "]", ";", "<<",
  ">>", ",", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "<<=", ">>=", "++",
  "--", "?", ":", "."
]
 
class Tokens:
 
    """Break the input file into a stream of tokens,
    provide functions to traverse the stream."""
 
    def __init__(self, filename):
       self.tokens = []
       self.filename = None
       self.lineno = None
       self.scan("built in", StringIO.StringIO(builtins))
       self.scan(filename)
 
    def scan(self, filename, input_file=None):
 
        """Convert the test file into tokens"""
        self.filename = filename
 
        if input_file is None:
            try:
                input_file = open(self.filename)
            except IOError:
                raise C2CHIPError("Cannot open file: "+self.filename)
 
        token = []
        tokens = []
        self.lineno = 1
        for line in input_file:
 
            #include files
            line = line+" "
            if line.strip().startswith("#include"):
                filename = self.filename
                lineno = self.lineno
                self.tokens.extend(tokens)
                directory = os.path.abspath(self.filename)
                directory = os.path.dirname(directory)
                self.filename = line.strip().replace("#include", "").strip(' ><"')
                self.filename = os.path.join(directory, self.filename)
                self.scan(self.filename)
                self.lineno = lineno
                self.filename = filename
                tokens = []
                continue
 
            newline = True
            for char in line:
 
                if not token:
                    token = char
 
                #c style comment
                elif (token + char).startswith("/*"):
                    if (token + char).endswith("*/"):
                        token = ""
                    else:
                        token += char
 
                #c++ style comment
                elif token.startswith("//"):
                    if newline:
                        token = char
                    else:
                        token += char
 
                #identifier
                elif token[0].isalpha():
                    if char.isalnum() or char== "_":
                        token += char
                    else:
                        tokens.append((self.filename, self.lineno, token))
                        token = char
 
                #number
                elif token[0].isdigit():
                    if char.upper() in "UXABCDEFL0123456789":
                        token += char
                    else:
                        tokens.append((self.filename, self.lineno, token))
                        token = char
 
                #string literal
                elif token.startswith('"'):
                    if char == '"' and previous_char != "\\":
                        token += char
                        tokens.append((self.filename, self.lineno, token))
                        token = ""
                    else:
                        #remove dummy space from the end of a line
                        if newline:
                            token = token[:-1]
                        previous_char = char
                        token += char
 
                #character literal
                elif token.startswith("'"):
                    if char == "'":
                        token += char
                        tokens.append((self.filename, self.lineno, token))
                        token = ""
                    else:
                        token += char
 
                #operator
                elif token in operators:
                    if token + char in operators:
                        token += char
                    else:
                        tokens.append((self.filename, self.lineno, token))
                        token = char
 
                else:
                    token = char
 
                newline = False
            self.lineno += 1
 
        self.tokens.extend(tokens)
 
    def error(self, string):
 
        """Generate an error message (including the filename and line number)"""
 
        raise C2CHIPError(string + "\n", self.filename, self.lineno)
 
    def peek(self):
 
        """Return the next token in the stream, but don't consume it"""
 
        if self.tokens:
            return self.tokens[0][2]
        else:
            return ""
 
    def get(self):
 
        """Return the next token in the stream, and consume it"""
 
        if self.tokens:
            self.lineno = self.tokens[0][1]
            self.filename = self.tokens[0][0]
        filename, lineno, token = self.tokens.pop(0)
        return token
 
    def end(self):
 
        """Return True if all the tokens have been consumed"""
 
        return not self.tokens
 
    def expect(self, expected):
 
        """Consume the next token in the stream,
        generate an error if it is not as expected."""
 
        filename, lineno, actual = self.tokens.pop(0)
        if self.tokens:
            self.lineno = self.tokens[0][1]
            self.filename = self.tokens[0][0]
        if actual == expected:
            return
        else:
            self.error("Expected: %s, got: %s"%(expected, actual))

Line No.	Rev	Author	Line
1	2	jondawson	`__author__ = "Jon Dawson"`
2			`__copyright__ = "Copyright (C) 2012, Jonathan P Dawson"`
3			`__version__ = "0.1"`
4
5			`import os.path`
6			`import StringIO`
7
8			`from chips.compiler.exceptions import C2CHIPError`
9			`from chips.compiler.builtins import builtins`
10
11			`operators = [`
12			`"!", "~", "+", "-", "*", "/", "//", "%", "=", "==", "<", ">", "<=", ">=",`
13			`"!=", "\|", "&", "^", "\|\|", "&&", "(", ")", "{", "}", "[", "]", ";", "<<",`
14			`">>", ",", "+=", "-=", "*=", "/=", "%=", "&=", "\|=", "<<=", ">>=", "++",`
15			`"--", "?", ":", "."`
16			`]`
17
18			`class Tokens:`
19
20			`"""Break the input file into a stream of tokens,`
21			`provide functions to traverse the stream."""`
22
23			`def __init__(self, filename):`
24			`self.tokens = []`
25			`self.filename = None`
26			`self.lineno = None`
27			`self.scan("built in", StringIO.StringIO(builtins))`
28			`self.scan(filename)`
29
30			`def scan(self, filename, input_file=None):`
31
32			`"""Convert the test file into tokens"""`
33			`self.filename = filename`
34
35			`if input_file is None:`
36			`try:`
37			`input_file = open(self.filename)`
38			`except IOError:`
39			`raise C2CHIPError("Cannot open file: "+self.filename)`
40
41			`token = []`
42			`tokens = []`
43			`self.lineno = 1`
44			`for line in input_file:`
45
46			`#include files`
47			`line = line+" "`
48			`if line.strip().startswith("#include"):`
49			`filename = self.filename`
50			`lineno = self.lineno`
51			`self.tokens.extend(tokens)`
52			`directory = os.path.abspath(self.filename)`
53			`directory = os.path.dirname(directory)`
54			`self.filename = line.strip().replace("#include", "").strip(' ><"')`
55			`self.filename = os.path.join(directory, self.filename)`
56			`self.scan(self.filename)`
57			`self.lineno = lineno`
58			`self.filename = filename`
59			`tokens = []`
60			`continue`
61
62			`newline = True`
63			`for char in line:`
64
65			`if not token:`
66			`token = char`
67
68			`#c style comment`
69			`elif (token + char).startswith("/*"):`
70			`if (token + char).endswith("*/"):`
71			`token = ""`
72			`else:`
73			`token += char`
74
75			`#c++ style comment`
76			`elif token.startswith("//"):`
77			`if newline:`
78			`token = char`
79			`else:`
80			`token += char`
81
82			`#identifier`
83			`elif token[0].isalpha():`
84			`if char.isalnum() or char== "_":`
85			`token += char`
86			`else:`
87			`tokens.append((self.filename, self.lineno, token))`
88			`token = char`
89
90			`#number`
91			`elif token[0].isdigit():`
92			`if char.upper() in "UXABCDEFL0123456789":`
93			`token += char`
94			`else:`
95			`tokens.append((self.filename, self.lineno, token))`
96			`token = char`
97
98			`#string literal`
99			`elif token.startswith('"'):`
100			`if char == '"' and previous_char != "\\":`
101			`token += char`
102			`tokens.append((self.filename, self.lineno, token))`
103			`token = ""`
104			`else:`
105			`#remove dummy space from the end of a line`
106			`if newline:`
107			`token = token[:-1]`
108			`previous_char = char`
109			`token += char`
110
111			`#character literal`
112			`elif token.startswith("'"):`
113			`if char == "'":`
114			`token += char`
115			`tokens.append((self.filename, self.lineno, token))`
116			`token = ""`
117			`else:`
118			`token += char`
119
120			`#operator`
121			`elif token in operators:`
122			`if token + char in operators:`
123			`token += char`
124			`else:`
125			`tokens.append((self.filename, self.lineno, token))`
126			`token = char`
127
128			`else:`
129			`token = char`
130
131			`newline = False`
132			`self.lineno += 1`
133
134			`self.tokens.extend(tokens)`
135
136			`def error(self, string):`
137
138			`"""Generate an error message (including the filename and line number)"""`
139
140			`raise C2CHIPError(string + "\n", self.filename, self.lineno)`
141
142			`def peek(self):`
143
144			`"""Return the next token in the stream, but don't consume it"""`
145
146			`if self.tokens:`
147			`return self.tokens[0][2]`
148			`else:`
149			`return ""`
150
151			`def get(self):`
152
153			`"""Return the next token in the stream, and consume it"""`
154
155			`if self.tokens:`
156			`self.lineno = self.tokens[0][1]`
157			`self.filename = self.tokens[0][0]`
158			`filename, lineno, token = self.tokens.pop(0)`
159			`return token`
160
161			`def end(self):`
162
163			`"""Return True if all the tokens have been consumed"""`
164
165			`return not self.tokens`
166
167			`def expect(self, expected):`
168
169			`"""Consume the next token in the stream,`
170			`generate an error if it is not as expected."""`
171
172			`filename, lineno, actual = self.tokens.pop(0)`
173			`if self.tokens:`
174			`self.lineno = self.tokens[0][1]`
175			`self.filename = self.tokens[0][0]`
176			`if actual == expected:`
177			`return`
178			`else:`
179			`self.error("Expected: %s, got: %s"%(expected, actual))`

Browse

Tools

Subversion Repositories tcp_socket

[/] [tcp_socket/] [trunk/] [chips2/] [chips/] [compiler/] [tokens.py] - Blame information for rev 2