__author__ = "Jon Dawson"
|
__author__ = "Jon Dawson"
|
__copyright__ = "Copyright (C) 2012, Jonathan P Dawson"
|
__copyright__ = "Copyright (C) 2012, Jonathan P Dawson"
|
__version__ = "0.1"
|
__version__ = "0.1"
|
|
|
import os.path
|
import os.path
|
import StringIO
|
import StringIO
|
|
|
from chips.compiler.exceptions import C2CHIPError
|
from chips.compiler.exceptions import C2CHIPError
|
from chips.compiler.builtins import builtins
|
from chips.compiler.builtins import builtins
|
|
from chips.compiler.library import libs
|
|
|
operators = [
|
operators = [
|
"!", "~", "+", "-", "*", "/", "//", "%", "=", "==", "<", ">", "<=", ">=",
|
"!", "~", "+", "-", "*", "/", "//", "%", "=", "==", "<", ">", "<=", ">=",
|
"!=", "|", "&", "^", "||", "&&", "(", ")", "{", "}", "[", "]", ";", "<<",
|
"!=", "|", "&", "^", "||", "&&", "(", ")", "{", "}", "[", "]", ";", "<<",
|
">>", ",", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "<<=", ">>=", "++",
|
">>", ",", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "<<=", ">>=", "^=",
|
"--", "?", ":", "."
|
"++", "--", "?", ":", "."
|
]
|
]
|
|
|
class Tokens:
|
class Tokens:
|
|
|
"""Break the input file into a stream of tokens,
|
"""Break the input file into a stream of tokens,
|
provide functions to traverse the stream."""
|
provide functions to traverse the stream."""
|
|
|
def __init__(self, filename):
|
def __init__(self, filename):
|
self.tokens = []
|
self.tokens = []
|
self.filename = None
|
self.filename = None
|
self.lineno = None
|
self.lineno = None
|
self.scan("built in", StringIO.StringIO(builtins))
|
self.scan("built in", StringIO.StringIO(builtins))
|
self.scan(filename)
|
self.scan(filename)
|
|
|
def scan(self, filename, input_file=None):
|
def scan(self, filename, input_file=None):
|
|
|
"""Convert the test file into tokens"""
|
"""Convert the test file into tokens"""
|
self.filename = filename
|
self.filename = filename
|
|
|
if input_file is None:
|
if input_file is None:
|
try:
|
try:
|
input_file = open(self.filename)
|
input_file = open(self.filename)
|
except IOError:
|
except IOError:
|
raise C2CHIPError("Cannot open file: "+self.filename)
|
raise C2CHIPError("Cannot open file: "+self.filename)
|
|
|
token = []
|
token = []
|
tokens = []
|
tokens = []
|
self.lineno = 1
|
self.lineno = 1
|
for line in input_file:
|
for line in input_file:
|
|
|
#include files
|
#include files
|
line = line+" "
|
line = line+" "
|
if line.strip().startswith("#include"):
|
if line.strip().startswith("#include"):
|
filename = self.filename
|
filename = self.filename
|
lineno = self.lineno
|
lineno = self.lineno
|
self.tokens.extend(tokens)
|
self.tokens.extend(tokens)
|
directory = os.path.abspath(self.filename)
|
directory = os.path.abspath(self.filename)
|
directory = os.path.dirname(directory)
|
directory = os.path.dirname(directory)
|
|
if line.strip().endswith(">"):
|
|
self.filename = "library"
|
|
library = line.strip().split("<")[1].strip(' ><"')
|
|
self.scan(self.filename, StringIO.StringIO(libs[library]))
|
|
else:
|
self.filename = line.strip().replace("#include", "").strip(' ><"')
|
self.filename = line.strip().replace("#include", "").strip(' ><"')
|
self.filename = os.path.join(directory, self.filename)
|
self.filename = os.path.join(directory, self.filename)
|
self.scan(self.filename)
|
self.scan(self.filename)
|
self.lineno = lineno
|
self.lineno = lineno
|
self.filename = filename
|
self.filename = filename
|
tokens = []
|
tokens = []
|
continue
|
continue
|
|
|
newline = True
|
newline = True
|
for char in line:
|
for char in line:
|
|
|
if not token:
|
if not token:
|
token = char
|
token = char
|
|
|
#c style comment
|
#c style comment
|
elif (token + char).startswith("/*"):
|
elif (token + char).startswith("/*"):
|
if (token + char).endswith("*/"):
|
if (token + char).endswith("*/"):
|
token = ""
|
token = ""
|
else:
|
else:
|
token += char
|
token += char
|
|
|
#c++ style comment
|
#c++ style comment
|
elif token.startswith("//"):
|
elif token.startswith("//"):
|
if newline:
|
if newline:
|
token = char
|
token = char
|
else:
|
else:
|
token += char
|
token += char
|
|
|
#identifier
|
#identifier
|
elif token[0].isalpha():
|
elif token[0].isalpha():
|
if char.isalnum() or char== "_":
|
if char.isalnum() or char== "_":
|
token += char
|
token += char
|
else:
|
else:
|
tokens.append((self.filename, self.lineno, token))
|
tokens.append((self.filename, self.lineno, token))
|
token = char
|
token = char
|
|
|
#number
|
#number
|
elif token[0].isdigit():
|
elif token[0].isdigit():
|
if char.upper() in "UXABCDEFL0123456789":
|
if char.upper() in "UXABCDEFL0123456789.":
|
token += char
|
token += char
|
else:
|
else:
|
tokens.append((self.filename, self.lineno, token))
|
tokens.append((self.filename, self.lineno, token))
|
token = char
|
token = char
|
|
|
#string literal
|
#string literal
|
elif token.startswith('"'):
|
elif token.startswith('"'):
|
if char == '"' and previous_char != "\\":
|
if char == '"' and previous_char != "\\":
|
token += char
|
token += char
|
tokens.append((self.filename, self.lineno, token))
|
tokens.append((self.filename, self.lineno, token))
|
token = ""
|
token = ""
|
else:
|
else:
|
#remove dummy space from the end of a line
|
#remove dummy space from the end of a line
|
if newline:
|
if newline:
|
token = token[:-1]
|
token = token[:-1]
|
previous_char = char
|
previous_char = char
|
token += char
|
token += char
|
|
|
#character literal
|
#character literal
|
elif token.startswith("'"):
|
elif token.startswith("'"):
|
if char == "'":
|
if char == "'":
|
token += char
|
token += char
|
tokens.append((self.filename, self.lineno, token))
|
tokens.append((self.filename, self.lineno, token))
|
token = ""
|
token = ""
|
else:
|
else:
|
token += char
|
token += char
|
|
|
#operator
|
#operator
|
elif token in operators:
|
elif token in operators:
|
if token + char in operators:
|
if token + char in operators:
|
token += char
|
token += char
|
else:
|
else:
|
tokens.append((self.filename, self.lineno, token))
|
tokens.append((self.filename, self.lineno, token))
|
token = char
|
token = char
|
|
|
else:
|
else:
|
token = char
|
token = char
|
|
|
newline = False
|
newline = False
|
self.lineno += 1
|
self.lineno += 1
|
|
|
self.tokens.extend(tokens)
|
self.tokens.extend(tokens)
|
|
|
def error(self, string):
|
def error(self, string):
|
|
|
"""Generate an error message (including the filename and line number)"""
|
"""Generate an error message (including the filename and line number)"""
|
|
|
raise C2CHIPError(string + "\n", self.filename, self.lineno)
|
raise C2CHIPError(string + "\n", self.filename, self.lineno)
|
|
|
def peek(self):
|
def peek(self):
|
|
|
"""Return the next token in the stream, but don't consume it"""
|
"""Return the next token in the stream, but don't consume it"""
|
|
|
if self.tokens:
|
if self.tokens:
|
return self.tokens[0][2]
|
return self.tokens[0][2]
|
else:
|
else:
|
return ""
|
return ""
|
|
|
def get(self):
|
def get(self):
|
|
|
"""Return the next token in the stream, and consume it"""
|
"""Return the next token in the stream, and consume it"""
|
|
|
if self.tokens:
|
if self.tokens:
|
self.lineno = self.tokens[0][1]
|
self.lineno = self.tokens[0][1]
|
self.filename = self.tokens[0][0]
|
self.filename = self.tokens[0][0]
|
filename, lineno, token = self.tokens.pop(0)
|
filename, lineno, token = self.tokens.pop(0)
|
return token
|
return token
|
|
|
def end(self):
|
def end(self):
|
|
|
"""Return True if all the tokens have been consumed"""
|
"""Return True if all the tokens have been consumed"""
|
|
|
return not self.tokens
|
return not self.tokens
|
|
|
def expect(self, expected):
|
def expect(self, expected):
|
|
|
"""Consume the next token in the stream,
|
"""Consume the next token in the stream,
|
generate an error if it is not as expected."""
|
generate an error if it is not as expected."""
|
|
|
filename, lineno, actual = self.tokens.pop(0)
|
filename, lineno, actual = self.tokens.pop(0)
|
if self.tokens:
|
if self.tokens:
|
self.lineno = self.tokens[0][1]
|
self.lineno = self.tokens[0][1]
|
self.filename = self.tokens[0][0]
|
self.filename = self.tokens[0][0]
|
if actual == expected:
|
if actual == expected:
|
return
|
return
|
else:
|
else:
|
self.error("Expected: %s, got: %s"%(expected, actual))
|
self.error("Expected: %s, got: %s"%(expected, actual))
|
|
|
