1 |
2 |
jamieiles |
#!/usr/bin/env python
|
2 |
|
|
#
|
3 |
|
|
# Copyright 2007 Neal Norwitz
|
4 |
|
|
# Portions Copyright 2007 Google Inc.
|
5 |
|
|
#
|
6 |
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7 |
|
|
# you may not use this file except in compliance with the License.
|
8 |
|
|
# You may obtain a copy of the License at
|
9 |
|
|
#
|
10 |
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
11 |
|
|
#
|
12 |
|
|
# Unless required by applicable law or agreed to in writing, software
|
13 |
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
14 |
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15 |
|
|
# See the License for the specific language governing permissions and
|
16 |
|
|
# limitations under the License.
|
17 |
|
|
|
18 |
|
|
"""Generate an Abstract Syntax Tree (AST) for C++."""
|
19 |
|
|
|
20 |
|
|
__author__ = 'nnorwitz@google.com (Neal Norwitz)'
|
21 |
|
|
|
22 |
|
|
|
23 |
|
|
# TODO:
|
24 |
|
|
# * Tokens should never be exported, need to convert to Nodes
|
25 |
|
|
# (return types, parameters, etc.)
|
26 |
|
|
# * Handle static class data for templatized classes
|
27 |
|
|
# * Handle casts (both C++ and C-style)
|
28 |
|
|
# * Handle conditions and loops (if/else, switch, for, while/do)
|
29 |
|
|
#
|
30 |
|
|
# TODO much, much later:
|
31 |
|
|
# * Handle #define
|
32 |
|
|
# * exceptions
|
33 |
|
|
|
34 |
|
|
|
35 |
|
|
try:
|
36 |
|
|
# Python 3.x
|
37 |
|
|
import builtins
|
38 |
|
|
except ImportError:
|
39 |
|
|
# Python 2.x
|
40 |
|
|
import __builtin__ as builtins
|
41 |
|
|
|
42 |
|
|
import sys
|
43 |
|
|
import traceback
|
44 |
|
|
|
45 |
|
|
from cpp import keywords
|
46 |
|
|
from cpp import tokenize
|
47 |
|
|
from cpp import utils
|
48 |
|
|
|
49 |
|
|
|
50 |
|
|
if not hasattr(builtins, 'reversed'):
|
51 |
|
|
# Support Python 2.3 and earlier.
|
52 |
|
|
def reversed(seq):
|
53 |
|
|
for i in range(len(seq)-1, -1, -1):
|
54 |
|
|
yield seq[i]
|
55 |
|
|
|
56 |
|
|
if not hasattr(builtins, 'next'):
|
57 |
|
|
# Support Python 2.5 and earlier.
|
58 |
|
|
def next(obj):
|
59 |
|
|
return obj.next()
|
60 |
|
|
|
61 |
|
|
|
62 |
|
|
VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3)
|
63 |
|
|
|
64 |
|
|
FUNCTION_NONE = 0x00
|
65 |
|
|
FUNCTION_CONST = 0x01
|
66 |
|
|
FUNCTION_VIRTUAL = 0x02
|
67 |
|
|
FUNCTION_PURE_VIRTUAL = 0x04
|
68 |
|
|
FUNCTION_CTOR = 0x08
|
69 |
|
|
FUNCTION_DTOR = 0x10
|
70 |
|
|
FUNCTION_ATTRIBUTE = 0x20
|
71 |
|
|
FUNCTION_UNKNOWN_ANNOTATION = 0x40
|
72 |
|
|
FUNCTION_THROW = 0x80
|
73 |
|
|
FUNCTION_OVERRIDE = 0x100
|
74 |
|
|
|
75 |
|
|
"""
|
76 |
|
|
These are currently unused. Should really handle these properly at some point.
|
77 |
|
|
|
78 |
|
|
TYPE_MODIFIER_INLINE = 0x010000
|
79 |
|
|
TYPE_MODIFIER_EXTERN = 0x020000
|
80 |
|
|
TYPE_MODIFIER_STATIC = 0x040000
|
81 |
|
|
TYPE_MODIFIER_CONST = 0x080000
|
82 |
|
|
TYPE_MODIFIER_REGISTER = 0x100000
|
83 |
|
|
TYPE_MODIFIER_VOLATILE = 0x200000
|
84 |
|
|
TYPE_MODIFIER_MUTABLE = 0x400000
|
85 |
|
|
|
86 |
|
|
TYPE_MODIFIER_MAP = {
|
87 |
|
|
'inline': TYPE_MODIFIER_INLINE,
|
88 |
|
|
'extern': TYPE_MODIFIER_EXTERN,
|
89 |
|
|
'static': TYPE_MODIFIER_STATIC,
|
90 |
|
|
'const': TYPE_MODIFIER_CONST,
|
91 |
|
|
'register': TYPE_MODIFIER_REGISTER,
|
92 |
|
|
'volatile': TYPE_MODIFIER_VOLATILE,
|
93 |
|
|
'mutable': TYPE_MODIFIER_MUTABLE,
|
94 |
|
|
}
|
95 |
|
|
"""
|
96 |
|
|
|
97 |
|
|
_INTERNAL_TOKEN = 'internal'
|
98 |
|
|
_NAMESPACE_POP = 'ns-pop'
|
99 |
|
|
|
100 |
|
|
|
101 |
|
|
# TODO(nnorwitz): use this as a singleton for templated_types, etc
|
102 |
|
|
# where we don't want to create a new empty dict each time. It is also const.
|
103 |
|
|
class _NullDict(object):
|
104 |
|
|
__contains__ = lambda self: False
|
105 |
|
|
keys = values = items = iterkeys = itervalues = iteritems = lambda self: ()
|
106 |
|
|
|
107 |
|
|
|
108 |
|
|
# TODO(nnorwitz): move AST nodes into a separate module.
|
109 |
|
|
class Node(object):
|
110 |
|
|
"""Base AST node."""
|
111 |
|
|
|
112 |
|
|
def __init__(self, start, end):
|
113 |
|
|
self.start = start
|
114 |
|
|
self.end = end
|
115 |
|
|
|
116 |
|
|
def IsDeclaration(self):
|
117 |
|
|
"""Returns bool if this node is a declaration."""
|
118 |
|
|
return False
|
119 |
|
|
|
120 |
|
|
def IsDefinition(self):
|
121 |
|
|
"""Returns bool if this node is a definition."""
|
122 |
|
|
return False
|
123 |
|
|
|
124 |
|
|
def IsExportable(self):
|
125 |
|
|
"""Returns bool if this node exportable from a header file."""
|
126 |
|
|
return False
|
127 |
|
|
|
128 |
|
|
def Requires(self, node):
|
129 |
|
|
"""Does this AST node require the definition of the node passed in?"""
|
130 |
|
|
return False
|
131 |
|
|
|
132 |
|
|
def XXX__str__(self):
|
133 |
|
|
return self._StringHelper(self.__class__.__name__, '')
|
134 |
|
|
|
135 |
|
|
def _StringHelper(self, name, suffix):
|
136 |
|
|
if not utils.DEBUG:
|
137 |
|
|
return '%s(%s)' % (name, suffix)
|
138 |
|
|
return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix)
|
139 |
|
|
|
140 |
|
|
def __repr__(self):
|
141 |
|
|
return str(self)
|
142 |
|
|
|
143 |
|
|
|
144 |
|
|
class Define(Node):
|
145 |
|
|
def __init__(self, start, end, name, definition):
|
146 |
|
|
Node.__init__(self, start, end)
|
147 |
|
|
self.name = name
|
148 |
|
|
self.definition = definition
|
149 |
|
|
|
150 |
|
|
def __str__(self):
|
151 |
|
|
value = '%s %s' % (self.name, self.definition)
|
152 |
|
|
return self._StringHelper(self.__class__.__name__, value)
|
153 |
|
|
|
154 |
|
|
|
155 |
|
|
class Include(Node):
|
156 |
|
|
def __init__(self, start, end, filename, system):
|
157 |
|
|
Node.__init__(self, start, end)
|
158 |
|
|
self.filename = filename
|
159 |
|
|
self.system = system
|
160 |
|
|
|
161 |
|
|
def __str__(self):
|
162 |
|
|
fmt = '"%s"'
|
163 |
|
|
if self.system:
|
164 |
|
|
fmt = '<%s>'
|
165 |
|
|
return self._StringHelper(self.__class__.__name__, fmt % self.filename)
|
166 |
|
|
|
167 |
|
|
|
168 |
|
|
class Goto(Node):
|
169 |
|
|
def __init__(self, start, end, label):
|
170 |
|
|
Node.__init__(self, start, end)
|
171 |
|
|
self.label = label
|
172 |
|
|
|
173 |
|
|
def __str__(self):
|
174 |
|
|
return self._StringHelper(self.__class__.__name__, str(self.label))
|
175 |
|
|
|
176 |
|
|
|
177 |
|
|
class Expr(Node):
|
178 |
|
|
def __init__(self, start, end, expr):
|
179 |
|
|
Node.__init__(self, start, end)
|
180 |
|
|
self.expr = expr
|
181 |
|
|
|
182 |
|
|
def Requires(self, node):
|
183 |
|
|
# TODO(nnorwitz): impl.
|
184 |
|
|
return False
|
185 |
|
|
|
186 |
|
|
def __str__(self):
|
187 |
|
|
return self._StringHelper(self.__class__.__name__, str(self.expr))
|
188 |
|
|
|
189 |
|
|
|
190 |
|
|
class Return(Expr):
|
191 |
|
|
pass
|
192 |
|
|
|
193 |
|
|
|
194 |
|
|
class Delete(Expr):
|
195 |
|
|
pass
|
196 |
|
|
|
197 |
|
|
|
198 |
|
|
class Friend(Expr):
|
199 |
|
|
def __init__(self, start, end, expr, namespace):
|
200 |
|
|
Expr.__init__(self, start, end, expr)
|
201 |
|
|
self.namespace = namespace[:]
|
202 |
|
|
|
203 |
|
|
|
204 |
|
|
class Using(Node):
|
205 |
|
|
def __init__(self, start, end, names):
|
206 |
|
|
Node.__init__(self, start, end)
|
207 |
|
|
self.names = names
|
208 |
|
|
|
209 |
|
|
def __str__(self):
|
210 |
|
|
return self._StringHelper(self.__class__.__name__, str(self.names))
|
211 |
|
|
|
212 |
|
|
|
213 |
|
|
class Parameter(Node):
|
214 |
|
|
def __init__(self, start, end, name, parameter_type, default):
|
215 |
|
|
Node.__init__(self, start, end)
|
216 |
|
|
self.name = name
|
217 |
|
|
self.type = parameter_type
|
218 |
|
|
self.default = default
|
219 |
|
|
|
220 |
|
|
def Requires(self, node):
|
221 |
|
|
# TODO(nnorwitz): handle namespaces, etc.
|
222 |
|
|
return self.type.name == node.name
|
223 |
|
|
|
224 |
|
|
def __str__(self):
|
225 |
|
|
name = str(self.type)
|
226 |
|
|
suffix = '%s %s' % (name, self.name)
|
227 |
|
|
if self.default:
|
228 |
|
|
suffix += ' = ' + ''.join([d.name for d in self.default])
|
229 |
|
|
return self._StringHelper(self.__class__.__name__, suffix)
|
230 |
|
|
|
231 |
|
|
|
232 |
|
|
class _GenericDeclaration(Node):
|
233 |
|
|
def __init__(self, start, end, name, namespace):
|
234 |
|
|
Node.__init__(self, start, end)
|
235 |
|
|
self.name = name
|
236 |
|
|
self.namespace = namespace[:]
|
237 |
|
|
|
238 |
|
|
def FullName(self):
|
239 |
|
|
prefix = ''
|
240 |
|
|
if self.namespace and self.namespace[-1]:
|
241 |
|
|
prefix = '::'.join(self.namespace) + '::'
|
242 |
|
|
return prefix + self.name
|
243 |
|
|
|
244 |
|
|
def _TypeStringHelper(self, suffix):
|
245 |
|
|
if self.namespace:
|
246 |
|
|
names = [n or '<anonymous>' for n in self.namespace]
|
247 |
|
|
suffix += ' in ' + '::'.join(names)
|
248 |
|
|
return self._StringHelper(self.__class__.__name__, suffix)
|
249 |
|
|
|
250 |
|
|
|
251 |
|
|
# TODO(nnorwitz): merge with Parameter in some way?
|
252 |
|
|
class VariableDeclaration(_GenericDeclaration):
|
253 |
|
|
def __init__(self, start, end, name, var_type, initial_value, namespace):
|
254 |
|
|
_GenericDeclaration.__init__(self, start, end, name, namespace)
|
255 |
|
|
self.type = var_type
|
256 |
|
|
self.initial_value = initial_value
|
257 |
|
|
|
258 |
|
|
def Requires(self, node):
|
259 |
|
|
# TODO(nnorwitz): handle namespaces, etc.
|
260 |
|
|
return self.type.name == node.name
|
261 |
|
|
|
262 |
|
|
def ToString(self):
|
263 |
|
|
"""Return a string that tries to reconstitute the variable decl."""
|
264 |
|
|
suffix = '%s %s' % (self.type, self.name)
|
265 |
|
|
if self.initial_value:
|
266 |
|
|
suffix += ' = ' + self.initial_value
|
267 |
|
|
return suffix
|
268 |
|
|
|
269 |
|
|
def __str__(self):
|
270 |
|
|
return self._StringHelper(self.__class__.__name__, self.ToString())
|
271 |
|
|
|
272 |
|
|
|
273 |
|
|
class Typedef(_GenericDeclaration):
|
274 |
|
|
def __init__(self, start, end, name, alias, namespace):
|
275 |
|
|
_GenericDeclaration.__init__(self, start, end, name, namespace)
|
276 |
|
|
self.alias = alias
|
277 |
|
|
|
278 |
|
|
def IsDefinition(self):
|
279 |
|
|
return True
|
280 |
|
|
|
281 |
|
|
def IsExportable(self):
|
282 |
|
|
return True
|
283 |
|
|
|
284 |
|
|
def Requires(self, node):
|
285 |
|
|
# TODO(nnorwitz): handle namespaces, etc.
|
286 |
|
|
name = node.name
|
287 |
|
|
for token in self.alias:
|
288 |
|
|
if token is not None and name == token.name:
|
289 |
|
|
return True
|
290 |
|
|
return False
|
291 |
|
|
|
292 |
|
|
def __str__(self):
|
293 |
|
|
suffix = '%s, %s' % (self.name, self.alias)
|
294 |
|
|
return self._TypeStringHelper(suffix)
|
295 |
|
|
|
296 |
|
|
|
297 |
|
|
class _NestedType(_GenericDeclaration):
|
298 |
|
|
def __init__(self, start, end, name, fields, namespace):
|
299 |
|
|
_GenericDeclaration.__init__(self, start, end, name, namespace)
|
300 |
|
|
self.fields = fields
|
301 |
|
|
|
302 |
|
|
def IsDefinition(self):
|
303 |
|
|
return True
|
304 |
|
|
|
305 |
|
|
def IsExportable(self):
|
306 |
|
|
return True
|
307 |
|
|
|
308 |
|
|
def __str__(self):
|
309 |
|
|
suffix = '%s, {%s}' % (self.name, self.fields)
|
310 |
|
|
return self._TypeStringHelper(suffix)
|
311 |
|
|
|
312 |
|
|
|
313 |
|
|
class Union(_NestedType):
|
314 |
|
|
pass
|
315 |
|
|
|
316 |
|
|
|
317 |
|
|
class Enum(_NestedType):
|
318 |
|
|
pass
|
319 |
|
|
|
320 |
|
|
|
321 |
|
|
class Class(_GenericDeclaration):
|
322 |
|
|
def __init__(self, start, end, name, bases, templated_types, body, namespace):
|
323 |
|
|
_GenericDeclaration.__init__(self, start, end, name, namespace)
|
324 |
|
|
self.bases = bases
|
325 |
|
|
self.body = body
|
326 |
|
|
self.templated_types = templated_types
|
327 |
|
|
|
328 |
|
|
def IsDeclaration(self):
|
329 |
|
|
return self.bases is None and self.body is None
|
330 |
|
|
|
331 |
|
|
def IsDefinition(self):
|
332 |
|
|
return not self.IsDeclaration()
|
333 |
|
|
|
334 |
|
|
def IsExportable(self):
|
335 |
|
|
return not self.IsDeclaration()
|
336 |
|
|
|
337 |
|
|
def Requires(self, node):
|
338 |
|
|
# TODO(nnorwitz): handle namespaces, etc.
|
339 |
|
|
if self.bases:
|
340 |
|
|
for token_list in self.bases:
|
341 |
|
|
# TODO(nnorwitz): bases are tokens, do name comparision.
|
342 |
|
|
for token in token_list:
|
343 |
|
|
if token.name == node.name:
|
344 |
|
|
return True
|
345 |
|
|
# TODO(nnorwitz): search in body too.
|
346 |
|
|
return False
|
347 |
|
|
|
348 |
|
|
def __str__(self):
|
349 |
|
|
name = self.name
|
350 |
|
|
if self.templated_types:
|
351 |
|
|
name += '<%s>' % self.templated_types
|
352 |
|
|
suffix = '%s, %s, %s' % (name, self.bases, self.body)
|
353 |
|
|
return self._TypeStringHelper(suffix)
|
354 |
|
|
|
355 |
|
|
|
356 |
|
|
class Struct(Class):
|
357 |
|
|
pass
|
358 |
|
|
|
359 |
|
|
|
360 |
|
|
class Function(_GenericDeclaration):
|
361 |
|
|
def __init__(self, start, end, name, return_type, parameters,
|
362 |
|
|
modifiers, templated_types, body, namespace):
|
363 |
|
|
_GenericDeclaration.__init__(self, start, end, name, namespace)
|
364 |
|
|
converter = TypeConverter(namespace)
|
365 |
|
|
self.return_type = converter.CreateReturnType(return_type)
|
366 |
|
|
self.parameters = converter.ToParameters(parameters)
|
367 |
|
|
self.modifiers = modifiers
|
368 |
|
|
self.body = body
|
369 |
|
|
self.templated_types = templated_types
|
370 |
|
|
|
371 |
|
|
def IsDeclaration(self):
|
372 |
|
|
return self.body is None
|
373 |
|
|
|
374 |
|
|
def IsDefinition(self):
|
375 |
|
|
return self.body is not None
|
376 |
|
|
|
377 |
|
|
def IsExportable(self):
|
378 |
|
|
if self.return_type and 'static' in self.return_type.modifiers:
|
379 |
|
|
return False
|
380 |
|
|
return None not in self.namespace
|
381 |
|
|
|
382 |
|
|
def Requires(self, node):
|
383 |
|
|
if self.parameters:
|
384 |
|
|
# TODO(nnorwitz): parameters are tokens, do name comparision.
|
385 |
|
|
for p in self.parameters:
|
386 |
|
|
if p.name == node.name:
|
387 |
|
|
return True
|
388 |
|
|
# TODO(nnorwitz): search in body too.
|
389 |
|
|
return False
|
390 |
|
|
|
391 |
|
|
def __str__(self):
|
392 |
|
|
# TODO(nnorwitz): add templated_types.
|
393 |
|
|
suffix = ('%s %s(%s), 0x%02x, %s' %
|
394 |
|
|
(self.return_type, self.name, self.parameters,
|
395 |
|
|
self.modifiers, self.body))
|
396 |
|
|
return self._TypeStringHelper(suffix)
|
397 |
|
|
|
398 |
|
|
|
399 |
|
|
class Method(Function):
|
400 |
|
|
def __init__(self, start, end, name, in_class, return_type, parameters,
|
401 |
|
|
modifiers, templated_types, body, namespace):
|
402 |
|
|
Function.__init__(self, start, end, name, return_type, parameters,
|
403 |
|
|
modifiers, templated_types, body, namespace)
|
404 |
|
|
# TODO(nnorwitz): in_class could also be a namespace which can
|
405 |
|
|
# mess up finding functions properly.
|
406 |
|
|
self.in_class = in_class
|
407 |
|
|
|
408 |
|
|
|
409 |
|
|
class Type(_GenericDeclaration):
|
410 |
|
|
"""Type used for any variable (eg class, primitive, struct, etc)."""
|
411 |
|
|
|
412 |
|
|
def __init__(self, start, end, name, templated_types, modifiers,
|
413 |
|
|
reference, pointer, array):
|
414 |
|
|
"""
|
415 |
|
|
Args:
|
416 |
|
|
name: str name of main type
|
417 |
|
|
templated_types: [Class (Type?)] template type info between <>
|
418 |
|
|
modifiers: [str] type modifiers (keywords) eg, const, mutable, etc.
|
419 |
|
|
reference, pointer, array: bools
|
420 |
|
|
"""
|
421 |
|
|
_GenericDeclaration.__init__(self, start, end, name, [])
|
422 |
|
|
self.templated_types = templated_types
|
423 |
|
|
if not name and modifiers:
|
424 |
|
|
self.name = modifiers.pop()
|
425 |
|
|
self.modifiers = modifiers
|
426 |
|
|
self.reference = reference
|
427 |
|
|
self.pointer = pointer
|
428 |
|
|
self.array = array
|
429 |
|
|
|
430 |
|
|
def __str__(self):
|
431 |
|
|
prefix = ''
|
432 |
|
|
if self.modifiers:
|
433 |
|
|
prefix = ' '.join(self.modifiers) + ' '
|
434 |
|
|
name = str(self.name)
|
435 |
|
|
if self.templated_types:
|
436 |
|
|
name += '<%s>' % self.templated_types
|
437 |
|
|
suffix = prefix + name
|
438 |
|
|
if self.reference:
|
439 |
|
|
suffix += '&'
|
440 |
|
|
if self.pointer:
|
441 |
|
|
suffix += '*'
|
442 |
|
|
if self.array:
|
443 |
|
|
suffix += '[]'
|
444 |
|
|
return self._TypeStringHelper(suffix)
|
445 |
|
|
|
446 |
|
|
# By definition, Is* are always False. A Type can only exist in
|
447 |
|
|
# some sort of variable declaration, parameter, or return value.
|
448 |
|
|
def IsDeclaration(self):
|
449 |
|
|
return False
|
450 |
|
|
|
451 |
|
|
def IsDefinition(self):
|
452 |
|
|
return False
|
453 |
|
|
|
454 |
|
|
def IsExportable(self):
|
455 |
|
|
return False
|
456 |
|
|
|
457 |
|
|
|
458 |
|
|
class TypeConverter(object):
|
459 |
|
|
|
460 |
|
|
def __init__(self, namespace_stack):
|
461 |
|
|
self.namespace_stack = namespace_stack
|
462 |
|
|
|
463 |
|
|
def _GetTemplateEnd(self, tokens, start):
|
464 |
|
|
count = 1
|
465 |
|
|
end = start
|
466 |
|
|
while 1:
|
467 |
|
|
token = tokens[end]
|
468 |
|
|
end += 1
|
469 |
|
|
if token.name == '<':
|
470 |
|
|
count += 1
|
471 |
|
|
elif token.name == '>':
|
472 |
|
|
count -= 1
|
473 |
|
|
if count == 0:
|
474 |
|
|
break
|
475 |
|
|
return tokens[start:end-1], end
|
476 |
|
|
|
477 |
|
|
def ToType(self, tokens):
|
478 |
|
|
"""Convert [Token,...] to [Class(...), ] useful for base classes.
|
479 |
|
|
For example, code like class Foo : public Bar<x, y> { ... };
|
480 |
|
|
the "Bar<x, y>" portion gets converted to an AST.
|
481 |
|
|
|
482 |
|
|
Returns:
|
483 |
|
|
[Class(...), ...]
|
484 |
|
|
"""
|
485 |
|
|
result = []
|
486 |
|
|
name_tokens = []
|
487 |
|
|
reference = pointer = array = False
|
488 |
|
|
|
489 |
|
|
def AddType(templated_types):
|
490 |
|
|
# Partition tokens into name and modifier tokens.
|
491 |
|
|
names = []
|
492 |
|
|
modifiers = []
|
493 |
|
|
for t in name_tokens:
|
494 |
|
|
if keywords.IsKeyword(t.name):
|
495 |
|
|
modifiers.append(t.name)
|
496 |
|
|
else:
|
497 |
|
|
names.append(t.name)
|
498 |
|
|
name = ''.join(names)
|
499 |
|
|
if name_tokens:
|
500 |
|
|
result.append(Type(name_tokens[0].start, name_tokens[-1].end,
|
501 |
|
|
name, templated_types, modifiers,
|
502 |
|
|
reference, pointer, array))
|
503 |
|
|
del name_tokens[:]
|
504 |
|
|
|
505 |
|
|
i = 0
|
506 |
|
|
end = len(tokens)
|
507 |
|
|
while i < end:
|
508 |
|
|
token = tokens[i]
|
509 |
|
|
if token.name == '<':
|
510 |
|
|
new_tokens, new_end = self._GetTemplateEnd(tokens, i+1)
|
511 |
|
|
AddType(self.ToType(new_tokens))
|
512 |
|
|
# If there is a comma after the template, we need to consume
|
513 |
|
|
# that here otherwise it becomes part of the name.
|
514 |
|
|
i = new_end
|
515 |
|
|
reference = pointer = array = False
|
516 |
|
|
elif token.name == ',':
|
517 |
|
|
AddType([])
|
518 |
|
|
reference = pointer = array = False
|
519 |
|
|
elif token.name == '*':
|
520 |
|
|
pointer = True
|
521 |
|
|
elif token.name == '&':
|
522 |
|
|
reference = True
|
523 |
|
|
elif token.name == '[':
|
524 |
|
|
pointer = True
|
525 |
|
|
elif token.name == ']':
|
526 |
|
|
pass
|
527 |
|
|
else:
|
528 |
|
|
name_tokens.append(token)
|
529 |
|
|
i += 1
|
530 |
|
|
|
531 |
|
|
if name_tokens:
|
532 |
|
|
# No '<' in the tokens, just a simple name and no template.
|
533 |
|
|
AddType([])
|
534 |
|
|
return result
|
535 |
|
|
|
536 |
|
|
def DeclarationToParts(self, parts, needs_name_removed):
|
537 |
|
|
name = None
|
538 |
|
|
default = []
|
539 |
|
|
if needs_name_removed:
|
540 |
|
|
# Handle default (initial) values properly.
|
541 |
|
|
for i, t in enumerate(parts):
|
542 |
|
|
if t.name == '=':
|
543 |
|
|
default = parts[i+1:]
|
544 |
|
|
name = parts[i-1].name
|
545 |
|
|
if name == ']' and parts[i-2].name == '[':
|
546 |
|
|
name = parts[i-3].name
|
547 |
|
|
i -= 1
|
548 |
|
|
parts = parts[:i-1]
|
549 |
|
|
break
|
550 |
|
|
else:
|
551 |
|
|
if parts[-1].token_type == tokenize.NAME:
|
552 |
|
|
name = parts.pop().name
|
553 |
|
|
else:
|
554 |
|
|
# TODO(nnorwitz): this is a hack that happens for code like
|
555 |
|
|
# Register(Foo<T>); where it thinks this is a function call
|
556 |
|
|
# but it's actually a declaration.
|
557 |
|
|
name = '???'
|
558 |
|
|
modifiers = []
|
559 |
|
|
type_name = []
|
560 |
|
|
other_tokens = []
|
561 |
|
|
templated_types = []
|
562 |
|
|
i = 0
|
563 |
|
|
end = len(parts)
|
564 |
|
|
while i < end:
|
565 |
|
|
p = parts[i]
|
566 |
|
|
if keywords.IsKeyword(p.name):
|
567 |
|
|
modifiers.append(p.name)
|
568 |
|
|
elif p.name == '<':
|
569 |
|
|
templated_tokens, new_end = self._GetTemplateEnd(parts, i+1)
|
570 |
|
|
templated_types = self.ToType(templated_tokens)
|
571 |
|
|
i = new_end - 1
|
572 |
|
|
# Don't add a spurious :: to data members being initialized.
|
573 |
|
|
next_index = i + 1
|
574 |
|
|
if next_index < end and parts[next_index].name == '::':
|
575 |
|
|
i += 1
|
576 |
|
|
elif p.name in ('[', ']', '='):
|
577 |
|
|
# These are handled elsewhere.
|
578 |
|
|
other_tokens.append(p)
|
579 |
|
|
elif p.name not in ('*', '&', '>'):
|
580 |
|
|
# Ensure that names have a space between them.
|
581 |
|
|
if (type_name and type_name[-1].token_type == tokenize.NAME and
|
582 |
|
|
p.token_type == tokenize.NAME):
|
583 |
|
|
type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
|
584 |
|
|
type_name.append(p)
|
585 |
|
|
else:
|
586 |
|
|
other_tokens.append(p)
|
587 |
|
|
i += 1
|
588 |
|
|
type_name = ''.join([t.name for t in type_name])
|
589 |
|
|
return name, type_name, templated_types, modifiers, default, other_tokens
|
590 |
|
|
|
591 |
|
|
def ToParameters(self, tokens):
|
592 |
|
|
if not tokens:
|
593 |
|
|
return []
|
594 |
|
|
|
595 |
|
|
result = []
|
596 |
|
|
name = type_name = ''
|
597 |
|
|
type_modifiers = []
|
598 |
|
|
pointer = reference = array = False
|
599 |
|
|
first_token = None
|
600 |
|
|
default = []
|
601 |
|
|
|
602 |
|
|
def AddParameter(end):
|
603 |
|
|
if default:
|
604 |
|
|
del default[0] # Remove flag.
|
605 |
|
|
parts = self.DeclarationToParts(type_modifiers, True)
|
606 |
|
|
(name, type_name, templated_types, modifiers,
|
607 |
|
|
unused_default, unused_other_tokens) = parts
|
608 |
|
|
parameter_type = Type(first_token.start, first_token.end,
|
609 |
|
|
type_name, templated_types, modifiers,
|
610 |
|
|
reference, pointer, array)
|
611 |
|
|
p = Parameter(first_token.start, end, name,
|
612 |
|
|
parameter_type, default)
|
613 |
|
|
result.append(p)
|
614 |
|
|
|
615 |
|
|
template_count = 0
|
616 |
|
|
for s in tokens:
|
617 |
|
|
if not first_token:
|
618 |
|
|
first_token = s
|
619 |
|
|
if s.name == '<':
|
620 |
|
|
template_count += 1
|
621 |
|
|
elif s.name == '>':
|
622 |
|
|
template_count -= 1
|
623 |
|
|
if template_count > 0:
|
624 |
|
|
type_modifiers.append(s)
|
625 |
|
|
continue
|
626 |
|
|
|
627 |
|
|
if s.name == ',':
|
628 |
|
|
AddParameter(s.start)
|
629 |
|
|
name = type_name = ''
|
630 |
|
|
type_modifiers = []
|
631 |
|
|
pointer = reference = array = False
|
632 |
|
|
first_token = None
|
633 |
|
|
default = []
|
634 |
|
|
elif s.name == '*':
|
635 |
|
|
pointer = True
|
636 |
|
|
elif s.name == '&':
|
637 |
|
|
reference = True
|
638 |
|
|
elif s.name == '[':
|
639 |
|
|
array = True
|
640 |
|
|
elif s.name == ']':
|
641 |
|
|
pass # Just don't add to type_modifiers.
|
642 |
|
|
elif s.name == '=':
|
643 |
|
|
# Got a default value. Add any value (None) as a flag.
|
644 |
|
|
default.append(None)
|
645 |
|
|
elif default:
|
646 |
|
|
default.append(s)
|
647 |
|
|
else:
|
648 |
|
|
type_modifiers.append(s)
|
649 |
|
|
AddParameter(tokens[-1].end)
|
650 |
|
|
return result
|
651 |
|
|
|
652 |
|
|
def CreateReturnType(self, return_type_seq):
|
653 |
|
|
if not return_type_seq:
|
654 |
|
|
return None
|
655 |
|
|
start = return_type_seq[0].start
|
656 |
|
|
end = return_type_seq[-1].end
|
657 |
|
|
_, name, templated_types, modifiers, default, other_tokens = \
|
658 |
|
|
self.DeclarationToParts(return_type_seq, False)
|
659 |
|
|
names = [n.name for n in other_tokens]
|
660 |
|
|
reference = '&' in names
|
661 |
|
|
pointer = '*' in names
|
662 |
|
|
array = '[' in names
|
663 |
|
|
return Type(start, end, name, templated_types, modifiers,
|
664 |
|
|
reference, pointer, array)
|
665 |
|
|
|
666 |
|
|
def GetTemplateIndices(self, names):
|
667 |
|
|
# names is a list of strings.
|
668 |
|
|
start = names.index('<')
|
669 |
|
|
end = len(names) - 1
|
670 |
|
|
while end > 0:
|
671 |
|
|
if names[end] == '>':
|
672 |
|
|
break
|
673 |
|
|
end -= 1
|
674 |
|
|
return start, end+1
|
675 |
|
|
|
676 |
|
|
class AstBuilder(object):
|
677 |
|
|
def __init__(self, token_stream, filename, in_class='', visibility=None,
|
678 |
|
|
namespace_stack=[]):
|
679 |
|
|
self.tokens = token_stream
|
680 |
|
|
self.filename = filename
|
681 |
|
|
# TODO(nnorwitz): use a better data structure (deque) for the queue.
|
682 |
|
|
# Switching directions of the "queue" improved perf by about 25%.
|
683 |
|
|
# Using a deque should be even better since we access from both sides.
|
684 |
|
|
self.token_queue = []
|
685 |
|
|
self.namespace_stack = namespace_stack[:]
|
686 |
|
|
self.in_class = in_class
|
687 |
|
|
if in_class is None:
|
688 |
|
|
self.in_class_name_only = None
|
689 |
|
|
else:
|
690 |
|
|
self.in_class_name_only = in_class.split('::')[-1]
|
691 |
|
|
self.visibility = visibility
|
692 |
|
|
self.in_function = False
|
693 |
|
|
self.current_token = None
|
694 |
|
|
# Keep the state whether we are currently handling a typedef or not.
|
695 |
|
|
self._handling_typedef = False
|
696 |
|
|
|
697 |
|
|
self.converter = TypeConverter(self.namespace_stack)
|
698 |
|
|
|
699 |
|
|
def HandleError(self, msg, token):
|
700 |
|
|
printable_queue = list(reversed(self.token_queue[-20:]))
|
701 |
|
|
sys.stderr.write('Got %s in %s @ %s %s\n' %
|
702 |
|
|
(msg, self.filename, token, printable_queue))
|
703 |
|
|
|
704 |
|
|
def Generate(self):
|
705 |
|
|
while 1:
|
706 |
|
|
token = self._GetNextToken()
|
707 |
|
|
if not token:
|
708 |
|
|
break
|
709 |
|
|
|
710 |
|
|
# Get the next token.
|
711 |
|
|
self.current_token = token
|
712 |
|
|
|
713 |
|
|
# Dispatch on the next token type.
|
714 |
|
|
if token.token_type == _INTERNAL_TOKEN:
|
715 |
|
|
if token.name == _NAMESPACE_POP:
|
716 |
|
|
self.namespace_stack.pop()
|
717 |
|
|
continue
|
718 |
|
|
|
719 |
|
|
try:
|
720 |
|
|
result = self._GenerateOne(token)
|
721 |
|
|
if result is not None:
|
722 |
|
|
yield result
|
723 |
|
|
except:
|
724 |
|
|
self.HandleError('exception', token)
|
725 |
|
|
raise
|
726 |
|
|
|
727 |
|
|
def _CreateVariable(self, pos_token, name, type_name, type_modifiers,
|
728 |
|
|
ref_pointer_name_seq, templated_types, value=None):
|
729 |
|
|
reference = '&' in ref_pointer_name_seq
|
730 |
|
|
pointer = '*' in ref_pointer_name_seq
|
731 |
|
|
array = '[' in ref_pointer_name_seq
|
732 |
|
|
var_type = Type(pos_token.start, pos_token.end, type_name,
|
733 |
|
|
templated_types, type_modifiers,
|
734 |
|
|
reference, pointer, array)
|
735 |
|
|
return VariableDeclaration(pos_token.start, pos_token.end,
|
736 |
|
|
name, var_type, value, self.namespace_stack)
|
737 |
|
|
|
738 |
|
|
def _GenerateOne(self, token):
|
739 |
|
|
if token.token_type == tokenize.NAME:
|
740 |
|
|
if (keywords.IsKeyword(token.name) and
|
741 |
|
|
not keywords.IsBuiltinType(token.name)):
|
742 |
|
|
method = getattr(self, 'handle_' + token.name)
|
743 |
|
|
return method()
|
744 |
|
|
elif token.name == self.in_class_name_only:
|
745 |
|
|
# The token name is the same as the class, must be a ctor if
|
746 |
|
|
# there is a paren. Otherwise, it's the return type.
|
747 |
|
|
# Peek ahead to get the next token to figure out which.
|
748 |
|
|
next = self._GetNextToken()
|
749 |
|
|
self._AddBackToken(next)
|
750 |
|
|
if next.token_type == tokenize.SYNTAX and next.name == '(':
|
751 |
|
|
return self._GetMethod([token], FUNCTION_CTOR, None, True)
|
752 |
|
|
# Fall through--handle like any other method.
|
753 |
|
|
|
754 |
|
|
# Handle data or function declaration/definition.
|
755 |
|
|
syntax = tokenize.SYNTAX
|
756 |
|
|
temp_tokens, last_token = \
|
757 |
|
|
self._GetVarTokensUpTo(syntax, '(', ';', '{', '[')
|
758 |
|
|
temp_tokens.insert(0, token)
|
759 |
|
|
if last_token.name == '(':
|
760 |
|
|
# If there is an assignment before the paren,
|
761 |
|
|
# this is an expression, not a method.
|
762 |
|
|
expr = bool([e for e in temp_tokens if e.name == '='])
|
763 |
|
|
if expr:
|
764 |
|
|
new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
|
765 |
|
|
temp_tokens.append(last_token)
|
766 |
|
|
temp_tokens.extend(new_temp)
|
767 |
|
|
last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)
|
768 |
|
|
|
769 |
|
|
if last_token.name == '[':
|
770 |
|
|
# Handle array, this isn't a method, unless it's an operator.
|
771 |
|
|
# TODO(nnorwitz): keep the size somewhere.
|
772 |
|
|
# unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']')
|
773 |
|
|
temp_tokens.append(last_token)
|
774 |
|
|
if temp_tokens[-2].name == 'operator':
|
775 |
|
|
temp_tokens.append(self._GetNextToken())
|
776 |
|
|
else:
|
777 |
|
|
temp_tokens2, last_token = \
|
778 |
|
|
self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
|
779 |
|
|
temp_tokens.extend(temp_tokens2)
|
780 |
|
|
|
781 |
|
|
if last_token.name == ';':
|
782 |
|
|
# Handle data, this isn't a method.
|
783 |
|
|
parts = self.converter.DeclarationToParts(temp_tokens, True)
|
784 |
|
|
(name, type_name, templated_types, modifiers, default,
|
785 |
|
|
unused_other_tokens) = parts
|
786 |
|
|
|
787 |
|
|
t0 = temp_tokens[0]
|
788 |
|
|
names = [t.name for t in temp_tokens]
|
789 |
|
|
if templated_types:
|
790 |
|
|
start, end = self.converter.GetTemplateIndices(names)
|
791 |
|
|
names = names[:start] + names[end:]
|
792 |
|
|
default = ''.join([t.name for t in default])
|
793 |
|
|
return self._CreateVariable(t0, name, type_name, modifiers,
|
794 |
|
|
names, templated_types, default)
|
795 |
|
|
if last_token.name == '{':
|
796 |
|
|
self._AddBackTokens(temp_tokens[1:])
|
797 |
|
|
self._AddBackToken(last_token)
|
798 |
|
|
method_name = temp_tokens[0].name
|
799 |
|
|
method = getattr(self, 'handle_' + method_name, None)
|
800 |
|
|
if not method:
|
801 |
|
|
# Must be declaring a variable.
|
802 |
|
|
# TODO(nnorwitz): handle the declaration.
|
803 |
|
|
return None
|
804 |
|
|
return method()
|
805 |
|
|
return self._GetMethod(temp_tokens, 0, None, False)
|
806 |
|
|
elif token.token_type == tokenize.SYNTAX:
|
807 |
|
|
if token.name == '~' and self.in_class:
|
808 |
|
|
# Must be a dtor (probably not in method body).
|
809 |
|
|
token = self._GetNextToken()
|
810 |
|
|
# self.in_class can contain A::Name, but the dtor will only
|
811 |
|
|
# be Name. Make sure to compare against the right value.
|
812 |
|
|
if (token.token_type == tokenize.NAME and
|
813 |
|
|
token.name == self.in_class_name_only):
|
814 |
|
|
return self._GetMethod([token], FUNCTION_DTOR, None, True)
|
815 |
|
|
# TODO(nnorwitz): handle a lot more syntax.
|
816 |
|
|
elif token.token_type == tokenize.PREPROCESSOR:
|
817 |
|
|
# TODO(nnorwitz): handle more preprocessor directives.
|
818 |
|
|
# token starts with a #, so remove it and strip whitespace.
|
819 |
|
|
name = token.name[1:].lstrip()
|
820 |
|
|
if name.startswith('include'):
|
821 |
|
|
# Remove "include".
|
822 |
|
|
name = name[7:].strip()
|
823 |
|
|
assert name
|
824 |
|
|
# Handle #include \<newline> "header-on-second-line.h".
|
825 |
|
|
if name.startswith('\\'):
|
826 |
|
|
name = name[1:].strip()
|
827 |
|
|
assert name[0] in '<"', token
|
828 |
|
|
assert name[-1] in '>"', token
|
829 |
|
|
system = name[0] == '<'
|
830 |
|
|
filename = name[1:-1]
|
831 |
|
|
return Include(token.start, token.end, filename, system)
|
832 |
|
|
if name.startswith('define'):
|
833 |
|
|
# Remove "define".
|
834 |
|
|
name = name[6:].strip()
|
835 |
|
|
assert name
|
836 |
|
|
value = ''
|
837 |
|
|
for i, c in enumerate(name):
|
838 |
|
|
if c.isspace():
|
839 |
|
|
value = name[i:].lstrip()
|
840 |
|
|
name = name[:i]
|
841 |
|
|
break
|
842 |
|
|
return Define(token.start, token.end, name, value)
|
843 |
|
|
if name.startswith('if') and name[2:3].isspace():
|
844 |
|
|
condition = name[3:].strip()
|
845 |
|
|
if condition.startswith('0') or condition.startswith('(0)'):
|
846 |
|
|
self._SkipIf0Blocks()
|
847 |
|
|
return None
|
848 |
|
|
|
849 |
|
|
def _GetTokensUpTo(self, expected_token_type, expected_token):
|
850 |
|
|
return self._GetVarTokensUpTo(expected_token_type, expected_token)[0]
|
851 |
|
|
|
852 |
|
|
def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens):
|
853 |
|
|
last_token = self._GetNextToken()
|
854 |
|
|
tokens = []
|
855 |
|
|
while (last_token.token_type != expected_token_type or
|
856 |
|
|
last_token.name not in expected_tokens):
|
857 |
|
|
tokens.append(last_token)
|
858 |
|
|
last_token = self._GetNextToken()
|
859 |
|
|
return tokens, last_token
|
860 |
|
|
|
861 |
|
|
# TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necesary.
|
862 |
|
|
def _IgnoreUpTo(self, token_type, token):
|
863 |
|
|
unused_tokens = self._GetTokensUpTo(token_type, token)
|
864 |
|
|
|
865 |
|
|
def _SkipIf0Blocks(self):
|
866 |
|
|
count = 1
|
867 |
|
|
while 1:
|
868 |
|
|
token = self._GetNextToken()
|
869 |
|
|
if token.token_type != tokenize.PREPROCESSOR:
|
870 |
|
|
continue
|
871 |
|
|
|
872 |
|
|
name = token.name[1:].lstrip()
|
873 |
|
|
if name.startswith('endif'):
|
874 |
|
|
count -= 1
|
875 |
|
|
if count == 0:
|
876 |
|
|
break
|
877 |
|
|
elif name.startswith('if'):
|
878 |
|
|
count += 1
|
879 |
|
|
|
880 |
|
|
def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None):
|
881 |
|
|
if GetNextToken is None:
|
882 |
|
|
GetNextToken = self._GetNextToken
|
883 |
|
|
# Assumes the current token is open_paren and we will consume
|
884 |
|
|
# and return up to the close_paren.
|
885 |
|
|
count = 1
|
886 |
|
|
token = GetNextToken()
|
887 |
|
|
while 1:
|
888 |
|
|
if token.token_type == tokenize.SYNTAX:
|
889 |
|
|
if token.name == open_paren:
|
890 |
|
|
count += 1
|
891 |
|
|
elif token.name == close_paren:
|
892 |
|
|
count -= 1
|
893 |
|
|
if count == 0:
|
894 |
|
|
break
|
895 |
|
|
yield token
|
896 |
|
|
token = GetNextToken()
|
897 |
|
|
yield token
|
898 |
|
|
|
899 |
|
|
def _GetParameters(self):
|
900 |
|
|
return self._GetMatchingChar('(', ')')
|
901 |
|
|
|
902 |
|
|
def GetScope(self):
|
903 |
|
|
return self._GetMatchingChar('{', '}')
|
904 |
|
|
|
905 |
|
|
def _GetNextToken(self):
|
906 |
|
|
if self.token_queue:
|
907 |
|
|
return self.token_queue.pop()
|
908 |
|
|
return next(self.tokens)
|
909 |
|
|
|
910 |
|
|
def _AddBackToken(self, token):
|
911 |
|
|
if token.whence == tokenize.WHENCE_STREAM:
|
912 |
|
|
token.whence = tokenize.WHENCE_QUEUE
|
913 |
|
|
self.token_queue.insert(0, token)
|
914 |
|
|
else:
|
915 |
|
|
assert token.whence == tokenize.WHENCE_QUEUE, token
|
916 |
|
|
self.token_queue.append(token)
|
917 |
|
|
|
918 |
|
|
def _AddBackTokens(self, tokens):
|
919 |
|
|
if tokens:
|
920 |
|
|
if tokens[-1].whence == tokenize.WHENCE_STREAM:
|
921 |
|
|
for token in tokens:
|
922 |
|
|
token.whence = tokenize.WHENCE_QUEUE
|
923 |
|
|
self.token_queue[:0] = reversed(tokens)
|
924 |
|
|
else:
|
925 |
|
|
assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
|
926 |
|
|
self.token_queue.extend(reversed(tokens))
|
927 |
|
|
|
928 |
|
|
def GetName(self, seq=None):
|
929 |
|
|
"""Returns ([tokens], next_token_info)."""
|
930 |
|
|
GetNextToken = self._GetNextToken
|
931 |
|
|
if seq is not None:
|
932 |
|
|
it = iter(seq)
|
933 |
|
|
GetNextToken = lambda: next(it)
|
934 |
|
|
next_token = GetNextToken()
|
935 |
|
|
tokens = []
|
936 |
|
|
last_token_was_name = False
|
937 |
|
|
while (next_token.token_type == tokenize.NAME or
|
938 |
|
|
(next_token.token_type == tokenize.SYNTAX and
|
939 |
|
|
next_token.name in ('::', '<'))):
|
940 |
|
|
# Two NAMEs in a row means the identifier should terminate.
|
941 |
|
|
# It's probably some sort of variable declaration.
|
942 |
|
|
if last_token_was_name and next_token.token_type == tokenize.NAME:
|
943 |
|
|
break
|
944 |
|
|
last_token_was_name = next_token.token_type == tokenize.NAME
|
945 |
|
|
tokens.append(next_token)
|
946 |
|
|
# Handle templated names.
|
947 |
|
|
if next_token.name == '<':
|
948 |
|
|
tokens.extend(self._GetMatchingChar('<', '>', GetNextToken))
|
949 |
|
|
last_token_was_name = True
|
950 |
|
|
next_token = GetNextToken()
|
951 |
|
|
return tokens, next_token
|
952 |
|
|
|
953 |
|
|
def GetMethod(self, modifiers, templated_types):
|
954 |
|
|
return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
|
955 |
|
|
assert len(return_type_and_name) >= 1
|
956 |
|
|
return self._GetMethod(return_type_and_name, modifiers, templated_types,
|
957 |
|
|
False)
|
958 |
|
|
|
959 |
|
|
def _GetMethod(self, return_type_and_name, modifiers, templated_types,
|
960 |
|
|
get_paren):
|
961 |
|
|
template_portion = None
|
962 |
|
|
if get_paren:
|
963 |
|
|
token = self._GetNextToken()
|
964 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
965 |
|
|
if token.name == '<':
|
966 |
|
|
# Handle templatized dtors.
|
967 |
|
|
template_portion = [token]
|
968 |
|
|
template_portion.extend(self._GetMatchingChar('<', '>'))
|
969 |
|
|
token = self._GetNextToken()
|
970 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
971 |
|
|
assert token.name == '(', token
|
972 |
|
|
|
973 |
|
|
name = return_type_and_name.pop()
|
974 |
|
|
# Handle templatized ctors.
|
975 |
|
|
if name.name == '>':
|
976 |
|
|
index = 1
|
977 |
|
|
while return_type_and_name[index].name != '<':
|
978 |
|
|
index += 1
|
979 |
|
|
template_portion = return_type_and_name[index:] + [name]
|
980 |
|
|
del return_type_and_name[index:]
|
981 |
|
|
name = return_type_and_name.pop()
|
982 |
|
|
elif name.name == ']':
|
983 |
|
|
rt = return_type_and_name
|
984 |
|
|
assert rt[-1].name == '[', return_type_and_name
|
985 |
|
|
assert rt[-2].name == 'operator', return_type_and_name
|
986 |
|
|
name_seq = return_type_and_name[-2:]
|
987 |
|
|
del return_type_and_name[-2:]
|
988 |
|
|
name = tokenize.Token(tokenize.NAME, 'operator[]',
|
989 |
|
|
name_seq[0].start, name.end)
|
990 |
|
|
# Get the open paren so _GetParameters() below works.
|
991 |
|
|
unused_open_paren = self._GetNextToken()
|
992 |
|
|
|
993 |
|
|
# TODO(nnorwitz): store template_portion.
|
994 |
|
|
return_type = return_type_and_name
|
995 |
|
|
indices = name
|
996 |
|
|
if return_type:
|
997 |
|
|
indices = return_type[0]
|
998 |
|
|
|
999 |
|
|
# Force ctor for templatized ctors.
|
1000 |
|
|
if name.name == self.in_class and not modifiers:
|
1001 |
|
|
modifiers |= FUNCTION_CTOR
|
1002 |
|
|
parameters = list(self._GetParameters())
|
1003 |
|
|
del parameters[-1] # Remove trailing ')'.
|
1004 |
|
|
|
1005 |
|
|
# Handling operator() is especially weird.
|
1006 |
|
|
if name.name == 'operator' and not parameters:
|
1007 |
|
|
token = self._GetNextToken()
|
1008 |
|
|
assert token.name == '(', token
|
1009 |
|
|
parameters = list(self._GetParameters())
|
1010 |
|
|
del parameters[-1] # Remove trailing ')'.
|
1011 |
|
|
|
1012 |
|
|
token = self._GetNextToken()
|
1013 |
|
|
while token.token_type == tokenize.NAME:
|
1014 |
|
|
modifier_token = token
|
1015 |
|
|
token = self._GetNextToken()
|
1016 |
|
|
if modifier_token.name == 'const':
|
1017 |
|
|
modifiers |= FUNCTION_CONST
|
1018 |
|
|
elif modifier_token.name == '__attribute__':
|
1019 |
|
|
# TODO(nnorwitz): handle more __attribute__ details.
|
1020 |
|
|
modifiers |= FUNCTION_ATTRIBUTE
|
1021 |
|
|
assert token.name == '(', token
|
1022 |
|
|
# Consume everything between the (parens).
|
1023 |
|
|
unused_tokens = list(self._GetMatchingChar('(', ')'))
|
1024 |
|
|
token = self._GetNextToken()
|
1025 |
|
|
elif modifier_token.name == 'throw':
|
1026 |
|
|
modifiers |= FUNCTION_THROW
|
1027 |
|
|
assert token.name == '(', token
|
1028 |
|
|
# Consume everything between the (parens).
|
1029 |
|
|
unused_tokens = list(self._GetMatchingChar('(', ')'))
|
1030 |
|
|
token = self._GetNextToken()
|
1031 |
|
|
elif modifier_token.name == 'override':
|
1032 |
|
|
modifiers |= FUNCTION_OVERRIDE
|
1033 |
|
|
elif modifier_token.name == modifier_token.name.upper():
|
1034 |
|
|
# HACK(nnorwitz): assume that all upper-case names
|
1035 |
|
|
# are some macro we aren't expanding.
|
1036 |
|
|
modifiers |= FUNCTION_UNKNOWN_ANNOTATION
|
1037 |
|
|
else:
|
1038 |
|
|
self.HandleError('unexpected token', modifier_token)
|
1039 |
|
|
|
1040 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
1041 |
|
|
# Handle ctor initializers.
|
1042 |
|
|
if token.name == ':':
|
1043 |
|
|
# TODO(nnorwitz): anything else to handle for initializer list?
|
1044 |
|
|
while token.name != ';' and token.name != '{':
|
1045 |
|
|
token = self._GetNextToken()
|
1046 |
|
|
|
1047 |
|
|
# Handle pointer to functions that are really data but look
|
1048 |
|
|
# like method declarations.
|
1049 |
|
|
if token.name == '(':
|
1050 |
|
|
if parameters[0].name == '*':
|
1051 |
|
|
# name contains the return type.
|
1052 |
|
|
name = parameters.pop()
|
1053 |
|
|
# parameters contains the name of the data.
|
1054 |
|
|
modifiers = [p.name for p in parameters]
|
1055 |
|
|
# Already at the ( to open the parameter list.
|
1056 |
|
|
function_parameters = list(self._GetMatchingChar('(', ')'))
|
1057 |
|
|
del function_parameters[-1] # Remove trailing ')'.
|
1058 |
|
|
# TODO(nnorwitz): store the function_parameters.
|
1059 |
|
|
token = self._GetNextToken()
|
1060 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
1061 |
|
|
assert token.name == ';', token
|
1062 |
|
|
return self._CreateVariable(indices, name.name, indices.name,
|
1063 |
|
|
modifiers, '', None)
|
1064 |
|
|
# At this point, we got something like:
|
1065 |
|
|
# return_type (type::*name_)(params);
|
1066 |
|
|
# This is a data member called name_ that is a function pointer.
|
1067 |
|
|
# With this code: void (sq_type::*field_)(string&);
|
1068 |
|
|
# We get: name=void return_type=[] parameters=sq_type ... field_
|
1069 |
|
|
# TODO(nnorwitz): is return_type always empty?
|
1070 |
|
|
# TODO(nnorwitz): this isn't even close to being correct.
|
1071 |
|
|
# Just put in something so we don't crash and can move on.
|
1072 |
|
|
real_name = parameters[-1]
|
1073 |
|
|
modifiers = [p.name for p in self._GetParameters()]
|
1074 |
|
|
del modifiers[-1] # Remove trailing ')'.
|
1075 |
|
|
return self._CreateVariable(indices, real_name.name, indices.name,
|
1076 |
|
|
modifiers, '', None)
|
1077 |
|
|
|
1078 |
|
|
if token.name == '{':
|
1079 |
|
|
body = list(self.GetScope())
|
1080 |
|
|
del body[-1] # Remove trailing '}'.
|
1081 |
|
|
else:
|
1082 |
|
|
body = None
|
1083 |
|
|
if token.name == '=':
|
1084 |
|
|
token = self._GetNextToken()
|
1085 |
|
|
|
1086 |
|
|
if token.name == 'default' or token.name == 'delete':
|
1087 |
|
|
# Ignore explicitly defaulted and deleted special members
|
1088 |
|
|
# in C++11.
|
1089 |
|
|
token = self._GetNextToken()
|
1090 |
|
|
else:
|
1091 |
|
|
# Handle pure-virtual declarations.
|
1092 |
|
|
assert token.token_type == tokenize.CONSTANT, token
|
1093 |
|
|
assert token.name == '0', token
|
1094 |
|
|
modifiers |= FUNCTION_PURE_VIRTUAL
|
1095 |
|
|
token = self._GetNextToken()
|
1096 |
|
|
|
1097 |
|
|
if token.name == '[':
|
1098 |
|
|
# TODO(nnorwitz): store tokens and improve parsing.
|
1099 |
|
|
# template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N];
|
1100 |
|
|
tokens = list(self._GetMatchingChar('[', ']'))
|
1101 |
|
|
token = self._GetNextToken()
|
1102 |
|
|
|
1103 |
|
|
assert token.name == ';', (token, return_type_and_name, parameters)
|
1104 |
|
|
|
1105 |
|
|
# Looks like we got a method, not a function.
|
1106 |
|
|
if len(return_type) > 2 and return_type[-1].name == '::':
|
1107 |
|
|
return_type, in_class = \
|
1108 |
|
|
self._GetReturnTypeAndClassName(return_type)
|
1109 |
|
|
return Method(indices.start, indices.end, name.name, in_class,
|
1110 |
|
|
return_type, parameters, modifiers, templated_types,
|
1111 |
|
|
body, self.namespace_stack)
|
1112 |
|
|
return Function(indices.start, indices.end, name.name, return_type,
|
1113 |
|
|
parameters, modifiers, templated_types, body,
|
1114 |
|
|
self.namespace_stack)
|
1115 |
|
|
|
1116 |
|
|
def _GetReturnTypeAndClassName(self, token_seq):
|
1117 |
|
|
# Splitting the return type from the class name in a method
|
1118 |
|
|
# can be tricky. For example, Return::Type::Is::Hard::To::Find().
|
1119 |
|
|
# Where is the return type and where is the class name?
|
1120 |
|
|
# The heuristic used is to pull the last name as the class name.
|
1121 |
|
|
# This includes all the templated type info.
|
1122 |
|
|
# TODO(nnorwitz): if there is only One name like in the
|
1123 |
|
|
# example above, punt and assume the last bit is the class name.
|
1124 |
|
|
|
1125 |
|
|
# Ignore a :: prefix, if exists so we can find the first real name.
|
1126 |
|
|
i = 0
|
1127 |
|
|
if token_seq[0].name == '::':
|
1128 |
|
|
i = 1
|
1129 |
|
|
# Ignore a :: suffix, if exists.
|
1130 |
|
|
end = len(token_seq) - 1
|
1131 |
|
|
if token_seq[end-1].name == '::':
|
1132 |
|
|
end -= 1
|
1133 |
|
|
|
1134 |
|
|
# Make a copy of the sequence so we can append a sentinel
|
1135 |
|
|
# value. This is required for GetName will has to have some
|
1136 |
|
|
# terminating condition beyond the last name.
|
1137 |
|
|
seq_copy = token_seq[i:end]
|
1138 |
|
|
seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
|
1139 |
|
|
names = []
|
1140 |
|
|
while i < end:
|
1141 |
|
|
# Iterate through the sequence parsing out each name.
|
1142 |
|
|
new_name, next = self.GetName(seq_copy[i:])
|
1143 |
|
|
assert new_name, 'Got empty new_name, next=%s' % next
|
1144 |
|
|
# We got a pointer or ref. Add it to the name.
|
1145 |
|
|
if next and next.token_type == tokenize.SYNTAX:
|
1146 |
|
|
new_name.append(next)
|
1147 |
|
|
names.append(new_name)
|
1148 |
|
|
i += len(new_name)
|
1149 |
|
|
|
1150 |
|
|
# Now that we have the names, it's time to undo what we did.
|
1151 |
|
|
|
1152 |
|
|
# Remove the sentinel value.
|
1153 |
|
|
names[-1].pop()
|
1154 |
|
|
# Flatten the token sequence for the return type.
|
1155 |
|
|
return_type = [e for seq in names[:-1] for e in seq]
|
1156 |
|
|
# The class name is the last name.
|
1157 |
|
|
class_name = names[-1]
|
1158 |
|
|
return return_type, class_name
|
1159 |
|
|
|
1160 |
|
|
def handle_bool(self):
|
1161 |
|
|
pass
|
1162 |
|
|
|
1163 |
|
|
def handle_char(self):
|
1164 |
|
|
pass
|
1165 |
|
|
|
1166 |
|
|
def handle_int(self):
|
1167 |
|
|
pass
|
1168 |
|
|
|
1169 |
|
|
def handle_long(self):
|
1170 |
|
|
pass
|
1171 |
|
|
|
1172 |
|
|
def handle_short(self):
|
1173 |
|
|
pass
|
1174 |
|
|
|
1175 |
|
|
def handle_double(self):
|
1176 |
|
|
pass
|
1177 |
|
|
|
1178 |
|
|
def handle_float(self):
|
1179 |
|
|
pass
|
1180 |
|
|
|
1181 |
|
|
def handle_void(self):
|
1182 |
|
|
pass
|
1183 |
|
|
|
1184 |
|
|
def handle_wchar_t(self):
|
1185 |
|
|
pass
|
1186 |
|
|
|
1187 |
|
|
def handle_unsigned(self):
|
1188 |
|
|
pass
|
1189 |
|
|
|
1190 |
|
|
def handle_signed(self):
|
1191 |
|
|
pass
|
1192 |
|
|
|
1193 |
|
|
def _GetNestedType(self, ctor):
|
1194 |
|
|
name = None
|
1195 |
|
|
name_tokens, token = self.GetName()
|
1196 |
|
|
if name_tokens:
|
1197 |
|
|
name = ''.join([t.name for t in name_tokens])
|
1198 |
|
|
|
1199 |
|
|
# Handle forward declarations.
|
1200 |
|
|
if token.token_type == tokenize.SYNTAX and token.name == ';':
|
1201 |
|
|
return ctor(token.start, token.end, name, None,
|
1202 |
|
|
self.namespace_stack)
|
1203 |
|
|
|
1204 |
|
|
if token.token_type == tokenize.NAME and self._handling_typedef:
|
1205 |
|
|
self._AddBackToken(token)
|
1206 |
|
|
return ctor(token.start, token.end, name, None,
|
1207 |
|
|
self.namespace_stack)
|
1208 |
|
|
|
1209 |
|
|
# Must be the type declaration.
|
1210 |
|
|
fields = list(self._GetMatchingChar('{', '}'))
|
1211 |
|
|
del fields[-1] # Remove trailing '}'.
|
1212 |
|
|
if token.token_type == tokenize.SYNTAX and token.name == '{':
|
1213 |
|
|
next = self._GetNextToken()
|
1214 |
|
|
new_type = ctor(token.start, token.end, name, fields,
|
1215 |
|
|
self.namespace_stack)
|
1216 |
|
|
# A name means this is an anonymous type and the name
|
1217 |
|
|
# is the variable declaration.
|
1218 |
|
|
if next.token_type != tokenize.NAME:
|
1219 |
|
|
return new_type
|
1220 |
|
|
name = new_type
|
1221 |
|
|
token = next
|
1222 |
|
|
|
1223 |
|
|
# Must be variable declaration using the type prefixed with keyword.
|
1224 |
|
|
assert token.token_type == tokenize.NAME, token
|
1225 |
|
|
return self._CreateVariable(token, token.name, name, [], '', None)
|
1226 |
|
|
|
1227 |
|
|
def handle_struct(self):
|
1228 |
|
|
# Special case the handling typedef/aliasing of structs here.
|
1229 |
|
|
# It would be a pain to handle in the class code.
|
1230 |
|
|
name_tokens, var_token = self.GetName()
|
1231 |
|
|
if name_tokens:
|
1232 |
|
|
next_token = self._GetNextToken()
|
1233 |
|
|
is_syntax = (var_token.token_type == tokenize.SYNTAX and
|
1234 |
|
|
var_token.name[0] in '*&')
|
1235 |
|
|
is_variable = (var_token.token_type == tokenize.NAME and
|
1236 |
|
|
next_token.name == ';')
|
1237 |
|
|
variable = var_token
|
1238 |
|
|
if is_syntax and not is_variable:
|
1239 |
|
|
variable = next_token
|
1240 |
|
|
temp = self._GetNextToken()
|
1241 |
|
|
if temp.token_type == tokenize.SYNTAX and temp.name == '(':
|
1242 |
|
|
# Handle methods declared to return a struct.
|
1243 |
|
|
t0 = name_tokens[0]
|
1244 |
|
|
struct = tokenize.Token(tokenize.NAME, 'struct',
|
1245 |
|
|
t0.start-7, t0.start-2)
|
1246 |
|
|
type_and_name = [struct]
|
1247 |
|
|
type_and_name.extend(name_tokens)
|
1248 |
|
|
type_and_name.extend((var_token, next_token))
|
1249 |
|
|
return self._GetMethod(type_and_name, 0, None, False)
|
1250 |
|
|
assert temp.name == ';', (temp, name_tokens, var_token)
|
1251 |
|
|
if is_syntax or (is_variable and not self._handling_typedef):
|
1252 |
|
|
modifiers = ['struct']
|
1253 |
|
|
type_name = ''.join([t.name for t in name_tokens])
|
1254 |
|
|
position = name_tokens[0]
|
1255 |
|
|
return self._CreateVariable(position, variable.name, type_name,
|
1256 |
|
|
modifiers, var_token.name, None)
|
1257 |
|
|
name_tokens.extend((var_token, next_token))
|
1258 |
|
|
self._AddBackTokens(name_tokens)
|
1259 |
|
|
else:
|
1260 |
|
|
self._AddBackToken(var_token)
|
1261 |
|
|
return self._GetClass(Struct, VISIBILITY_PUBLIC, None)
|
1262 |
|
|
|
1263 |
|
|
def handle_union(self):
|
1264 |
|
|
return self._GetNestedType(Union)
|
1265 |
|
|
|
1266 |
|
|
def handle_enum(self):
|
1267 |
|
|
return self._GetNestedType(Enum)
|
1268 |
|
|
|
1269 |
|
|
def handle_auto(self):
|
1270 |
|
|
# TODO(nnorwitz): warn about using auto? Probably not since it
|
1271 |
|
|
# will be reclaimed and useful for C++0x.
|
1272 |
|
|
pass
|
1273 |
|
|
|
1274 |
|
|
def handle_register(self):
|
1275 |
|
|
pass
|
1276 |
|
|
|
1277 |
|
|
def handle_const(self):
|
1278 |
|
|
pass
|
1279 |
|
|
|
1280 |
|
|
def handle_inline(self):
|
1281 |
|
|
pass
|
1282 |
|
|
|
1283 |
|
|
def handle_extern(self):
|
1284 |
|
|
pass
|
1285 |
|
|
|
1286 |
|
|
def handle_static(self):
|
1287 |
|
|
pass
|
1288 |
|
|
|
1289 |
|
|
def handle_virtual(self):
|
1290 |
|
|
# What follows must be a method.
|
1291 |
|
|
token = token2 = self._GetNextToken()
|
1292 |
|
|
if token.name == 'inline':
|
1293 |
|
|
# HACK(nnorwitz): handle inline dtors by ignoring 'inline'.
|
1294 |
|
|
token2 = self._GetNextToken()
|
1295 |
|
|
if token2.token_type == tokenize.SYNTAX and token2.name == '~':
|
1296 |
|
|
return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None)
|
1297 |
|
|
assert token.token_type == tokenize.NAME or token.name == '::', token
|
1298 |
|
|
return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') # )
|
1299 |
|
|
return_type_and_name.insert(0, token)
|
1300 |
|
|
if token2 is not token:
|
1301 |
|
|
return_type_and_name.insert(1, token2)
|
1302 |
|
|
return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL,
|
1303 |
|
|
None, False)
|
1304 |
|
|
|
1305 |
|
|
def handle_volatile(self):
|
1306 |
|
|
pass
|
1307 |
|
|
|
1308 |
|
|
def handle_mutable(self):
|
1309 |
|
|
pass
|
1310 |
|
|
|
1311 |
|
|
def handle_public(self):
|
1312 |
|
|
assert self.in_class
|
1313 |
|
|
self.visibility = VISIBILITY_PUBLIC
|
1314 |
|
|
|
1315 |
|
|
def handle_protected(self):
|
1316 |
|
|
assert self.in_class
|
1317 |
|
|
self.visibility = VISIBILITY_PROTECTED
|
1318 |
|
|
|
1319 |
|
|
def handle_private(self):
|
1320 |
|
|
assert self.in_class
|
1321 |
|
|
self.visibility = VISIBILITY_PRIVATE
|
1322 |
|
|
|
1323 |
|
|
def handle_friend(self):
|
1324 |
|
|
tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
|
1325 |
|
|
assert tokens
|
1326 |
|
|
t0 = tokens[0]
|
1327 |
|
|
return Friend(t0.start, t0.end, tokens, self.namespace_stack)
|
1328 |
|
|
|
1329 |
|
|
def handle_static_cast(self):
|
1330 |
|
|
pass
|
1331 |
|
|
|
1332 |
|
|
def handle_const_cast(self):
|
1333 |
|
|
pass
|
1334 |
|
|
|
1335 |
|
|
def handle_dynamic_cast(self):
|
1336 |
|
|
pass
|
1337 |
|
|
|
1338 |
|
|
def handle_reinterpret_cast(self):
|
1339 |
|
|
pass
|
1340 |
|
|
|
1341 |
|
|
def handle_new(self):
|
1342 |
|
|
pass
|
1343 |
|
|
|
1344 |
|
|
def handle_delete(self):
|
1345 |
|
|
tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
|
1346 |
|
|
assert tokens
|
1347 |
|
|
return Delete(tokens[0].start, tokens[0].end, tokens)
|
1348 |
|
|
|
1349 |
|
|
def handle_typedef(self):
|
1350 |
|
|
token = self._GetNextToken()
|
1351 |
|
|
if (token.token_type == tokenize.NAME and
|
1352 |
|
|
keywords.IsKeyword(token.name)):
|
1353 |
|
|
# Token must be struct/enum/union/class.
|
1354 |
|
|
method = getattr(self, 'handle_' + token.name)
|
1355 |
|
|
self._handling_typedef = True
|
1356 |
|
|
tokens = [method()]
|
1357 |
|
|
self._handling_typedef = False
|
1358 |
|
|
else:
|
1359 |
|
|
tokens = [token]
|
1360 |
|
|
|
1361 |
|
|
# Get the remainder of the typedef up to the semi-colon.
|
1362 |
|
|
tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))
|
1363 |
|
|
|
1364 |
|
|
# TODO(nnorwitz): clean all this up.
|
1365 |
|
|
assert tokens
|
1366 |
|
|
name = tokens.pop()
|
1367 |
|
|
indices = name
|
1368 |
|
|
if tokens:
|
1369 |
|
|
indices = tokens[0]
|
1370 |
|
|
if not indices:
|
1371 |
|
|
indices = token
|
1372 |
|
|
if name.name == ')':
|
1373 |
|
|
# HACK(nnorwitz): Handle pointers to functions "properly".
|
1374 |
|
|
if (len(tokens) >= 4 and
|
1375 |
|
|
tokens[1].name == '(' and tokens[2].name == '*'):
|
1376 |
|
|
tokens.append(name)
|
1377 |
|
|
name = tokens[3]
|
1378 |
|
|
elif name.name == ']':
|
1379 |
|
|
# HACK(nnorwitz): Handle arrays properly.
|
1380 |
|
|
if len(tokens) >= 2:
|
1381 |
|
|
tokens.append(name)
|
1382 |
|
|
name = tokens[1]
|
1383 |
|
|
new_type = tokens
|
1384 |
|
|
if tokens and isinstance(tokens[0], tokenize.Token):
|
1385 |
|
|
new_type = self.converter.ToType(tokens)[0]
|
1386 |
|
|
return Typedef(indices.start, indices.end, name.name,
|
1387 |
|
|
new_type, self.namespace_stack)
|
1388 |
|
|
|
1389 |
|
|
def handle_typeid(self):
|
1390 |
|
|
pass # Not needed yet.
|
1391 |
|
|
|
1392 |
|
|
def handle_typename(self):
|
1393 |
|
|
pass # Not needed yet.
|
1394 |
|
|
|
1395 |
|
|
def _GetTemplatedTypes(self):
|
1396 |
|
|
result = {}
|
1397 |
|
|
tokens = list(self._GetMatchingChar('<', '>'))
|
1398 |
|
|
len_tokens = len(tokens) - 1 # Ignore trailing '>'.
|
1399 |
|
|
i = 0
|
1400 |
|
|
while i < len_tokens:
|
1401 |
|
|
key = tokens[i].name
|
1402 |
|
|
i += 1
|
1403 |
|
|
if keywords.IsKeyword(key) or key == ',':
|
1404 |
|
|
continue
|
1405 |
|
|
type_name = default = None
|
1406 |
|
|
if i < len_tokens:
|
1407 |
|
|
i += 1
|
1408 |
|
|
if tokens[i-1].name == '=':
|
1409 |
|
|
assert i < len_tokens, '%s %s' % (i, tokens)
|
1410 |
|
|
default, unused_next_token = self.GetName(tokens[i:])
|
1411 |
|
|
i += len(default)
|
1412 |
|
|
else:
|
1413 |
|
|
if tokens[i-1].name != ',':
|
1414 |
|
|
# We got something like: Type variable.
|
1415 |
|
|
# Re-adjust the key (variable) and type_name (Type).
|
1416 |
|
|
key = tokens[i-1].name
|
1417 |
|
|
type_name = tokens[i-2]
|
1418 |
|
|
|
1419 |
|
|
result[key] = (type_name, default)
|
1420 |
|
|
return result
|
1421 |
|
|
|
1422 |
|
|
def handle_template(self):
|
1423 |
|
|
token = self._GetNextToken()
|
1424 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
1425 |
|
|
assert token.name == '<', token
|
1426 |
|
|
templated_types = self._GetTemplatedTypes()
|
1427 |
|
|
# TODO(nnorwitz): for now, just ignore the template params.
|
1428 |
|
|
token = self._GetNextToken()
|
1429 |
|
|
if token.token_type == tokenize.NAME:
|
1430 |
|
|
if token.name == 'class':
|
1431 |
|
|
return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types)
|
1432 |
|
|
elif token.name == 'struct':
|
1433 |
|
|
return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types)
|
1434 |
|
|
elif token.name == 'friend':
|
1435 |
|
|
return self.handle_friend()
|
1436 |
|
|
self._AddBackToken(token)
|
1437 |
|
|
tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
|
1438 |
|
|
tokens.append(last)
|
1439 |
|
|
self._AddBackTokens(tokens)
|
1440 |
|
|
if last.name == '(':
|
1441 |
|
|
return self.GetMethod(FUNCTION_NONE, templated_types)
|
1442 |
|
|
# Must be a variable definition.
|
1443 |
|
|
return None
|
1444 |
|
|
|
1445 |
|
|
def handle_true(self):
|
1446 |
|
|
pass # Nothing to do.
|
1447 |
|
|
|
1448 |
|
|
def handle_false(self):
|
1449 |
|
|
pass # Nothing to do.
|
1450 |
|
|
|
1451 |
|
|
def handle_asm(self):
|
1452 |
|
|
pass # Not needed yet.
|
1453 |
|
|
|
1454 |
|
|
def handle_class(self):
|
1455 |
|
|
return self._GetClass(Class, VISIBILITY_PRIVATE, None)
|
1456 |
|
|
|
1457 |
|
|
def _GetBases(self):
|
1458 |
|
|
# Get base classes.
|
1459 |
|
|
bases = []
|
1460 |
|
|
while 1:
|
1461 |
|
|
token = self._GetNextToken()
|
1462 |
|
|
assert token.token_type == tokenize.NAME, token
|
1463 |
|
|
# TODO(nnorwitz): store kind of inheritance...maybe.
|
1464 |
|
|
if token.name not in ('public', 'protected', 'private'):
|
1465 |
|
|
# If inheritance type is not specified, it is private.
|
1466 |
|
|
# Just put the token back so we can form a name.
|
1467 |
|
|
# TODO(nnorwitz): it would be good to warn about this.
|
1468 |
|
|
self._AddBackToken(token)
|
1469 |
|
|
else:
|
1470 |
|
|
# Check for virtual inheritance.
|
1471 |
|
|
token = self._GetNextToken()
|
1472 |
|
|
if token.name != 'virtual':
|
1473 |
|
|
self._AddBackToken(token)
|
1474 |
|
|
else:
|
1475 |
|
|
# TODO(nnorwitz): store that we got virtual for this base.
|
1476 |
|
|
pass
|
1477 |
|
|
base, next_token = self.GetName()
|
1478 |
|
|
bases_ast = self.converter.ToType(base)
|
1479 |
|
|
assert len(bases_ast) == 1, bases_ast
|
1480 |
|
|
bases.append(bases_ast[0])
|
1481 |
|
|
assert next_token.token_type == tokenize.SYNTAX, next_token
|
1482 |
|
|
if next_token.name == '{':
|
1483 |
|
|
token = next_token
|
1484 |
|
|
break
|
1485 |
|
|
# Support multiple inheritance.
|
1486 |
|
|
assert next_token.name == ',', next_token
|
1487 |
|
|
return bases, token
|
1488 |
|
|
|
1489 |
|
|
def _GetClass(self, class_type, visibility, templated_types):
|
1490 |
|
|
class_name = None
|
1491 |
|
|
class_token = self._GetNextToken()
|
1492 |
|
|
if class_token.token_type != tokenize.NAME:
|
1493 |
|
|
assert class_token.token_type == tokenize.SYNTAX, class_token
|
1494 |
|
|
token = class_token
|
1495 |
|
|
else:
|
1496 |
|
|
# Skip any macro (e.g. storage class specifiers) after the
|
1497 |
|
|
# 'class' keyword.
|
1498 |
|
|
next_token = self._GetNextToken()
|
1499 |
|
|
if next_token.token_type == tokenize.NAME:
|
1500 |
|
|
self._AddBackToken(next_token)
|
1501 |
|
|
else:
|
1502 |
|
|
self._AddBackTokens([class_token, next_token])
|
1503 |
|
|
name_tokens, token = self.GetName()
|
1504 |
|
|
class_name = ''.join([t.name for t in name_tokens])
|
1505 |
|
|
bases = None
|
1506 |
|
|
if token.token_type == tokenize.SYNTAX:
|
1507 |
|
|
if token.name == ';':
|
1508 |
|
|
# Forward declaration.
|
1509 |
|
|
return class_type(class_token.start, class_token.end,
|
1510 |
|
|
class_name, None, templated_types, None,
|
1511 |
|
|
self.namespace_stack)
|
1512 |
|
|
if token.name in '*&':
|
1513 |
|
|
# Inline forward declaration. Could be method or data.
|
1514 |
|
|
name_token = self._GetNextToken()
|
1515 |
|
|
next_token = self._GetNextToken()
|
1516 |
|
|
if next_token.name == ';':
|
1517 |
|
|
# Handle data
|
1518 |
|
|
modifiers = ['class']
|
1519 |
|
|
return self._CreateVariable(class_token, name_token.name,
|
1520 |
|
|
class_name,
|
1521 |
|
|
modifiers, token.name, None)
|
1522 |
|
|
else:
|
1523 |
|
|
# Assume this is a method.
|
1524 |
|
|
tokens = (class_token, token, name_token, next_token)
|
1525 |
|
|
self._AddBackTokens(tokens)
|
1526 |
|
|
return self.GetMethod(FUNCTION_NONE, None)
|
1527 |
|
|
if token.name == ':':
|
1528 |
|
|
bases, token = self._GetBases()
|
1529 |
|
|
|
1530 |
|
|
body = None
|
1531 |
|
|
if token.token_type == tokenize.SYNTAX and token.name == '{':
|
1532 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
1533 |
|
|
assert token.name == '{', token
|
1534 |
|
|
|
1535 |
|
|
ast = AstBuilder(self.GetScope(), self.filename, class_name,
|
1536 |
|
|
visibility, self.namespace_stack)
|
1537 |
|
|
body = list(ast.Generate())
|
1538 |
|
|
|
1539 |
|
|
if not self._handling_typedef:
|
1540 |
|
|
token = self._GetNextToken()
|
1541 |
|
|
if token.token_type != tokenize.NAME:
|
1542 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
1543 |
|
|
assert token.name == ';', token
|
1544 |
|
|
else:
|
1545 |
|
|
new_class = class_type(class_token.start, class_token.end,
|
1546 |
|
|
class_name, bases, None,
|
1547 |
|
|
body, self.namespace_stack)
|
1548 |
|
|
|
1549 |
|
|
modifiers = []
|
1550 |
|
|
return self._CreateVariable(class_token,
|
1551 |
|
|
token.name, new_class,
|
1552 |
|
|
modifiers, token.name, None)
|
1553 |
|
|
else:
|
1554 |
|
|
if not self._handling_typedef:
|
1555 |
|
|
self.HandleError('non-typedef token', token)
|
1556 |
|
|
self._AddBackToken(token)
|
1557 |
|
|
|
1558 |
|
|
return class_type(class_token.start, class_token.end, class_name,
|
1559 |
|
|
bases, templated_types, body, self.namespace_stack)
|
1560 |
|
|
|
1561 |
|
|
def handle_namespace(self):
|
1562 |
|
|
token = self._GetNextToken()
|
1563 |
|
|
# Support anonymous namespaces.
|
1564 |
|
|
name = None
|
1565 |
|
|
if token.token_type == tokenize.NAME:
|
1566 |
|
|
name = token.name
|
1567 |
|
|
token = self._GetNextToken()
|
1568 |
|
|
self.namespace_stack.append(name)
|
1569 |
|
|
assert token.token_type == tokenize.SYNTAX, token
|
1570 |
|
|
# Create an internal token that denotes when the namespace is complete.
|
1571 |
|
|
internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
|
1572 |
|
|
None, None)
|
1573 |
|
|
internal_token.whence = token.whence
|
1574 |
|
|
if token.name == '=':
|
1575 |
|
|
# TODO(nnorwitz): handle aliasing namespaces.
|
1576 |
|
|
name, next_token = self.GetName()
|
1577 |
|
|
assert next_token.name == ';', next_token
|
1578 |
|
|
self._AddBackToken(internal_token)
|
1579 |
|
|
else:
|
1580 |
|
|
assert token.name == '{', token
|
1581 |
|
|
tokens = list(self.GetScope())
|
1582 |
|
|
# Replace the trailing } with the internal namespace pop token.
|
1583 |
|
|
tokens[-1] = internal_token
|
1584 |
|
|
# Handle namespace with nothing in it.
|
1585 |
|
|
self._AddBackTokens(tokens)
|
1586 |
|
|
return None
|
1587 |
|
|
|
1588 |
|
|
def handle_using(self):
|
1589 |
|
|
tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
|
1590 |
|
|
assert tokens
|
1591 |
|
|
return Using(tokens[0].start, tokens[0].end, tokens)
|
1592 |
|
|
|
1593 |
|
|
def handle_explicit(self):
|
1594 |
|
|
assert self.in_class
|
1595 |
|
|
# Nothing much to do.
|
1596 |
|
|
# TODO(nnorwitz): maybe verify the method name == class name.
|
1597 |
|
|
# This must be a ctor.
|
1598 |
|
|
return self.GetMethod(FUNCTION_CTOR, None)
|
1599 |
|
|
|
1600 |
|
|
def handle_this(self):
|
1601 |
|
|
pass # Nothing to do.
|
1602 |
|
|
|
1603 |
|
|
def handle_operator(self):
|
1604 |
|
|
# Pull off the next token(s?) and make that part of the method name.
|
1605 |
|
|
pass
|
1606 |
|
|
|
1607 |
|
|
def handle_sizeof(self):
|
1608 |
|
|
pass
|
1609 |
|
|
|
1610 |
|
|
def handle_case(self):
|
1611 |
|
|
pass
|
1612 |
|
|
|
1613 |
|
|
def handle_switch(self):
|
1614 |
|
|
pass
|
1615 |
|
|
|
1616 |
|
|
def handle_default(self):
|
1617 |
|
|
token = self._GetNextToken()
|
1618 |
|
|
assert token.token_type == tokenize.SYNTAX
|
1619 |
|
|
assert token.name == ':'
|
1620 |
|
|
|
1621 |
|
|
def handle_if(self):
|
1622 |
|
|
pass
|
1623 |
|
|
|
1624 |
|
|
def handle_else(self):
|
1625 |
|
|
pass
|
1626 |
|
|
|
1627 |
|
|
def handle_return(self):
|
1628 |
|
|
tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
|
1629 |
|
|
if not tokens:
|
1630 |
|
|
return Return(self.current_token.start, self.current_token.end, None)
|
1631 |
|
|
return Return(tokens[0].start, tokens[0].end, tokens)
|
1632 |
|
|
|
1633 |
|
|
def handle_goto(self):
|
1634 |
|
|
tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
|
1635 |
|
|
assert len(tokens) == 1, str(tokens)
|
1636 |
|
|
return Goto(tokens[0].start, tokens[0].end, tokens[0].name)
|
1637 |
|
|
|
1638 |
|
|
def handle_try(self):
|
1639 |
|
|
pass # Not needed yet.
|
1640 |
|
|
|
1641 |
|
|
def handle_catch(self):
|
1642 |
|
|
pass # Not needed yet.
|
1643 |
|
|
|
1644 |
|
|
def handle_throw(self):
|
1645 |
|
|
pass # Not needed yet.
|
1646 |
|
|
|
1647 |
|
|
def handle_while(self):
|
1648 |
|
|
pass
|
1649 |
|
|
|
1650 |
|
|
def handle_do(self):
|
1651 |
|
|
pass
|
1652 |
|
|
|
1653 |
|
|
def handle_for(self):
|
1654 |
|
|
pass
|
1655 |
|
|
|
1656 |
|
|
def handle_break(self):
|
1657 |
|
|
self._IgnoreUpTo(tokenize.SYNTAX, ';')
|
1658 |
|
|
|
1659 |
|
|
def handle_continue(self):
|
1660 |
|
|
self._IgnoreUpTo(tokenize.SYNTAX, ';')
|
1661 |
|
|
|
1662 |
|
|
|
1663 |
|
|
def BuilderFromSource(source, filename):
|
1664 |
|
|
"""Utility method that returns an AstBuilder from source code.
|
1665 |
|
|
|
1666 |
|
|
Args:
|
1667 |
|
|
source: 'C++ source code'
|
1668 |
|
|
filename: 'file1'
|
1669 |
|
|
|
1670 |
|
|
Returns:
|
1671 |
|
|
AstBuilder
|
1672 |
|
|
"""
|
1673 |
|
|
return AstBuilder(tokenize.GetTokens(source), filename)
|
1674 |
|
|
|
1675 |
|
|
|
1676 |
|
|
def PrintIndentifiers(filename, should_print):
|
1677 |
|
|
"""Prints all identifiers for a C++ source file.
|
1678 |
|
|
|
1679 |
|
|
Args:
|
1680 |
|
|
filename: 'file1'
|
1681 |
|
|
should_print: predicate with signature: bool Function(token)
|
1682 |
|
|
"""
|
1683 |
|
|
source = utils.ReadFile(filename, False)
|
1684 |
|
|
if source is None:
|
1685 |
|
|
sys.stderr.write('Unable to find: %s\n' % filename)
|
1686 |
|
|
return
|
1687 |
|
|
|
1688 |
|
|
#print('Processing %s' % actual_filename)
|
1689 |
|
|
builder = BuilderFromSource(source, filename)
|
1690 |
|
|
try:
|
1691 |
|
|
for node in builder.Generate():
|
1692 |
|
|
if should_print(node):
|
1693 |
|
|
print(node.name)
|
1694 |
|
|
except KeyboardInterrupt:
|
1695 |
|
|
return
|
1696 |
|
|
except:
|
1697 |
|
|
pass
|
1698 |
|
|
|
1699 |
|
|
|
1700 |
|
|
def PrintAllIndentifiers(filenames, should_print):
|
1701 |
|
|
"""Prints all identifiers for each C++ source file in filenames.
|
1702 |
|
|
|
1703 |
|
|
Args:
|
1704 |
|
|
filenames: ['file1', 'file2', ...]
|
1705 |
|
|
should_print: predicate with signature: bool Function(token)
|
1706 |
|
|
"""
|
1707 |
|
|
for path in filenames:
|
1708 |
|
|
PrintIndentifiers(path, should_print)
|
1709 |
|
|
|
1710 |
|
|
|
1711 |
|
|
def main(argv):
|
1712 |
|
|
for filename in argv[1:]:
|
1713 |
|
|
source = utils.ReadFile(filename)
|
1714 |
|
|
if source is None:
|
1715 |
|
|
continue
|
1716 |
|
|
|
1717 |
|
|
print('Processing %s' % filename)
|
1718 |
|
|
builder = BuilderFromSource(source, filename)
|
1719 |
|
|
try:
|
1720 |
|
|
entire_ast = filter(None, builder.Generate())
|
1721 |
|
|
except KeyboardInterrupt:
|
1722 |
|
|
return
|
1723 |
|
|
except:
|
1724 |
|
|
# Already printed a warning, print the traceback and continue.
|
1725 |
|
|
traceback.print_exc()
|
1726 |
|
|
else:
|
1727 |
|
|
if utils.DEBUG:
|
1728 |
|
|
for ast in entire_ast:
|
1729 |
|
|
print(ast)
|
1730 |
|
|
|
1731 |
|
|
|
1732 |
|
|
if __name__ == '__main__':
|
1733 |
|
|
main(sys.argv)
|