1 #!/usr/bin/env python 2 # 3 # Copyright 2007 Neal Norwitz 4 # Portions Copyright 2007 Google Inc. 5 # 6 # Licensed under the Apache License, Version 2.0 (the "License"); 7 # you may not use this file except in compliance with the License. 8 # You may obtain a copy of the License at 9 # 10 # http://www.apache.org/licenses/LICENSE-2.0 11 # 12 # Unless required by applicable law or agreed to in writing, software 13 # distributed under the License is distributed on an "AS IS" BASIS, 14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 # See the License for the specific language governing permissions and 16 # limitations under the License. 17 18 """Generate an Abstract Syntax Tree (AST) for C++.""" 19 20 __author__ = 'nnorwitz (at] google.com (Neal Norwitz)' 21 22 23 # TODO: 24 # * Tokens should never be exported, need to convert to Nodes 25 # (return types, parameters, etc.) 26 # * Handle static class data for templatized classes 27 # * Handle casts (both C++ and C-style) 28 # * Handle conditions and loops (if/else, switch, for, while/do) 29 # 30 # TODO much, much later: 31 # * Handle #define 32 # * exceptions 33 34 35 try: 36 # Python 3.x 37 import builtins 38 except ImportError: 39 # Python 2.x 40 import __builtin__ as builtins 41 42 import sys 43 import traceback 44 45 from cpp import keywords 46 from cpp import tokenize 47 from cpp import utils 48 49 50 if not hasattr(builtins, 'reversed'): 51 # Support Python 2.3 and earlier. 52 def reversed(seq): 53 for i in range(len(seq)-1, -1, -1): 54 yield seq[i] 55 56 if not hasattr(builtins, 'next'): 57 # Support Python 2.5 and earlier. 58 def next(obj): 59 return obj.next() 60 61 62 VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3) 63 64 FUNCTION_NONE = 0x00 65 FUNCTION_CONST = 0x01 66 FUNCTION_VIRTUAL = 0x02 67 FUNCTION_PURE_VIRTUAL = 0x04 68 FUNCTION_CTOR = 0x08 69 FUNCTION_DTOR = 0x10 70 FUNCTION_ATTRIBUTE = 0x20 71 FUNCTION_UNKNOWN_ANNOTATION = 0x40 72 FUNCTION_THROW = 0x80 73 FUNCTION_OVERRIDE = 0x100 74 75 """ 76 These are currently unused. Should really handle these properly at some point. 77 78 TYPE_MODIFIER_INLINE = 0x010000 79 TYPE_MODIFIER_EXTERN = 0x020000 80 TYPE_MODIFIER_STATIC = 0x040000 81 TYPE_MODIFIER_CONST = 0x080000 82 TYPE_MODIFIER_REGISTER = 0x100000 83 TYPE_MODIFIER_VOLATILE = 0x200000 84 TYPE_MODIFIER_MUTABLE = 0x400000 85 86 TYPE_MODIFIER_MAP = { 87 'inline': TYPE_MODIFIER_INLINE, 88 'extern': TYPE_MODIFIER_EXTERN, 89 'static': TYPE_MODIFIER_STATIC, 90 'const': TYPE_MODIFIER_CONST, 91 'register': TYPE_MODIFIER_REGISTER, 92 'volatile': TYPE_MODIFIER_VOLATILE, 93 'mutable': TYPE_MODIFIER_MUTABLE, 94 } 95 """ 96 97 _INTERNAL_TOKEN = 'internal' 98 _NAMESPACE_POP = 'ns-pop' 99 100 101 # TODO(nnorwitz): use this as a singleton for templated_types, etc 102 # where we don't want to create a new empty dict each time. It is also const. 103 class _NullDict(object): 104 __contains__ = lambda self: False 105 keys = values = items = iterkeys = itervalues = iteritems = lambda self: () 106 107 108 # TODO(nnorwitz): move AST nodes into a separate module. 109 class Node(object): 110 """Base AST node.""" 111 112 def __init__(self, start, end): 113 self.start = start 114 self.end = end 115 116 def IsDeclaration(self): 117 """Returns bool if this node is a declaration.""" 118 return False 119 120 def IsDefinition(self): 121 """Returns bool if this node is a definition.""" 122 return False 123 124 def IsExportable(self): 125 """Returns bool if this node exportable from a header file.""" 126 return False 127 128 def Requires(self, node): 129 """Does this AST node require the definition of the node passed in?""" 130 return False 131 132 def XXX__str__(self): 133 return self._StringHelper(self.__class__.__name__, '') 134 135 def _StringHelper(self, name, suffix): 136 if not utils.DEBUG: 137 return '%s(%s)' % (name, suffix) 138 return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix) 139 140 def __repr__(self): 141 return str(self) 142 143 144 class Define(Node): 145 def __init__(self, start, end, name, definition): 146 Node.__init__(self, start, end) 147 self.name = name 148 self.definition = definition 149 150 def __str__(self): 151 value = '%s %s' % (self.name, self.definition) 152 return self._StringHelper(self.__class__.__name__, value) 153 154 155 class Include(Node): 156 def __init__(self, start, end, filename, system): 157 Node.__init__(self, start, end) 158 self.filename = filename 159 self.system = system 160 161 def __str__(self): 162 fmt = '"%s"' 163 if self.system: 164 fmt = '<%s>' 165 return self._StringHelper(self.__class__.__name__, fmt % self.filename) 166 167 168 class Goto(Node): 169 def __init__(self, start, end, label): 170 Node.__init__(self, start, end) 171 self.label = label 172 173 def __str__(self): 174 return self._StringHelper(self.__class__.__name__, str(self.label)) 175 176 177 class Expr(Node): 178 def __init__(self, start, end, expr): 179 Node.__init__(self, start, end) 180 self.expr = expr 181 182 def Requires(self, node): 183 # TODO(nnorwitz): impl. 184 return False 185 186 def __str__(self): 187 return self._StringHelper(self.__class__.__name__, str(self.expr)) 188 189 190 class Return(Expr): 191 pass 192 193 194 class Delete(Expr): 195 pass 196 197 198 class Friend(Expr): 199 def __init__(self, start, end, expr, namespace): 200 Expr.__init__(self, start, end, expr) 201 self.namespace = namespace[:] 202 203 204 class Using(Node): 205 def __init__(self, start, end, names): 206 Node.__init__(self, start, end) 207 self.names = names 208 209 def __str__(self): 210 return self._StringHelper(self.__class__.__name__, str(self.names)) 211 212 213 class Parameter(Node): 214 def __init__(self, start, end, name, parameter_type, default): 215 Node.__init__(self, start, end) 216 self.name = name 217 self.type = parameter_type 218 self.default = default 219 220 def Requires(self, node): 221 # TODO(nnorwitz): handle namespaces, etc. 222 return self.type.name == node.name 223 224 def __str__(self): 225 name = str(self.type) 226 suffix = '%s %s' % (name, self.name) 227 if self.default: 228 suffix += ' = ' + ''.join([d.name for d in self.default]) 229 return self._StringHelper(self.__class__.__name__, suffix) 230 231 232 class _GenericDeclaration(Node): 233 def __init__(self, start, end, name, namespace): 234 Node.__init__(self, start, end) 235 self.name = name 236 self.namespace = namespace[:] 237 238 def FullName(self): 239 prefix = '' 240 if self.namespace and self.namespace[-1]: 241 prefix = '::'.join(self.namespace) + '::' 242 return prefix + self.name 243 244 def _TypeStringHelper(self, suffix): 245 if self.namespace: 246 names = [n or '<anonymous>' for n in self.namespace] 247 suffix += ' in ' + '::'.join(names) 248 return self._StringHelper(self.__class__.__name__, suffix) 249 250 251 # TODO(nnorwitz): merge with Parameter in some way? 252 class VariableDeclaration(_GenericDeclaration): 253 def __init__(self, start, end, name, var_type, initial_value, namespace): 254 _GenericDeclaration.__init__(self, start, end, name, namespace) 255 self.type = var_type 256 self.initial_value = initial_value 257 258 def Requires(self, node): 259 # TODO(nnorwitz): handle namespaces, etc. 260 return self.type.name == node.name 261 262 def ToString(self): 263 """Return a string that tries to reconstitute the variable decl.""" 264 suffix = '%s %s' % (self.type, self.name) 265 if self.initial_value: 266 suffix += ' = ' + self.initial_value 267 return suffix 268 269 def __str__(self): 270 return self._StringHelper(self.__class__.__name__, self.ToString()) 271 272 273 class Typedef(_GenericDeclaration): 274 def __init__(self, start, end, name, alias, namespace): 275 _GenericDeclaration.__init__(self, start, end, name, namespace) 276 self.alias = alias 277 278 def IsDefinition(self): 279 return True 280 281 def IsExportable(self): 282 return True 283 284 def Requires(self, node): 285 # TODO(nnorwitz): handle namespaces, etc. 286 name = node.name 287 for token in self.alias: 288 if token is not None and name == token.name: 289 return True 290 return False 291 292 def __str__(self): 293 suffix = '%s, %s' % (self.name, self.alias) 294 return self._TypeStringHelper(suffix) 295 296 297 class _NestedType(_GenericDeclaration): 298 def __init__(self, start, end, name, fields, namespace): 299 _GenericDeclaration.__init__(self, start, end, name, namespace) 300 self.fields = fields 301 302 def IsDefinition(self): 303 return True 304 305 def IsExportable(self): 306 return True 307 308 def __str__(self): 309 suffix = '%s, {%s}' % (self.name, self.fields) 310 return self._TypeStringHelper(suffix) 311 312 313 class Union(_NestedType): 314 pass 315 316 317 class Enum(_NestedType): 318 pass 319 320 321 class Class(_GenericDeclaration): 322 def __init__(self, start, end, name, bases, templated_types, body, namespace): 323 _GenericDeclaration.__init__(self, start, end, name, namespace) 324 self.bases = bases 325 self.body = body 326 self.templated_types = templated_types 327 328 def IsDeclaration(self): 329 return self.bases is None and self.body is None 330 331 def IsDefinition(self): 332 return not self.IsDeclaration() 333 334 def IsExportable(self): 335 return not self.IsDeclaration() 336 337 def Requires(self, node): 338 # TODO(nnorwitz): handle namespaces, etc. 339 if self.bases: 340 for token_list in self.bases: 341 # TODO(nnorwitz): bases are tokens, do name comparision. 342 for token in token_list: 343 if token.name == node.name: 344 return True 345 # TODO(nnorwitz): search in body too. 346 return False 347 348 def __str__(self): 349 name = self.name 350 if self.templated_types: 351 name += '<%s>' % self.templated_types 352 suffix = '%s, %s, %s' % (name, self.bases, self.body) 353 return self._TypeStringHelper(suffix) 354 355 356 class Struct(Class): 357 pass 358 359 360 class Function(_GenericDeclaration): 361 def __init__(self, start, end, name, return_type, parameters, 362 modifiers, templated_types, body, namespace): 363 _GenericDeclaration.__init__(self, start, end, name, namespace) 364 converter = TypeConverter(namespace) 365 self.return_type = converter.CreateReturnType(return_type) 366 self.parameters = converter.ToParameters(parameters) 367 self.modifiers = modifiers 368 self.body = body 369 self.templated_types = templated_types 370 371 def IsDeclaration(self): 372 return self.body is None 373 374 def IsDefinition(self): 375 return self.body is not None 376 377 def IsExportable(self): 378 if self.return_type and 'static' in self.return_type.modifiers: 379 return False 380 return None not in self.namespace 381 382 def Requires(self, node): 383 if self.parameters: 384 # TODO(nnorwitz): parameters are tokens, do name comparision. 385 for p in self.parameters: 386 if p.name == node.name: 387 return True 388 # TODO(nnorwitz): search in body too. 389 return False 390 391 def __str__(self): 392 # TODO(nnorwitz): add templated_types. 393 suffix = ('%s %s(%s), 0x%02x, %s' % 394 (self.return_type, self.name, self.parameters, 395 self.modifiers, self.body)) 396 return self._TypeStringHelper(suffix) 397 398 399 class Method(Function): 400 def __init__(self, start, end, name, in_class, return_type, parameters, 401 modifiers, templated_types, body, namespace): 402 Function.__init__(self, start, end, name, return_type, parameters, 403 modifiers, templated_types, body, namespace) 404 # TODO(nnorwitz): in_class could also be a namespace which can 405 # mess up finding functions properly. 406 self.in_class = in_class 407 408 409 class Type(_GenericDeclaration): 410 """Type used for any variable (eg class, primitive, struct, etc).""" 411 412 def __init__(self, start, end, name, templated_types, modifiers, 413 reference, pointer, array): 414 """ 415 Args: 416 name: str name of main type 417 templated_types: [Class (Type?)] template type info between <> 418 modifiers: [str] type modifiers (keywords) eg, const, mutable, etc. 419 reference, pointer, array: bools 420 """ 421 _GenericDeclaration.__init__(self, start, end, name, []) 422 self.templated_types = templated_types 423 if not name and modifiers: 424 self.name = modifiers.pop() 425 self.modifiers = modifiers 426 self.reference = reference 427 self.pointer = pointer 428 self.array = array 429 430 def __str__(self): 431 prefix = '' 432 if self.modifiers: 433 prefix = ' '.join(self.modifiers) + ' ' 434 name = str(self.name) 435 if self.templated_types: 436 name += '<%s>' % self.templated_types 437 suffix = prefix + name 438 if self.reference: 439 suffix += '&' 440 if self.pointer: 441 suffix += '*' 442 if self.array: 443 suffix += '[]' 444 return self._TypeStringHelper(suffix) 445 446 # By definition, Is* are always False. A Type can only exist in 447 # some sort of variable declaration, parameter, or return value. 448 def IsDeclaration(self): 449 return False 450 451 def IsDefinition(self): 452 return False 453 454 def IsExportable(self): 455 return False 456 457 458 class TypeConverter(object): 459 460 def __init__(self, namespace_stack): 461 self.namespace_stack = namespace_stack 462 463 def _GetTemplateEnd(self, tokens, start): 464 count = 1 465 end = start 466 while 1: 467 token = tokens[end] 468 end += 1 469 if token.name == '<': 470 count += 1 471 elif token.name == '>': 472 count -= 1 473 if count == 0: 474 break 475 return tokens[start:end-1], end 476 477 def ToType(self, tokens): 478 """Convert [Token,...] to [Class(...), ] useful for base classes. 479 For example, code like class Foo : public Bar<x, y> { ... }; 480 the "Bar<x, y>" portion gets converted to an AST. 481 482 Returns: 483 [Class(...), ...] 484 """ 485 result = [] 486 name_tokens = [] 487 reference = pointer = array = False 488 489 def AddType(templated_types): 490 # Partition tokens into name and modifier tokens. 491 names = [] 492 modifiers = [] 493 for t in name_tokens: 494 if keywords.IsKeyword(t.name): 495 modifiers.append(t.name) 496 else: 497 names.append(t.name) 498 name = ''.join(names) 499 if name_tokens: 500 result.append(Type(name_tokens[0].start, name_tokens[-1].end, 501 name, templated_types, modifiers, 502 reference, pointer, array)) 503 del name_tokens[:] 504 505 i = 0 506 end = len(tokens) 507 while i < end: 508 token = tokens[i] 509 if token.name == '<': 510 new_tokens, new_end = self._GetTemplateEnd(tokens, i+1) 511 AddType(self.ToType(new_tokens)) 512 # If there is a comma after the template, we need to consume 513 # that here otherwise it becomes part of the name. 514 i = new_end 515 reference = pointer = array = False 516 elif token.name == ',': 517 AddType([]) 518 reference = pointer = array = False 519 elif token.name == '*': 520 pointer = True 521 elif token.name == '&': 522 reference = True 523 elif token.name == '[': 524 pointer = True 525 elif token.name == ']': 526 pass 527 else: 528 name_tokens.append(token) 529 i += 1 530 531 if name_tokens: 532 # No '<' in the tokens, just a simple name and no template. 533 AddType([]) 534 return result 535 536 def DeclarationToParts(self, parts, needs_name_removed): 537 name = None 538 default = [] 539 if needs_name_removed: 540 # Handle default (initial) values properly. 541 for i, t in enumerate(parts): 542 if t.name == '=': 543 default = parts[i+1:] 544 name = parts[i-1].name 545 if name == ']' and parts[i-2].name == '[': 546 name = parts[i-3].name 547 i -= 1 548 parts = parts[:i-1] 549 break 550 else: 551 if parts[-1].token_type == tokenize.NAME: 552 name = parts.pop().name 553 else: 554 # TODO(nnorwitz): this is a hack that happens for code like 555 # Register(Foo<T>); where it thinks this is a function call 556 # but it's actually a declaration. 557 name = '???' 558 modifiers = [] 559 type_name = [] 560 other_tokens = [] 561 templated_types = [] 562 i = 0 563 end = len(parts) 564 while i < end: 565 p = parts[i] 566 if keywords.IsKeyword(p.name): 567 modifiers.append(p.name) 568 elif p.name == '<': 569 templated_tokens, new_end = self._GetTemplateEnd(parts, i+1) 570 templated_types = self.ToType(templated_tokens) 571 i = new_end - 1 572 # Don't add a spurious :: to data members being initialized. 573 next_index = i + 1 574 if next_index < end and parts[next_index].name == '::': 575 i += 1 576 elif p.name in ('[', ']', '='): 577 # These are handled elsewhere. 578 other_tokens.append(p) 579 elif p.name not in ('*', '&', '>'): 580 # Ensure that names have a space between them. 581 if (type_name and type_name[-1].token_type == tokenize.NAME and 582 p.token_type == tokenize.NAME): 583 type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0)) 584 type_name.append(p) 585 else: 586 other_tokens.append(p) 587 i += 1 588 type_name = ''.join([t.name for t in type_name]) 589 return name, type_name, templated_types, modifiers, default, other_tokens 590 591 def ToParameters(self, tokens): 592 if not tokens: 593 return [] 594 595 result = [] 596 name = type_name = '' 597 type_modifiers = [] 598 pointer = reference = array = False 599 first_token = None 600 default = [] 601 602 def AddParameter(end): 603 if default: 604 del default[0] # Remove flag. 605 parts = self.DeclarationToParts(type_modifiers, True) 606 (name, type_name, templated_types, modifiers, 607 unused_default, unused_other_tokens) = parts 608 parameter_type = Type(first_token.start, first_token.end, 609 type_name, templated_types, modifiers, 610 reference, pointer, array) 611 p = Parameter(first_token.start, end, name, 612 parameter_type, default) 613 result.append(p) 614 615 template_count = 0 616 for s in tokens: 617 if not first_token: 618 first_token = s 619 if s.name == '<': 620 template_count += 1 621 elif s.name == '>': 622 template_count -= 1 623 if template_count > 0: 624 type_modifiers.append(s) 625 continue 626 627 if s.name == ',': 628 AddParameter(s.start) 629 name = type_name = '' 630 type_modifiers = [] 631 pointer = reference = array = False 632 first_token = None 633 default = [] 634 elif s.name == '*': 635 pointer = True 636 elif s.name == '&': 637 reference = True 638 elif s.name == '[': 639 array = True 640 elif s.name == ']': 641 pass # Just don't add to type_modifiers. 642 elif s.name == '=': 643 # Got a default value. Add any value (None) as a flag. 644 default.append(None) 645 elif default: 646 default.append(s) 647 else: 648 type_modifiers.append(s) 649 AddParameter(tokens[-1].end) 650 return result 651 652 def CreateReturnType(self, return_type_seq): 653 if not return_type_seq: 654 return None 655 start = return_type_seq[0].start 656 end = return_type_seq[-1].end 657 _, name, templated_types, modifiers, default, other_tokens = \ 658 self.DeclarationToParts(return_type_seq, False) 659 names = [n.name for n in other_tokens] 660 reference = '&' in names 661 pointer = '*' in names 662 array = '[' in names 663 return Type(start, end, name, templated_types, modifiers, 664 reference, pointer, array) 665 666 def GetTemplateIndices(self, names): 667 # names is a list of strings. 668 start = names.index('<') 669 end = len(names) - 1 670 while end > 0: 671 if names[end] == '>': 672 break 673 end -= 1 674 return start, end+1 675 676 class AstBuilder(object): 677 def __init__(self, token_stream, filename, in_class='', visibility=None, 678 namespace_stack=[]): 679 self.tokens = token_stream 680 self.filename = filename 681 # TODO(nnorwitz): use a better data structure (deque) for the queue. 682 # Switching directions of the "queue" improved perf by about 25%. 683 # Using a deque should be even better since we access from both sides. 684 self.token_queue = [] 685 self.namespace_stack = namespace_stack[:] 686 self.in_class = in_class 687 if in_class is None: 688 self.in_class_name_only = None 689 else: 690 self.in_class_name_only = in_class.split('::')[-1] 691 self.visibility = visibility 692 self.in_function = False 693 self.current_token = None 694 # Keep the state whether we are currently handling a typedef or not. 695 self._handling_typedef = False 696 697 self.converter = TypeConverter(self.namespace_stack) 698 699 def HandleError(self, msg, token): 700 printable_queue = list(reversed(self.token_queue[-20:])) 701 sys.stderr.write('Got %s in %s @ %s %s\n' % 702 (msg, self.filename, token, printable_queue)) 703 704 def Generate(self): 705 while 1: 706 token = self._GetNextToken() 707 if not token: 708 break 709 710 # Get the next token. 711 self.current_token = token 712 713 # Dispatch on the next token type. 714 if token.token_type == _INTERNAL_TOKEN: 715 if token.name == _NAMESPACE_POP: 716 self.namespace_stack.pop() 717 continue 718 719 try: 720 result = self._GenerateOne(token) 721 if result is not None: 722 yield result 723 except: 724 self.HandleError('exception', token) 725 raise 726 727 def _CreateVariable(self, pos_token, name, type_name, type_modifiers, 728 ref_pointer_name_seq, templated_types, value=None): 729 reference = '&' in ref_pointer_name_seq 730 pointer = '*' in ref_pointer_name_seq 731 array = '[' in ref_pointer_name_seq 732 var_type = Type(pos_token.start, pos_token.end, type_name, 733 templated_types, type_modifiers, 734 reference, pointer, array) 735 return VariableDeclaration(pos_token.start, pos_token.end, 736 name, var_type, value, self.namespace_stack) 737 738 def _GenerateOne(self, token): 739 if token.token_type == tokenize.NAME: 740 if (keywords.IsKeyword(token.name) and 741 not keywords.IsBuiltinType(token.name)): 742 method = getattr(self, 'handle_' + token.name) 743 return method() 744 elif token.name == self.in_class_name_only: 745 # The token name is the same as the class, must be a ctor if 746 # there is a paren. Otherwise, it's the return type. 747 # Peek ahead to get the next token to figure out which. 748 next = self._GetNextToken() 749 self._AddBackToken(next) 750 if next.token_type == tokenize.SYNTAX and next.name == '(': 751 return self._GetMethod([token], FUNCTION_CTOR, None, True) 752 # Fall through--handle like any other method. 753 754 # Handle data or function declaration/definition. 755 syntax = tokenize.SYNTAX 756 temp_tokens, last_token = \ 757 self._GetVarTokensUpTo(syntax, '(', ';', '{', '[') 758 temp_tokens.insert(0, token) 759 if last_token.name == '(': 760 # If there is an assignment before the paren, 761 # this is an expression, not a method. 762 expr = bool([e for e in temp_tokens if e.name == '=']) 763 if expr: 764 new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';') 765 temp_tokens.append(last_token) 766 temp_tokens.extend(new_temp) 767 last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0) 768 769 if last_token.name == '[': 770 # Handle array, this isn't a method, unless it's an operator. 771 # TODO(nnorwitz): keep the size somewhere. 772 # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']') 773 temp_tokens.append(last_token) 774 if temp_tokens[-2].name == 'operator': 775 temp_tokens.append(self._GetNextToken()) 776 else: 777 temp_tokens2, last_token = \ 778 self._GetVarTokensUpTo(tokenize.SYNTAX, ';') 779 temp_tokens.extend(temp_tokens2) 780 781 if last_token.name == ';': 782 # Handle data, this isn't a method. 783 parts = self.converter.DeclarationToParts(temp_tokens, True) 784 (name, type_name, templated_types, modifiers, default, 785 unused_other_tokens) = parts 786 787 t0 = temp_tokens[0] 788 names = [t.name for t in temp_tokens] 789 if templated_types: 790 start, end = self.converter.GetTemplateIndices(names) 791 names = names[:start] + names[end:] 792 default = ''.join([t.name for t in default]) 793 return self._CreateVariable(t0, name, type_name, modifiers, 794 names, templated_types, default) 795 if last_token.name == '{': 796 self._AddBackTokens(temp_tokens[1:]) 797 self._AddBackToken(last_token) 798 method_name = temp_tokens[0].name 799 method = getattr(self, 'handle_' + method_name, None) 800 if not method: 801 # Must be declaring a variable. 802 # TODO(nnorwitz): handle the declaration. 803 return None 804 return method() 805 return self._GetMethod(temp_tokens, 0, None, False) 806 elif token.token_type == tokenize.SYNTAX: 807 if token.name == '~' and self.in_class: 808 # Must be a dtor (probably not in method body). 809 token = self._GetNextToken() 810 # self.in_class can contain A::Name, but the dtor will only 811 # be Name. Make sure to compare against the right value. 812 if (token.token_type == tokenize.NAME and 813 token.name == self.in_class_name_only): 814 return self._GetMethod([token], FUNCTION_DTOR, None, True) 815 # TODO(nnorwitz): handle a lot more syntax. 816 elif token.token_type == tokenize.PREPROCESSOR: 817 # TODO(nnorwitz): handle more preprocessor directives. 818 # token starts with a #, so remove it and strip whitespace. 819 name = token.name[1:].lstrip() 820 if name.startswith('include'): 821 # Remove "include". 822 name = name[7:].strip() 823 assert name 824 # Handle #include \<newline> "header-on-second-line.h". 825 if name.startswith('\\'): 826 name = name[1:].strip() 827 assert name[0] in '<"', token 828 assert name[-1] in '>"', token 829 system = name[0] == '<' 830 filename = name[1:-1] 831 return Include(token.start, token.end, filename, system) 832 if name.startswith('define'): 833 # Remove "define". 834 name = name[6:].strip() 835 assert name 836 value = '' 837 for i, c in enumerate(name): 838 if c.isspace(): 839 value = name[i:].lstrip() 840 name = name[:i] 841 break 842 return Define(token.start, token.end, name, value) 843 if name.startswith('if') and name[2:3].isspace(): 844 condition = name[3:].strip() 845 if condition.startswith('0') or condition.startswith('(0)'): 846 self._SkipIf0Blocks() 847 return None 848 849 def _GetTokensUpTo(self, expected_token_type, expected_token): 850 return self._GetVarTokensUpTo(expected_token_type, expected_token)[0] 851 852 def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens): 853 last_token = self._GetNextToken() 854 tokens = [] 855 while (last_token.token_type != expected_token_type or 856 last_token.name not in expected_tokens): 857 tokens.append(last_token) 858 last_token = self._GetNextToken() 859 return tokens, last_token 860 861 # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necesary. 862 def _IgnoreUpTo(self, token_type, token): 863 unused_tokens = self._GetTokensUpTo(token_type, token) 864 865 def _SkipIf0Blocks(self): 866 count = 1 867 while 1: 868 token = self._GetNextToken() 869 if token.token_type != tokenize.PREPROCESSOR: 870 continue 871 872 name = token.name[1:].lstrip() 873 if name.startswith('endif'): 874 count -= 1 875 if count == 0: 876 break 877 elif name.startswith('if'): 878 count += 1 879 880 def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None): 881 if GetNextToken is None: 882 GetNextToken = self._GetNextToken 883 # Assumes the current token is open_paren and we will consume 884 # and return up to the close_paren. 885 count = 1 886 token = GetNextToken() 887 while 1: 888 if token.token_type == tokenize.SYNTAX: 889 if token.name == open_paren: 890 count += 1 891 elif token.name == close_paren: 892 count -= 1 893 if count == 0: 894 break 895 yield token 896 token = GetNextToken() 897 yield token 898 899 def _GetParameters(self): 900 return self._GetMatchingChar('(', ')') 901 902 def GetScope(self): 903 return self._GetMatchingChar('{', '}') 904 905 def _GetNextToken(self): 906 if self.token_queue: 907 return self.token_queue.pop() 908 return next(self.tokens) 909 910 def _AddBackToken(self, token): 911 if token.whence == tokenize.WHENCE_STREAM: 912 token.whence = tokenize.WHENCE_QUEUE 913 self.token_queue.insert(0, token) 914 else: 915 assert token.whence == tokenize.WHENCE_QUEUE, token 916 self.token_queue.append(token) 917 918 def _AddBackTokens(self, tokens): 919 if tokens: 920 if tokens[-1].whence == tokenize.WHENCE_STREAM: 921 for token in tokens: 922 token.whence = tokenize.WHENCE_QUEUE 923 self.token_queue[:0] = reversed(tokens) 924 else: 925 assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens 926 self.token_queue.extend(reversed(tokens)) 927 928 def GetName(self, seq=None): 929 """Returns ([tokens], next_token_info).""" 930 GetNextToken = self._GetNextToken 931 if seq is not None: 932 it = iter(seq) 933 GetNextToken = lambda: next(it) 934 next_token = GetNextToken() 935 tokens = [] 936 last_token_was_name = False 937 while (next_token.token_type == tokenize.NAME or 938 (next_token.token_type == tokenize.SYNTAX and 939 next_token.name in ('::', '<'))): 940 # Two NAMEs in a row means the identifier should terminate. 941 # It's probably some sort of variable declaration. 942 if last_token_was_name and next_token.token_type == tokenize.NAME: 943 break 944 last_token_was_name = next_token.token_type == tokenize.NAME 945 tokens.append(next_token) 946 # Handle templated names. 947 if next_token.name == '<': 948 tokens.extend(self._GetMatchingChar('<', '>', GetNextToken)) 949 last_token_was_name = True 950 next_token = GetNextToken() 951 return tokens, next_token 952 953 def GetMethod(self, modifiers, templated_types): 954 return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') 955 assert len(return_type_and_name) >= 1 956 return self._GetMethod(return_type_and_name, modifiers, templated_types, 957 False) 958 959 def _GetMethod(self, return_type_and_name, modifiers, templated_types, 960 get_paren): 961 template_portion = None 962 if get_paren: 963 token = self._GetNextToken() 964 assert token.token_type == tokenize.SYNTAX, token 965 if token.name == '<': 966 # Handle templatized dtors. 967 template_portion = [token] 968 template_portion.extend(self._GetMatchingChar('<', '>')) 969 token = self._GetNextToken() 970 assert token.token_type == tokenize.SYNTAX, token 971 assert token.name == '(', token 972 973 name = return_type_and_name.pop() 974 # Handle templatized ctors. 975 if name.name == '>': 976 index = 1 977 while return_type_and_name[index].name != '<': 978 index += 1 979 template_portion = return_type_and_name[index:] + [name] 980 del return_type_and_name[index:] 981 name = return_type_and_name.pop() 982 elif name.name == ']': 983 rt = return_type_and_name 984 assert rt[-1].name == '[', return_type_and_name 985 assert rt[-2].name == 'operator', return_type_and_name 986 name_seq = return_type_and_name[-2:] 987 del return_type_and_name[-2:] 988 name = tokenize.Token(tokenize.NAME, 'operator[]', 989 name_seq[0].start, name.end) 990 # Get the open paren so _GetParameters() below works. 991 unused_open_paren = self._GetNextToken() 992 993 # TODO(nnorwitz): store template_portion. 994 return_type = return_type_and_name 995 indices = name 996 if return_type: 997 indices = return_type[0] 998 999 # Force ctor for templatized ctors. 1000 if name.name == self.in_class and not modifiers: 1001 modifiers |= FUNCTION_CTOR 1002 parameters = list(self._GetParameters()) 1003 del parameters[-1] # Remove trailing ')'. 1004 1005 # Handling operator() is especially weird. 1006 if name.name == 'operator' and not parameters: 1007 token = self._GetNextToken() 1008 assert token.name == '(', token 1009 parameters = list(self._GetParameters()) 1010 del parameters[-1] # Remove trailing ')'. 1011 1012 token = self._GetNextToken() 1013 while token.token_type == tokenize.NAME: 1014 modifier_token = token 1015 token = self._GetNextToken() 1016 if modifier_token.name == 'const': 1017 modifiers |= FUNCTION_CONST 1018 elif modifier_token.name == '__attribute__': 1019 # TODO(nnorwitz): handle more __attribute__ details. 1020 modifiers |= FUNCTION_ATTRIBUTE 1021 assert token.name == '(', token 1022 # Consume everything between the (parens). 1023 unused_tokens = list(self._GetMatchingChar('(', ')')) 1024 token = self._GetNextToken() 1025 elif modifier_token.name == 'throw': 1026 modifiers |= FUNCTION_THROW 1027 assert token.name == '(', token 1028 # Consume everything between the (parens). 1029 unused_tokens = list(self._GetMatchingChar('(', ')')) 1030 token = self._GetNextToken() 1031 elif modifier_token.name == 'override': 1032 modifiers |= FUNCTION_OVERRIDE 1033 elif modifier_token.name == modifier_token.name.upper(): 1034 # HACK(nnorwitz): assume that all upper-case names 1035 # are some macro we aren't expanding. 1036 modifiers |= FUNCTION_UNKNOWN_ANNOTATION 1037 else: 1038 self.HandleError('unexpected token', modifier_token) 1039 1040 assert token.token_type == tokenize.SYNTAX, token 1041 # Handle ctor initializers. 1042 if token.name == ':': 1043 # TODO(nnorwitz): anything else to handle for initializer list? 1044 while token.name != ';' and token.name != '{': 1045 token = self._GetNextToken() 1046 1047 # Handle pointer to functions that are really data but look 1048 # like method declarations. 1049 if token.name == '(': 1050 if parameters[0].name == '*': 1051 # name contains the return type. 1052 name = parameters.pop() 1053 # parameters contains the name of the data. 1054 modifiers = [p.name for p in parameters] 1055 # Already at the ( to open the parameter list. 1056 function_parameters = list(self._GetMatchingChar('(', ')')) 1057 del function_parameters[-1] # Remove trailing ')'. 1058 # TODO(nnorwitz): store the function_parameters. 1059 token = self._GetNextToken() 1060 assert token.token_type == tokenize.SYNTAX, token 1061 assert token.name == ';', token 1062 return self._CreateVariable(indices, name.name, indices.name, 1063 modifiers, '', None) 1064 # At this point, we got something like: 1065 # return_type (type::*name_)(params); 1066 # This is a data member called name_ that is a function pointer. 1067 # With this code: void (sq_type::*field_)(string&); 1068 # We get: name=void return_type=[] parameters=sq_type ... field_ 1069 # TODO(nnorwitz): is return_type always empty? 1070 # TODO(nnorwitz): this isn't even close to being correct. 1071 # Just put in something so we don't crash and can move on. 1072 real_name = parameters[-1] 1073 modifiers = [p.name for p in self._GetParameters()] 1074 del modifiers[-1] # Remove trailing ')'. 1075 return self._CreateVariable(indices, real_name.name, indices.name, 1076 modifiers, '', None) 1077 1078 if token.name == '{': 1079 body = list(self.GetScope()) 1080 del body[-1] # Remove trailing '}'. 1081 else: 1082 body = None 1083 if token.name == '=': 1084 token = self._GetNextToken() 1085 1086 if token.name == 'default' or token.name == 'delete': 1087 # Ignore explicitly defaulted and deleted special members 1088 # in C++11. 1089 token = self._GetNextToken() 1090 else: 1091 # Handle pure-virtual declarations. 1092 assert token.token_type == tokenize.CONSTANT, token 1093 assert token.name == '0', token 1094 modifiers |= FUNCTION_PURE_VIRTUAL 1095 token = self._GetNextToken() 1096 1097 if token.name == '[': 1098 # TODO(nnorwitz): store tokens and improve parsing. 1099 # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N]; 1100 tokens = list(self._GetMatchingChar('[', ']')) 1101 token = self._GetNextToken() 1102 1103 assert token.name == ';', (token, return_type_and_name, parameters) 1104 1105 # Looks like we got a method, not a function. 1106 if len(return_type) > 2 and return_type[-1].name == '::': 1107 return_type, in_class = \ 1108 self._GetReturnTypeAndClassName(return_type) 1109 return Method(indices.start, indices.end, name.name, in_class, 1110 return_type, parameters, modifiers, templated_types, 1111 body, self.namespace_stack) 1112 return Function(indices.start, indices.end, name.name, return_type, 1113 parameters, modifiers, templated_types, body, 1114 self.namespace_stack) 1115 1116 def _GetReturnTypeAndClassName(self, token_seq): 1117 # Splitting the return type from the class name in a method 1118 # can be tricky. For example, Return::Type::Is::Hard::To::Find(). 1119 # Where is the return type and where is the class name? 1120 # The heuristic used is to pull the last name as the class name. 1121 # This includes all the templated type info. 1122 # TODO(nnorwitz): if there is only One name like in the 1123 # example above, punt and assume the last bit is the class name. 1124 1125 # Ignore a :: prefix, if exists so we can find the first real name. 1126 i = 0 1127 if token_seq[0].name == '::': 1128 i = 1 1129 # Ignore a :: suffix, if exists. 1130 end = len(token_seq) - 1 1131 if token_seq[end-1].name == '::': 1132 end -= 1 1133 1134 # Make a copy of the sequence so we can append a sentinel 1135 # value. This is required for GetName will has to have some 1136 # terminating condition beyond the last name. 1137 seq_copy = token_seq[i:end] 1138 seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0)) 1139 names = [] 1140 while i < end: 1141 # Iterate through the sequence parsing out each name. 1142 new_name, next = self.GetName(seq_copy[i:]) 1143 assert new_name, 'Got empty new_name, next=%s' % next 1144 # We got a pointer or ref. Add it to the name. 1145 if next and next.token_type == tokenize.SYNTAX: 1146 new_name.append(next) 1147 names.append(new_name) 1148 i += len(new_name) 1149 1150 # Now that we have the names, it's time to undo what we did. 1151 1152 # Remove the sentinel value. 1153 names[-1].pop() 1154 # Flatten the token sequence for the return type. 1155 return_type = [e for seq in names[:-1] for e in seq] 1156 # The class name is the last name. 1157 class_name = names[-1] 1158 return return_type, class_name 1159 1160 def handle_bool(self): 1161 pass 1162 1163 def handle_char(self): 1164 pass 1165 1166 def handle_int(self): 1167 pass 1168 1169 def handle_long(self): 1170 pass 1171 1172 def handle_short(self): 1173 pass 1174 1175 def handle_double(self): 1176 pass 1177 1178 def handle_float(self): 1179 pass 1180 1181 def handle_void(self): 1182 pass 1183 1184 def handle_wchar_t(self): 1185 pass 1186 1187 def handle_unsigned(self): 1188 pass 1189 1190 def handle_signed(self): 1191 pass 1192 1193 def _GetNestedType(self, ctor): 1194 name = None 1195 name_tokens, token = self.GetName() 1196 if name_tokens: 1197 name = ''.join([t.name for t in name_tokens]) 1198 1199 # Handle forward declarations. 1200 if token.token_type == tokenize.SYNTAX and token.name == ';': 1201 return ctor(token.start, token.end, name, None, 1202 self.namespace_stack) 1203 1204 if token.token_type == tokenize.NAME and self._handling_typedef: 1205 self._AddBackToken(token) 1206 return ctor(token.start, token.end, name, None, 1207 self.namespace_stack) 1208 1209 # Must be the type declaration. 1210 fields = list(self._GetMatchingChar('{', '}')) 1211 del fields[-1] # Remove trailing '}'. 1212 if token.token_type == tokenize.SYNTAX and token.name == '{': 1213 next = self._GetNextToken() 1214 new_type = ctor(token.start, token.end, name, fields, 1215 self.namespace_stack) 1216 # A name means this is an anonymous type and the name 1217 # is the variable declaration. 1218 if next.token_type != tokenize.NAME: 1219 return new_type 1220 name = new_type 1221 token = next 1222 1223 # Must be variable declaration using the type prefixed with keyword. 1224 assert token.token_type == tokenize.NAME, token 1225 return self._CreateVariable(token, token.name, name, [], '', None) 1226 1227 def handle_struct(self): 1228 # Special case the handling typedef/aliasing of structs here. 1229 # It would be a pain to handle in the class code. 1230 name_tokens, var_token = self.GetName() 1231 if name_tokens: 1232 next_token = self._GetNextToken() 1233 is_syntax = (var_token.token_type == tokenize.SYNTAX and 1234 var_token.name[0] in '*&') 1235 is_variable = (var_token.token_type == tokenize.NAME and 1236 next_token.name == ';') 1237 variable = var_token 1238 if is_syntax and not is_variable: 1239 variable = next_token 1240 temp = self._GetNextToken() 1241 if temp.token_type == tokenize.SYNTAX and temp.name == '(': 1242 # Handle methods declared to return a struct. 1243 t0 = name_tokens[0] 1244 struct = tokenize.Token(tokenize.NAME, 'struct', 1245 t0.start-7, t0.start-2) 1246 type_and_name = [struct] 1247 type_and_name.extend(name_tokens) 1248 type_and_name.extend((var_token, next_token)) 1249 return self._GetMethod(type_and_name, 0, None, False) 1250 assert temp.name == ';', (temp, name_tokens, var_token) 1251 if is_syntax or (is_variable and not self._handling_typedef): 1252 modifiers = ['struct'] 1253 type_name = ''.join([t.name for t in name_tokens]) 1254 position = name_tokens[0] 1255 return self._CreateVariable(position, variable.name, type_name, 1256 modifiers, var_token.name, None) 1257 name_tokens.extend((var_token, next_token)) 1258 self._AddBackTokens(name_tokens) 1259 else: 1260 self._AddBackToken(var_token) 1261 return self._GetClass(Struct, VISIBILITY_PUBLIC, None) 1262 1263 def handle_union(self): 1264 return self._GetNestedType(Union) 1265 1266 def handle_enum(self): 1267 return self._GetNestedType(Enum) 1268 1269 def handle_auto(self): 1270 # TODO(nnorwitz): warn about using auto? Probably not since it 1271 # will be reclaimed and useful for C++0x. 1272 pass 1273 1274 def handle_register(self): 1275 pass 1276 1277 def handle_const(self): 1278 pass 1279 1280 def handle_inline(self): 1281 pass 1282 1283 def handle_extern(self): 1284 pass 1285 1286 def handle_static(self): 1287 pass 1288 1289 def handle_virtual(self): 1290 # What follows must be a method. 1291 token = token2 = self._GetNextToken() 1292 if token.name == 'inline': 1293 # HACK(nnorwitz): handle inline dtors by ignoring 'inline'. 1294 token2 = self._GetNextToken() 1295 if token2.token_type == tokenize.SYNTAX and token2.name == '~': 1296 return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None) 1297 assert token.token_type == tokenize.NAME or token.name == '::', token 1298 return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') # ) 1299 return_type_and_name.insert(0, token) 1300 if token2 is not token: 1301 return_type_and_name.insert(1, token2) 1302 return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL, 1303 None, False) 1304 1305 def handle_volatile(self): 1306 pass 1307 1308 def handle_mutable(self): 1309 pass 1310 1311 def handle_public(self): 1312 assert self.in_class 1313 self.visibility = VISIBILITY_PUBLIC 1314 1315 def handle_protected(self): 1316 assert self.in_class 1317 self.visibility = VISIBILITY_PROTECTED 1318 1319 def handle_private(self): 1320 assert self.in_class 1321 self.visibility = VISIBILITY_PRIVATE 1322 1323 def handle_friend(self): 1324 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') 1325 assert tokens 1326 t0 = tokens[0] 1327 return Friend(t0.start, t0.end, tokens, self.namespace_stack) 1328 1329 def handle_static_cast(self): 1330 pass 1331 1332 def handle_const_cast(self): 1333 pass 1334 1335 def handle_dynamic_cast(self): 1336 pass 1337 1338 def handle_reinterpret_cast(self): 1339 pass 1340 1341 def handle_new(self): 1342 pass 1343 1344 def handle_delete(self): 1345 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') 1346 assert tokens 1347 return Delete(tokens[0].start, tokens[0].end, tokens) 1348 1349 def handle_typedef(self): 1350 token = self._GetNextToken() 1351 if (token.token_type == tokenize.NAME and 1352 keywords.IsKeyword(token.name)): 1353 # Token must be struct/enum/union/class. 1354 method = getattr(self, 'handle_' + token.name) 1355 self._handling_typedef = True 1356 tokens = [method()] 1357 self._handling_typedef = False 1358 else: 1359 tokens = [token] 1360 1361 # Get the remainder of the typedef up to the semi-colon. 1362 tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';')) 1363 1364 # TODO(nnorwitz): clean all this up. 1365 assert tokens 1366 name = tokens.pop() 1367 indices = name 1368 if tokens: 1369 indices = tokens[0] 1370 if not indices: 1371 indices = token 1372 if name.name == ')': 1373 # HACK(nnorwitz): Handle pointers to functions "properly". 1374 if (len(tokens) >= 4 and 1375 tokens[1].name == '(' and tokens[2].name == '*'): 1376 tokens.append(name) 1377 name = tokens[3] 1378 elif name.name == ']': 1379 # HACK(nnorwitz): Handle arrays properly. 1380 if len(tokens) >= 2: 1381 tokens.append(name) 1382 name = tokens[1] 1383 new_type = tokens 1384 if tokens and isinstance(tokens[0], tokenize.Token): 1385 new_type = self.converter.ToType(tokens)[0] 1386 return Typedef(indices.start, indices.end, name.name, 1387 new_type, self.namespace_stack) 1388 1389 def handle_typeid(self): 1390 pass # Not needed yet. 1391 1392 def handle_typename(self): 1393 pass # Not needed yet. 1394 1395 def _GetTemplatedTypes(self): 1396 result = {} 1397 tokens = list(self._GetMatchingChar('<', '>')) 1398 len_tokens = len(tokens) - 1 # Ignore trailing '>'. 1399 i = 0 1400 while i < len_tokens: 1401 key = tokens[i].name 1402 i += 1 1403 if keywords.IsKeyword(key) or key == ',': 1404 continue 1405 type_name = default = None 1406 if i < len_tokens: 1407 i += 1 1408 if tokens[i-1].name == '=': 1409 assert i < len_tokens, '%s %s' % (i, tokens) 1410 default, unused_next_token = self.GetName(tokens[i:]) 1411 i += len(default) 1412 else: 1413 if tokens[i-1].name != ',': 1414 # We got something like: Type variable. 1415 # Re-adjust the key (variable) and type_name (Type). 1416 key = tokens[i-1].name 1417 type_name = tokens[i-2] 1418 1419 result[key] = (type_name, default) 1420 return result 1421 1422 def handle_template(self): 1423 token = self._GetNextToken() 1424 assert token.token_type == tokenize.SYNTAX, token 1425 assert token.name == '<', token 1426 templated_types = self._GetTemplatedTypes() 1427 # TODO(nnorwitz): for now, just ignore the template params. 1428 token = self._GetNextToken() 1429 if token.token_type == tokenize.NAME: 1430 if token.name == 'class': 1431 return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types) 1432 elif token.name == 'struct': 1433 return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types) 1434 elif token.name == 'friend': 1435 return self.handle_friend() 1436 self._AddBackToken(token) 1437 tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';') 1438 tokens.append(last) 1439 self._AddBackTokens(tokens) 1440 if last.name == '(': 1441 return self.GetMethod(FUNCTION_NONE, templated_types) 1442 # Must be a variable definition. 1443 return None 1444 1445 def handle_true(self): 1446 pass # Nothing to do. 1447 1448 def handle_false(self): 1449 pass # Nothing to do. 1450 1451 def handle_asm(self): 1452 pass # Not needed yet. 1453 1454 def handle_class(self): 1455 return self._GetClass(Class, VISIBILITY_PRIVATE, None) 1456 1457 def _GetBases(self): 1458 # Get base classes. 1459 bases = [] 1460 while 1: 1461 token = self._GetNextToken() 1462 assert token.token_type == tokenize.NAME, token 1463 # TODO(nnorwitz): store kind of inheritance...maybe. 1464 if token.name not in ('public', 'protected', 'private'): 1465 # If inheritance type is not specified, it is private. 1466 # Just put the token back so we can form a name. 1467 # TODO(nnorwitz): it would be good to warn about this. 1468 self._AddBackToken(token) 1469 else: 1470 # Check for virtual inheritance. 1471 token = self._GetNextToken() 1472 if token.name != 'virtual': 1473 self._AddBackToken(token) 1474 else: 1475 # TODO(nnorwitz): store that we got virtual for this base. 1476 pass 1477 base, next_token = self.GetName() 1478 bases_ast = self.converter.ToType(base) 1479 assert len(bases_ast) == 1, bases_ast 1480 bases.append(bases_ast[0]) 1481 assert next_token.token_type == tokenize.SYNTAX, next_token 1482 if next_token.name == '{': 1483 token = next_token 1484 break 1485 # Support multiple inheritance. 1486 assert next_token.name == ',', next_token 1487 return bases, token 1488 1489 def _GetClass(self, class_type, visibility, templated_types): 1490 class_name = None 1491 class_token = self._GetNextToken() 1492 if class_token.token_type != tokenize.NAME: 1493 assert class_token.token_type == tokenize.SYNTAX, class_token 1494 token = class_token 1495 else: 1496 # Skip any macro (e.g. storage class specifiers) after the 1497 # 'class' keyword. 1498 next_token = self._GetNextToken() 1499 if next_token.token_type == tokenize.NAME: 1500 self._AddBackToken(next_token) 1501 else: 1502 self._AddBackTokens([class_token, next_token]) 1503 name_tokens, token = self.GetName() 1504 class_name = ''.join([t.name for t in name_tokens]) 1505 bases = None 1506 if token.token_type == tokenize.SYNTAX: 1507 if token.name == ';': 1508 # Forward declaration. 1509 return class_type(class_token.start, class_token.end, 1510 class_name, None, templated_types, None, 1511 self.namespace_stack) 1512 if token.name in '*&': 1513 # Inline forward declaration. Could be method or data. 1514 name_token = self._GetNextToken() 1515 next_token = self._GetNextToken() 1516 if next_token.name == ';': 1517 # Handle data 1518 modifiers = ['class'] 1519 return self._CreateVariable(class_token, name_token.name, 1520 class_name, 1521 modifiers, token.name, None) 1522 else: 1523 # Assume this is a method. 1524 tokens = (class_token, token, name_token, next_token) 1525 self._AddBackTokens(tokens) 1526 return self.GetMethod(FUNCTION_NONE, None) 1527 if token.name == ':': 1528 bases, token = self._GetBases() 1529 1530 body = None 1531 if token.token_type == tokenize.SYNTAX and token.name == '{': 1532 assert token.token_type == tokenize.SYNTAX, token 1533 assert token.name == '{', token 1534 1535 ast = AstBuilder(self.GetScope(), self.filename, class_name, 1536 visibility, self.namespace_stack) 1537 body = list(ast.Generate()) 1538 1539 if not self._handling_typedef: 1540 token = self._GetNextToken() 1541 if token.token_type != tokenize.NAME: 1542 assert token.token_type == tokenize.SYNTAX, token 1543 assert token.name == ';', token 1544 else: 1545 new_class = class_type(class_token.start, class_token.end, 1546 class_name, bases, None, 1547 body, self.namespace_stack) 1548 1549 modifiers = [] 1550 return self._CreateVariable(class_token, 1551 token.name, new_class, 1552 modifiers, token.name, None) 1553 else: 1554 if not self._handling_typedef: 1555 self.HandleError('non-typedef token', token) 1556 self._AddBackToken(token) 1557 1558 return class_type(class_token.start, class_token.end, class_name, 1559 bases, templated_types, body, self.namespace_stack) 1560 1561 def handle_namespace(self): 1562 token = self._GetNextToken() 1563 # Support anonymous namespaces. 1564 name = None 1565 if token.token_type == tokenize.NAME: 1566 name = token.name 1567 token = self._GetNextToken() 1568 self.namespace_stack.append(name) 1569 assert token.token_type == tokenize.SYNTAX, token 1570 # Create an internal token that denotes when the namespace is complete. 1571 internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP, 1572 None, None) 1573 internal_token.whence = token.whence 1574 if token.name == '=': 1575 # TODO(nnorwitz): handle aliasing namespaces. 1576 name, next_token = self.GetName() 1577 assert next_token.name == ';', next_token 1578 self._AddBackToken(internal_token) 1579 else: 1580 assert token.name == '{', token 1581 tokens = list(self.GetScope()) 1582 # Replace the trailing } with the internal namespace pop token. 1583 tokens[-1] = internal_token 1584 # Handle namespace with nothing in it. 1585 self._AddBackTokens(tokens) 1586 return None 1587 1588 def handle_using(self): 1589 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') 1590 assert tokens 1591 return Using(tokens[0].start, tokens[0].end, tokens) 1592 1593 def handle_explicit(self): 1594 assert self.in_class 1595 # Nothing much to do. 1596 # TODO(nnorwitz): maybe verify the method name == class name. 1597 # This must be a ctor. 1598 return self.GetMethod(FUNCTION_CTOR, None) 1599 1600 def handle_this(self): 1601 pass # Nothing to do. 1602 1603 def handle_operator(self): 1604 # Pull off the next token(s?) and make that part of the method name. 1605 pass 1606 1607 def handle_sizeof(self): 1608 pass 1609 1610 def handle_case(self): 1611 pass 1612 1613 def handle_switch(self): 1614 pass 1615 1616 def handle_default(self): 1617 token = self._GetNextToken() 1618 assert token.token_type == tokenize.SYNTAX 1619 assert token.name == ':' 1620 1621 def handle_if(self): 1622 pass 1623 1624 def handle_else(self): 1625 pass 1626 1627 def handle_return(self): 1628 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') 1629 if not tokens: 1630 return Return(self.current_token.start, self.current_token.end, None) 1631 return Return(tokens[0].start, tokens[0].end, tokens) 1632 1633 def handle_goto(self): 1634 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') 1635 assert len(tokens) == 1, str(tokens) 1636 return Goto(tokens[0].start, tokens[0].end, tokens[0].name) 1637 1638 def handle_try(self): 1639 pass # Not needed yet. 1640 1641 def handle_catch(self): 1642 pass # Not needed yet. 1643 1644 def handle_throw(self): 1645 pass # Not needed yet. 1646 1647 def handle_while(self): 1648 pass 1649 1650 def handle_do(self): 1651 pass 1652 1653 def handle_for(self): 1654 pass 1655 1656 def handle_break(self): 1657 self._IgnoreUpTo(tokenize.SYNTAX, ';') 1658 1659 def handle_continue(self): 1660 self._IgnoreUpTo(tokenize.SYNTAX, ';') 1661 1662 1663 def BuilderFromSource(source, filename): 1664 """Utility method that returns an AstBuilder from source code. 1665 1666 Args: 1667 source: 'C++ source code' 1668 filename: 'file1' 1669 1670 Returns: 1671 AstBuilder 1672 """ 1673 return AstBuilder(tokenize.GetTokens(source), filename) 1674 1675 1676 def PrintIndentifiers(filename, should_print): 1677 """Prints all identifiers for a C++ source file. 1678 1679 Args: 1680 filename: 'file1' 1681 should_print: predicate with signature: bool Function(token) 1682 """ 1683 source = utils.ReadFile(filename, False) 1684 if source is None: 1685 sys.stderr.write('Unable to find: %s\n' % filename) 1686 return 1687 1688 #print('Processing %s' % actual_filename) 1689 builder = BuilderFromSource(source, filename) 1690 try: 1691 for node in builder.Generate(): 1692 if should_print(node): 1693 print(node.name) 1694 except KeyboardInterrupt: 1695 return 1696 except: 1697 pass 1698 1699 1700 def PrintAllIndentifiers(filenames, should_print): 1701 """Prints all identifiers for each C++ source file in filenames. 1702 1703 Args: 1704 filenames: ['file1', 'file2', ...] 1705 should_print: predicate with signature: bool Function(token) 1706 """ 1707 for path in filenames: 1708 PrintIndentifiers(path, should_print) 1709 1710 1711 def main(argv): 1712 for filename in argv[1:]: 1713 source = utils.ReadFile(filename) 1714 if source is None: 1715 continue 1716 1717 print('Processing %s' % filename) 1718 builder = BuilderFromSource(source, filename) 1719 try: 1720 entire_ast = filter(None, builder.Generate()) 1721 except KeyboardInterrupt: 1722 return 1723 except: 1724 # Already printed a warning, print the traceback and continue. 1725 traceback.print_exc() 1726 else: 1727 if utils.DEBUG: 1728 for ast in entire_ast: 1729 print(ast) 1730 1731 1732 if __name__ == '__main__': 1733 main(sys.argv) 1734
