1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """ Parser for PPAPI IDL """ 7 8 # 9 # IDL Parser 10 # 11 # The parser is uses the PLY yacc library to build a set of parsing rules based 12 # on WebIDL. 13 # 14 # WebIDL, and WebIDL regular expressions can be found at: 15 # http://dev.w3.org/2006/webapi/WebIDL/ 16 # PLY can be found at: 17 # http://www.dabeaz.com/ply/ 18 # 19 # The parser generates a tree by recursively matching sets of items against 20 # defined patterns. When a match is made, that set of items is reduced 21 # to a new item. The new item can provide a match for parent patterns. 22 # In this way an AST is built (reduced) depth first. 23 24 25 import getopt 26 import glob 27 import os.path 28 import re 29 import sys 30 import time 31 32 from idl_ast import IDLAst 33 from idl_log import ErrOut, InfoOut, WarnOut 34 from idl_lexer import IDLLexer 35 from idl_node import IDLAttribute, IDLFile, IDLNode 36 from idl_option import GetOption, Option, ParseOptions 37 from idl_lint import Lint 38 39 from ply import lex 40 from ply import yacc 41 42 Option('build_debug', 'Debug tree building.') 43 Option('parse_debug', 'Debug parse reduction steps.') 44 Option('token_debug', 'Debug token generation.') 45 Option('dump_tree', 'Dump the tree.') 46 Option('srcroot', 'Working directory.', default=os.path.join('..', 'api')) 47 Option('include_private', 'Include private IDL directory in default API paths.') 48 49 # 50 # ERROR_REMAP 51 # 52 # Maps the standard error formula into a more friendly error message. 53 # 54 ERROR_REMAP = { 55 'Unexpected ")" after "(".' : 'Empty argument list.', 56 'Unexpected ")" after ",".' : 'Missing argument.', 57 'Unexpected "}" after ",".' : 'Trailing comma in block.', 58 'Unexpected "}" after "{".' : 'Unexpected empty block.', 59 'Unexpected comment after "}".' : 'Unexpected trailing comment.', 60 'Unexpected "{" after keyword "enum".' : 'Enum missing name.', 61 'Unexpected "{" after keyword "struct".' : 'Struct missing name.', 62 'Unexpected "{" after keyword "interface".' : 'Interface missing name.', 63 } 64 65 # DumpReduction 66 # 67 # Prints out the set of items which matched a particular pattern and the 68 # new item or set it was reduced to. 69 def DumpReduction(cls, p): 70 if p[0] is None: 71 InfoOut.Log("OBJ: %s(%d) - None\n" % (cls, len(p))) 72 InfoOut.Log(" [%s]\n" % [str(x) for x in p[1:]]) 73 else: 74 out = "" 75 for index in range(len(p) - 1): 76 out += " >%s< " % str(p[index + 1]) 77 InfoOut.Log("OBJ: %s(%d) - %s : %s\n" % (cls, len(p), str(p[0]), out)) 78 79 80 # CopyToList 81 # 82 # Takes an input item, list, or None, and returns a new list of that set. 83 def CopyToList(item): 84 # If the item is 'Empty' make it an empty list 85 if not item: item = [] 86 87 # If the item is not a list 88 if type(item) is not type([]): item = [item] 89 90 # Make a copy we can modify 91 return list(item) 92 93 94 95 # ListFromConcat 96 # 97 # Generate a new List by joining of two sets of inputs which can be an 98 # individual item, a list of items, or None. 99 def ListFromConcat(*items): 100 itemsout = [] 101 for item in items: 102 itemlist = CopyToList(item) 103 itemsout.extend(itemlist) 104 105 return itemsout 106 107 108 # TokenTypeName 109 # 110 # Generate a string which has the type and value of the token. 111 def TokenTypeName(t): 112 if t.type == 'SYMBOL': return 'symbol %s' % t.value 113 if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']: 114 return 'value %s' % t.value 115 if t.type == 'STRING' : return 'string "%s"' % t.value 116 if t.type == 'COMMENT' : return 'comment' 117 if t.type == t.value: return '"%s"' % t.value 118 return 'keyword "%s"' % t.value 119 120 121 # 122 # IDL Parser 123 # 124 # The Parser inherits the from the Lexer to provide PLY with the tokenizing 125 # definitions. Parsing patterns are encoded as function where p_<name> is 126 # is called any time a patern matching the function documentation is found. 127 # Paterns are expressed in the form of: 128 # """ <new item> : <item> .... 129 # | <item> ....""" 130 # 131 # Where new item is the result of a match against one or more sets of items 132 # separated by the "|". 133 # 134 # The function is called with an object 'p' where p[0] is the output object 135 # and p[n] is the set of inputs for positive values of 'n'. Len(p) can be 136 # used to distinguish between multiple item sets in the pattern. 137 # 138 # For more details on parsing refer to the PLY documentation at 139 # http://www.dabeaz.com/ply/ 140 # 141 # 142 # The parser uses the following conventions: 143 # a <type>_block defines a block of <type> definitions in the form of: 144 # [comment] [ext_attr_block] <type> <name> '{' <type>_list '}' ';' 145 # A block is reduced by returning an object of <type> with a name of <name> 146 # which in turn has <type>_list as children. 147 # 148 # A [comment] is a optional C style comment block enclosed in /* ... */ which 149 # is appended to the adjacent node as a child. 150 # 151 # A [ext_attr_block] is an optional list of Extended Attributes which is 152 # appended to the adjacent node as a child. 153 # 154 # a <type>_list defines a list of <type> items which will be passed as a 155 # list of children to the parent pattern. A list is in the form of: 156 # [comment] [ext_attr_block] <...DEF...> ';' <type>_list | (empty) 157 # or 158 # [comment] [ext_attr_block] <...DEF...> <type>_cont 159 # 160 # In the first form, the list is reduced recursively, where the right side 161 # <type>_list is first reduced then joined with pattern currently being 162 # matched. The list is terminated with the (empty) pattern is matched. 163 # 164 # In the second form the list is reduced recursively, where the right side 165 # <type>_cont is first reduced then joined with the pattern currently being 166 # matched. The type_<cont> is in the form of: 167 # ',' <type>_list | (empty) 168 # The <type>_cont form is used to consume the ',' which only occurs when 169 # there is more than one object in the list. The <type>_cont also provides 170 # the terminating (empty) definition. 171 # 172 173 174 class IDLParser(IDLLexer): 175 # TOP 176 # 177 # This pattern defines the top of the parse tree. The parse tree is in the 178 # the form of: 179 # 180 # top 181 # *modifiers 182 # *comments 183 # *ext_attr_block 184 # ext_attr_list 185 # attr_arg_list 186 # *integer, value 187 # *param_list 188 # *typeref 189 # 190 # top_list 191 # describe_block 192 # describe_list 193 # enum_block 194 # enum_item 195 # interface_block 196 # member 197 # label_block 198 # label_item 199 # struct_block 200 # member 201 # typedef_decl 202 # typedef_data 203 # typedef_func 204 # 205 # (* sub matches found at multiple levels and are not truly children of top) 206 # 207 # We force all input files to start with two comments. The first comment is a 208 # Copyright notice followed by a set of file wide Extended Attributes, followed 209 # by the file comment and finally by file level patterns. 210 # 211 # Find the Copyright, File comment, and optional file wide attributes. We 212 # use a match with COMMENT instead of comments to force the token to be 213 # present. The extended attributes and the top_list become siblings which 214 # in turn are children of the file object created from the results of top. 215 def p_top(self, p): 216 """top : COMMENT COMMENT ext_attr_block top_list""" 217 218 Copyright = self.BuildComment('Copyright', p, 1) 219 Filedoc = self.BuildComment('Comment', p, 2) 220 221 p[0] = ListFromConcat(Copyright, Filedoc, p[3], p[4]) 222 if self.parse_debug: DumpReduction('top', p) 223 224 def p_top_short(self, p): 225 """top : COMMENT ext_attr_block top_list""" 226 Copyright = self.BuildComment('Copyright', p, 1) 227 Filedoc = IDLNode('Comment', self.lexobj.filename, p.lineno(2)-1, 228 p.lexpos(2)-1, [self.BuildAttribute('NAME', ''), 229 self.BuildAttribute('FORM', 'cc')]) 230 p[0] = ListFromConcat(Copyright, Filedoc, p[2], p[3]) 231 if self.parse_debug: DumpReduction('top', p) 232 233 # Build a list of top level items. 234 def p_top_list(self, p): 235 """top_list : callback_decl top_list 236 | describe_block top_list 237 | dictionary_block top_list 238 | enum_block top_list 239 | inline top_list 240 | interface_block top_list 241 | label_block top_list 242 | namespace top_list 243 | struct_block top_list 244 | typedef_decl top_list 245 | bad_decl top_list 246 | """ 247 if len(p) > 2: 248 p[0] = ListFromConcat(p[1], p[2]) 249 if self.parse_debug: DumpReduction('top_list', p) 250 251 # Recover from error and continue parsing at the next top match. 252 def p_top_error(self, p): 253 """top_list : error top_list""" 254 p[0] = p[2] 255 256 # Recover from error and continue parsing at the next top match. 257 def p_bad_decl(self, p): 258 """bad_decl : modifiers SYMBOL error '}' ';'""" 259 p[0] = [] 260 261 # 262 # Modifier List 263 # 264 # 265 def p_modifiers(self, p): 266 """modifiers : comments ext_attr_block""" 267 p[0] = ListFromConcat(p[1], p[2]) 268 if self.parse_debug: DumpReduction('modifiers', p) 269 270 # 271 # Scoped name is a name with an optional scope. 272 # 273 # Used for types and namespace names. eg. foo_bar.hello_world, or 274 # foo_bar.hello_world.SomeType. 275 # 276 def p_scoped_name(self, p): 277 """scoped_name : SYMBOL scoped_name_rest""" 278 p[0] = ''.join(p[1:]) 279 if self.parse_debug: DumpReduction('scoped_name', p) 280 281 def p_scoped_name_rest(self, p): 282 """scoped_name_rest : '.' scoped_name 283 |""" 284 p[0] = ''.join(p[1:]) 285 if self.parse_debug: DumpReduction('scoped_name_rest', p) 286 287 # 288 # Type reference 289 # 290 # 291 def p_typeref(self, p): 292 """typeref : scoped_name""" 293 p[0] = p[1] 294 if self.parse_debug: DumpReduction('typeref', p) 295 296 297 # 298 # Comments 299 # 300 # Comments are optional list of C style comment objects. Comments are returned 301 # as a list or None. 302 # 303 def p_comments(self, p): 304 """comments : COMMENT comments 305 | """ 306 if len(p) > 1: 307 child = self.BuildComment('Comment', p, 1) 308 p[0] = ListFromConcat(child, p[2]) 309 if self.parse_debug: DumpReduction('comments', p) 310 else: 311 if self.parse_debug: DumpReduction('no comments', p) 312 313 314 # 315 # Namespace 316 # 317 # A namespace provides a named scope to an enclosed top_list. 318 # 319 def p_namespace(self, p): 320 """namespace : modifiers NAMESPACE namespace_name '{' top_list '}' ';'""" 321 children = ListFromConcat(p[1], p[5]) 322 p[0] = self.BuildNamed('Namespace', p, 3, children) 323 324 # We allow namespace names of the form foo.bar.baz. 325 def p_namespace_name(self, p): 326 """namespace_name : scoped_name""" 327 p[0] = p[1] 328 329 330 # 331 # Dictionary 332 # 333 # A dictionary is a named list of optional and required members. 334 # 335 def p_dictionary_block(self, p): 336 """dictionary_block : modifiers DICTIONARY SYMBOL '{' struct_list '}' ';'""" 337 p[0] = self.BuildNamed('Dictionary', p, 3, ListFromConcat(p[1], p[5])) 338 339 def p_dictionary_errorA(self, p): 340 """dictionary_block : modifiers DICTIONARY error ';'""" 341 p[0] = [] 342 343 def p_dictionary_errorB(self, p): 344 """dictionary_block : modifiers DICTIONARY error '{' struct_list '}' ';'""" 345 p[0] = [] 346 347 # 348 # Callback 349 # 350 # A callback is essentially a single function declaration (outside of an 351 # Interface). 352 # 353 def p_callback_decl(self, p): 354 """callback_decl : modifiers CALLBACK SYMBOL '=' SYMBOL param_list ';'""" 355 children = ListFromConcat(p[1], p[6]) 356 p[0] = self.BuildNamed('Callback', p, 3, children) 357 358 359 # 360 # Inline 361 # 362 # Inline blocks define option code to be emitted based on language tag, 363 # in the form of: 364 # #inline <LANGUAGE> 365 # <CODE> 366 # #endinl 367 # 368 def p_inline(self, p): 369 """inline : modifiers INLINE""" 370 words = p[2].split() 371 name = self.BuildAttribute('NAME', words[1]) 372 lines = p[2].split('\n') 373 value = self.BuildAttribute('VALUE', '\n'.join(lines[1:-1]) + '\n') 374 children = ListFromConcat(name, value, p[1]) 375 p[0] = self.BuildProduction('Inline', p, 2, children) 376 if self.parse_debug: DumpReduction('inline', p) 377 378 # Extended Attributes 379 # 380 # Extended Attributes denote properties which will be applied to a node in the 381 # AST. A list of extended attributes are denoted by a brackets '[' ... ']' 382 # enclosing a comma separated list of extended attributes in the form of: 383 # 384 # Name 385 # Name=HEX | INT | OCT | FLOAT 386 # Name="STRING" 387 # Name=Function(arg ...) 388 # TODO(noelallen) -Not currently supported: 389 # ** Name(arg ...) ... 390 # ** Name=Scope::Value 391 # 392 # Extended Attributes are returned as a list or None. 393 394 def p_ext_attr_block(self, p): 395 """ext_attr_block : '[' ext_attr_list ']' 396 | """ 397 if len(p) > 1: 398 p[0] = p[2] 399 if self.parse_debug: DumpReduction('ext_attr_block', p) 400 else: 401 if self.parse_debug: DumpReduction('no ext_attr_block', p) 402 403 def p_ext_attr_list(self, p): 404 """ext_attr_list : SYMBOL '=' SYMBOL ext_attr_cont 405 | SYMBOL '=' value ext_attr_cont 406 | SYMBOL '=' SYMBOL param_list ext_attr_cont 407 | SYMBOL ext_attr_cont""" 408 # If there are 4 tokens plus a return slot, this must be in the form 409 # SYMBOL = SYMBOL|value ext_attr_cont 410 if len(p) == 5: 411 p[0] = ListFromConcat(self.BuildAttribute(p[1], p[3]), p[4]) 412 # If there are 5 tokens plus a return slot, this must be in the form 413 # SYMBOL = SYMBOL (param_list) ext_attr_cont 414 elif len(p) == 6: 415 member = self.BuildNamed('Member', p, 3, [p[4]]) 416 p[0] = ListFromConcat(self.BuildAttribute(p[1], member), p[5]) 417 # Otherwise, this must be: SYMBOL ext_attr_cont 418 else: 419 p[0] = ListFromConcat(self.BuildAttribute(p[1], 'True'), p[2]) 420 if self.parse_debug: DumpReduction('ext_attribute_list', p) 421 422 def p_ext_attr_list_values(self, p): 423 """ext_attr_list : SYMBOL '=' '(' values ')' ext_attr_cont 424 | SYMBOL '=' '(' symbols ')' ext_attr_cont""" 425 p[0] = ListFromConcat(self.BuildAttribute(p[1], p[4]), p[6]) 426 427 def p_values(self, p): 428 """values : value values_cont""" 429 p[0] = ListFromConcat(p[1], p[2]) 430 431 def p_symbols(self, p): 432 """symbols : SYMBOL symbols_cont""" 433 p[0] = ListFromConcat(p[1], p[2]) 434 435 def p_symbols_cont(self, p): 436 """symbols_cont : ',' SYMBOL symbols_cont 437 | """ 438 if len(p) > 1: p[0] = ListFromConcat(p[2], p[3]) 439 440 def p_values_cont(self, p): 441 """values_cont : ',' value values_cont 442 | """ 443 if len(p) > 1: p[0] = ListFromConcat(p[2], p[3]) 444 445 def p_ext_attr_cont(self, p): 446 """ext_attr_cont : ',' ext_attr_list 447 |""" 448 if len(p) > 1: p[0] = p[2] 449 if self.parse_debug: DumpReduction('ext_attribute_cont', p) 450 451 def p_ext_attr_func(self, p): 452 """ext_attr_list : SYMBOL '(' attr_arg_list ')' ext_attr_cont""" 453 p[0] = ListFromConcat(self.BuildAttribute(p[1] + '()', p[3]), p[5]) 454 if self.parse_debug: DumpReduction('attr_arg_func', p) 455 456 def p_ext_attr_arg_list(self, p): 457 """attr_arg_list : SYMBOL attr_arg_cont 458 | value attr_arg_cont""" 459 p[0] = ListFromConcat(p[1], p[2]) 460 461 def p_attr_arg_cont(self, p): 462 """attr_arg_cont : ',' attr_arg_list 463 | """ 464 if self.parse_debug: DumpReduction('attr_arg_cont', p) 465 if len(p) > 1: p[0] = p[2] 466 467 def p_attr_arg_error(self, p): 468 """attr_arg_cont : error attr_arg_cont""" 469 p[0] = p[2] 470 if self.parse_debug: DumpReduction('attr_arg_error', p) 471 472 473 # 474 # Describe 475 # 476 # A describe block is defined at the top level. It provides a mechanism for 477 # attributing a group of ext_attr to a describe_list. Members of the 478 # describe list are language specific 'Type' declarations 479 # 480 def p_describe_block(self, p): 481 """describe_block : modifiers DESCRIBE '{' describe_list '}' ';'""" 482 children = ListFromConcat(p[1], p[4]) 483 p[0] = self.BuildProduction('Describe', p, 2, children) 484 if self.parse_debug: DumpReduction('describe_block', p) 485 486 # Recover from describe error and continue parsing at the next top match. 487 def p_describe_error(self, p): 488 """describe_list : error describe_list""" 489 p[0] = [] 490 491 def p_describe_list(self, p): 492 """describe_list : modifiers SYMBOL ';' describe_list 493 | modifiers ENUM ';' describe_list 494 | modifiers STRUCT ';' describe_list 495 | modifiers TYPEDEF ';' describe_list 496 | """ 497 if len(p) > 1: 498 Type = self.BuildNamed('Type', p, 2, p[1]) 499 p[0] = ListFromConcat(Type, p[4]) 500 501 # 502 # Constant Values (integer, value) 503 # 504 # Constant values can be found at various levels. A Constant value is returns 505 # as the string value after validated against a FLOAT, HEX, INT, OCT or 506 # STRING pattern as appropriate. 507 # 508 def p_value(self, p): 509 """value : FLOAT 510 | HEX 511 | INT 512 | OCT 513 | STRING""" 514 p[0] = p[1] 515 if self.parse_debug: DumpReduction('value', p) 516 517 def p_value_lshift(self, p): 518 """value : integer LSHIFT INT""" 519 p[0] = "%s << %s" % (p[1], p[3]) 520 if self.parse_debug: DumpReduction('value', p) 521 522 # Integers are numbers which may not be floats used in cases like array sizes. 523 def p_integer(self, p): 524 """integer : HEX 525 | INT 526 | OCT""" 527 p[0] = p[1] 528 if self.parse_debug: DumpReduction('integer', p) 529 530 # 531 # Expression 532 # 533 # A simple arithmetic expression. 534 # 535 precedence = ( 536 ('left','|','&','^'), 537 ('left','LSHIFT','RSHIFT'), 538 ('left','+','-'), 539 ('left','*','/'), 540 ('right','UMINUS','~'), 541 ) 542 543 def p_expression_binop(self, p): 544 """expression : expression LSHIFT expression 545 | expression RSHIFT expression 546 | expression '|' expression 547 | expression '&' expression 548 | expression '^' expression 549 | expression '+' expression 550 | expression '-' expression 551 | expression '*' expression 552 | expression '/' expression""" 553 p[0] = "%s %s %s" % (str(p[1]), str(p[2]), str(p[3])) 554 if self.parse_debug: DumpReduction('expression_binop', p) 555 556 def p_expression_unop(self, p): 557 """expression : '-' expression %prec UMINUS 558 | '~' expression %prec '~'""" 559 p[0] = "%s%s" % (str(p[1]), str(p[2])) 560 if self.parse_debug: DumpReduction('expression_unop', p) 561 562 def p_expression_term(self, p): 563 """expression : '(' expression ')'""" 564 p[0] = "%s%s%s" % (str(p[1]), str(p[2]), str(p[3])) 565 if self.parse_debug: DumpReduction('expression_term', p) 566 567 def p_expression_symbol(self, p): 568 """expression : SYMBOL""" 569 p[0] = p[1] 570 if self.parse_debug: DumpReduction('expression_symbol', p) 571 572 def p_expression_integer(self, p): 573 """expression : integer""" 574 p[0] = p[1] 575 if self.parse_debug: DumpReduction('expression_integer', p) 576 577 # 578 # Array List 579 # 580 # Defined a list of array sizes (if any). 581 # 582 def p_arrays(self, p): 583 """arrays : '[' ']' arrays 584 | '[' integer ']' arrays 585 | """ 586 # If there are 3 tokens plus a return slot it is an unsized array 587 if len(p) == 4: 588 array = self.BuildProduction('Array', p, 1) 589 p[0] = ListFromConcat(array, p[3]) 590 # If there are 4 tokens plus a return slot it is a fixed array 591 elif len(p) == 5: 592 count = self.BuildAttribute('FIXED', p[2]) 593 array = self.BuildProduction('Array', p, 2, [count]) 594 p[0] = ListFromConcat(array, p[4]) 595 # If there is only a return slot, do not fill it for this terminator. 596 elif len(p) == 1: return 597 if self.parse_debug: DumpReduction('arrays', p) 598 599 600 # An identifier is a legal value for a parameter or attribute name. Lots of 601 # existing IDL files use "callback" as a parameter/attribute name, so we allow 602 # a SYMBOL or the CALLBACK keyword. 603 def p_identifier(self, p): 604 """identifier : SYMBOL 605 | CALLBACK""" 606 p[0] = p[1] 607 # Save the line number of the underlying token (otherwise it gets 608 # discarded), since we use it in the productions with an identifier in 609 # them. 610 p.set_lineno(0, p.lineno(1)) 611 612 613 # 614 # Union 615 # 616 # A union allows multiple choices of types for a parameter or member. 617 # 618 619 def p_union_option(self, p): 620 """union_option : modifiers SYMBOL arrays""" 621 typeref = self.BuildAttribute('TYPEREF', p[2]) 622 children = ListFromConcat(p[1], typeref, p[3]) 623 p[0] = self.BuildProduction('Option', p, 2, children) 624 625 def p_union_list(self, p): 626 """union_list : union_option OR union_list 627 | union_option""" 628 if len(p) > 2: 629 p[0] = ListFromConcat(p[1], p[3]) 630 else: 631 p[0] = p[1] 632 633 # 634 # Parameter List 635 # 636 # A parameter list is a collection of arguments which are passed to a 637 # function. 638 # 639 def p_param_list(self, p): 640 """param_list : '(' param_item param_cont ')' 641 | '(' ')' """ 642 if len(p) > 3: 643 args = ListFromConcat(p[2], p[3]) 644 else: 645 args = [] 646 p[0] = self.BuildProduction('Callspec', p, 1, args) 647 if self.parse_debug: DumpReduction('param_list', p) 648 649 def p_param_item(self, p): 650 """param_item : modifiers optional typeref arrays identifier""" 651 typeref = self.BuildAttribute('TYPEREF', p[3]) 652 children = ListFromConcat(p[1], p[2], typeref, p[4]) 653 p[0] = self.BuildNamed('Param', p, 5, children) 654 if self.parse_debug: DumpReduction('param_item', p) 655 656 def p_param_item_union(self, p): 657 """param_item : modifiers optional '(' union_list ')' identifier""" 658 union = self.BuildAttribute('Union', True) 659 children = ListFromConcat(p[1], p[2], p[4], union) 660 p[0] = self.BuildNamed('Param', p, 6, children) 661 if self.parse_debug: DumpReduction('param_item', p) 662 663 def p_optional(self, p): 664 """optional : OPTIONAL 665 | """ 666 if len(p) == 2: 667 p[0] = self.BuildAttribute('OPTIONAL', True) 668 669 670 def p_param_cont(self, p): 671 """param_cont : ',' param_item param_cont 672 | """ 673 if len(p) > 1: 674 p[0] = ListFromConcat(p[2], p[3]) 675 if self.parse_debug: DumpReduction('param_cont', p) 676 677 def p_param_error(self, p): 678 """param_cont : error param_cont""" 679 p[0] = p[2] 680 681 682 # 683 # Typedef 684 # 685 # A typedef creates a new referencable type. The typedef can specify an array 686 # definition as well as a function declaration. 687 # 688 def p_typedef_data(self, p): 689 """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL ';' """ 690 typeref = self.BuildAttribute('TYPEREF', p[3]) 691 children = ListFromConcat(p[1], typeref) 692 p[0] = self.BuildNamed('Typedef', p, 4, children) 693 if self.parse_debug: DumpReduction('typedef_data', p) 694 695 def p_typedef_array(self, p): 696 """typedef_decl : modifiers TYPEDEF SYMBOL arrays SYMBOL ';' """ 697 typeref = self.BuildAttribute('TYPEREF', p[3]) 698 children = ListFromConcat(p[1], typeref, p[4]) 699 p[0] = self.BuildNamed('Typedef', p, 5, children) 700 if self.parse_debug: DumpReduction('typedef_array', p) 701 702 def p_typedef_func(self, p): 703 """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL param_list ';' """ 704 typeref = self.BuildAttribute('TYPEREF', p[3]) 705 children = ListFromConcat(p[1], typeref, p[5]) 706 p[0] = self.BuildNamed('Typedef', p, 4, children) 707 if self.parse_debug: DumpReduction('typedef_func', p) 708 709 # 710 # Enumeration 711 # 712 # An enumeration is a set of named integer constants. An enumeration 713 # is valid type which can be referenced in other definitions. 714 # 715 def p_enum_block(self, p): 716 """enum_block : modifiers ENUM SYMBOL '{' enum_list '}' ';'""" 717 p[0] = self.BuildNamed('Enum', p, 3, ListFromConcat(p[1], p[5])) 718 if self.parse_debug: DumpReduction('enum_block', p) 719 720 # Recover from enum error and continue parsing at the next top match. 721 def p_enum_errorA(self, p): 722 """enum_block : modifiers ENUM error '{' enum_list '}' ';'""" 723 p[0] = [] 724 725 def p_enum_errorB(self, p): 726 """enum_block : modifiers ENUM error ';'""" 727 p[0] = [] 728 729 def p_enum_list(self, p): 730 """enum_list : modifiers SYMBOL '=' expression enum_cont 731 | modifiers SYMBOL enum_cont""" 732 if len(p) > 4: 733 val = self.BuildAttribute('VALUE', p[4]) 734 enum = self.BuildNamed('EnumItem', p, 2, ListFromConcat(val, p[1])) 735 p[0] = ListFromConcat(enum, p[5]) 736 else: 737 enum = self.BuildNamed('EnumItem', p, 2, p[1]) 738 p[0] = ListFromConcat(enum, p[3]) 739 if self.parse_debug: DumpReduction('enum_list', p) 740 741 def p_enum_cont(self, p): 742 """enum_cont : ',' enum_list 743 |""" 744 if len(p) > 1: p[0] = p[2] 745 if self.parse_debug: DumpReduction('enum_cont', p) 746 747 def p_enum_cont_error(self, p): 748 """enum_cont : error enum_cont""" 749 p[0] = p[2] 750 if self.parse_debug: DumpReduction('enum_error', p) 751 752 753 # 754 # Label 755 # 756 # A label is a special kind of enumeration which allows us to go from a 757 # set of labels 758 # 759 def p_label_block(self, p): 760 """label_block : modifiers LABEL SYMBOL '{' label_list '}' ';'""" 761 p[0] = self.BuildNamed('Label', p, 3, ListFromConcat(p[1], p[5])) 762 if self.parse_debug: DumpReduction('label_block', p) 763 764 def p_label_list(self, p): 765 """label_list : modifiers SYMBOL '=' FLOAT label_cont""" 766 val = self.BuildAttribute('VALUE', p[4]) 767 label = self.BuildNamed('LabelItem', p, 2, ListFromConcat(val, p[1])) 768 p[0] = ListFromConcat(label, p[5]) 769 if self.parse_debug: DumpReduction('label_list', p) 770 771 def p_label_cont(self, p): 772 """label_cont : ',' label_list 773 |""" 774 if len(p) > 1: p[0] = p[2] 775 if self.parse_debug: DumpReduction('label_cont', p) 776 777 def p_label_cont_error(self, p): 778 """label_cont : error label_cont""" 779 p[0] = p[2] 780 if self.parse_debug: DumpReduction('label_error', p) 781 782 783 # 784 # Members 785 # 786 # A member attribute or function of a struct or interface. 787 # 788 def p_member_attribute(self, p): 789 """member_attribute : modifiers typeref arrays questionmark identifier""" 790 typeref = self.BuildAttribute('TYPEREF', p[2]) 791 children = ListFromConcat(p[1], typeref, p[3], p[4]) 792 p[0] = self.BuildNamed('Member', p, 5, children) 793 if self.parse_debug: DumpReduction('attribute', p) 794 795 def p_member_attribute_union(self, p): 796 """member_attribute : modifiers '(' union_list ')' questionmark identifier""" 797 union = self.BuildAttribute('Union', True) 798 children = ListFromConcat(p[1], p[3], p[5], union) 799 p[0] = self.BuildNamed('Member', p, 6, children) 800 if self.parse_debug: DumpReduction('attribute', p) 801 802 def p_member_function(self, p): 803 """member_function : modifiers static typeref arrays SYMBOL param_list""" 804 typeref = self.BuildAttribute('TYPEREF', p[3]) 805 children = ListFromConcat(p[1], p[2], typeref, p[4], p[6]) 806 p[0] = self.BuildNamed('Member', p, 5, children) 807 if self.parse_debug: DumpReduction('function', p) 808 809 def p_static(self, p): 810 """static : STATIC 811 | """ 812 if len(p) == 2: 813 p[0] = self.BuildAttribute('STATIC', True) 814 815 def p_questionmark(self, p): 816 """questionmark : '?' 817 | """ 818 if len(p) == 2: 819 p[0] = self.BuildAttribute('OPTIONAL', True) 820 821 # 822 # Interface 823 # 824 # An interface is a named collection of functions. 825 # 826 def p_interface_block(self, p): 827 """interface_block : modifiers INTERFACE SYMBOL '{' interface_list '}' ';'""" 828 p[0] = self.BuildNamed('Interface', p, 3, ListFromConcat(p[1], p[5])) 829 if self.parse_debug: DumpReduction('interface_block', p) 830 831 def p_interface_error(self, p): 832 """interface_block : modifiers INTERFACE error '{' interface_list '}' ';'""" 833 p[0] = [] 834 835 def p_interface_list(self, p): 836 """interface_list : member_function ';' interface_list 837 | """ 838 if len(p) > 1 : 839 p[0] = ListFromConcat(p[1], p[3]) 840 if self.parse_debug: DumpReduction('interface_list', p) 841 842 843 # 844 # Struct 845 # 846 # A struct is a named collection of members which in turn reference other 847 # types. The struct is a referencable type. 848 # 849 def p_struct_block(self, p): 850 """struct_block : modifiers STRUCT SYMBOL '{' struct_list '}' ';'""" 851 children = ListFromConcat(p[1], p[5]) 852 p[0] = self.BuildNamed('Struct', p, 3, children) 853 if self.parse_debug: DumpReduction('struct_block', p) 854 855 # Recover from struct error and continue parsing at the next top match. 856 def p_struct_error(self, p): 857 """enum_block : modifiers STRUCT error '{' struct_list '}' ';'""" 858 p[0] = [] 859 860 def p_struct_list(self, p): 861 """struct_list : member_attribute ';' struct_list 862 | member_function ';' struct_list 863 |""" 864 if len(p) > 1: p[0] = ListFromConcat(p[1], p[3]) 865 866 867 # 868 # Parser Errors 869 # 870 # p_error is called whenever the parser can not find a pattern match for 871 # a set of items from the current state. The p_error function defined here 872 # is triggered logging an error, and parsing recover happens as the 873 # p_<type>_error functions defined above are called. This allows the parser 874 # to continue so as to capture more than one error per file. 875 # 876 def p_error(self, t): 877 filename = self.lexobj.filename 878 self.parse_errors += 1 879 if t: 880 lineno = t.lineno 881 pos = t.lexpos 882 prev = self.yaccobj.symstack[-1] 883 if type(prev) == lex.LexToken: 884 msg = "Unexpected %s after %s." % ( 885 TokenTypeName(t), TokenTypeName(prev)) 886 else: 887 msg = "Unexpected %s." % (t.value) 888 else: 889 lineno = self.last.lineno 890 pos = self.last.lexpos 891 msg = "Unexpected end of file after %s." % TokenTypeName(self.last) 892 self.yaccobj.restart() 893 894 # Attempt to remap the error to a friendlier form 895 if msg in ERROR_REMAP: 896 msg = ERROR_REMAP[msg] 897 898 # Log the error 899 ErrOut.LogLine(filename, lineno, pos, msg) 900 901 def Warn(self, node, msg): 902 WarnOut.LogLine(node.filename, node.lineno, node.pos, msg) 903 self.parse_warnings += 1 904 905 def __init__(self): 906 IDLLexer.__init__(self) 907 self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False, 908 optimize=0, write_tables=0) 909 910 self.build_debug = GetOption('build_debug') 911 self.parse_debug = GetOption('parse_debug') 912 self.token_debug = GetOption('token_debug') 913 self.verbose = GetOption('verbose') 914 self.parse_errors = 0 915 916 # 917 # Tokenizer 918 # 919 # The token function returns the next token provided by IDLLexer for matching 920 # against the leaf paterns. 921 # 922 def token(self): 923 tok = self.lexobj.token() 924 if tok: 925 self.last = tok 926 if self.token_debug: 927 InfoOut.Log("TOKEN %s(%s)" % (tok.type, tok.value)) 928 return tok 929 930 # 931 # BuildProduction 932 # 933 # Production is the set of items sent to a grammar rule resulting in a new 934 # item being returned. 935 # 936 # p - Is the Yacc production object containing the stack of items 937 # index - Index into the production of the name for the item being produced. 938 # cls - The type of item being producted 939 # childlist - The children of the new item 940 def BuildProduction(self, cls, p, index, childlist=None): 941 if not childlist: childlist = [] 942 filename = self.lexobj.filename 943 lineno = p.lineno(index) 944 pos = p.lexpos(index) 945 out = IDLNode(cls, filename, lineno, pos, childlist) 946 if self.build_debug: 947 InfoOut.Log("Building %s" % out) 948 return out 949 950 def BuildNamed(self, cls, p, index, childlist=None): 951 if not childlist: childlist = [] 952 childlist.append(self.BuildAttribute('NAME', p[index])) 953 return self.BuildProduction(cls, p, index, childlist) 954 955 def BuildComment(self, cls, p, index): 956 name = p[index] 957 958 # Remove comment markers 959 lines = [] 960 if name[:2] == '//': 961 # For C++ style, remove any leading whitespace and the '//' marker from 962 # each line. 963 form = 'cc' 964 for line in name.split('\n'): 965 start = line.find('//') 966 lines.append(line[start+2:]) 967 else: 968 # For C style, remove ending '*/'' 969 form = 'c' 970 for line in name[:-2].split('\n'): 971 # Remove characters until start marker for this line '*' if found 972 # otherwise it should be blank. 973 offs = line.find('*') 974 if offs >= 0: 975 line = line[offs + 1:].rstrip() 976 else: 977 line = '' 978 lines.append(line) 979 name = '\n'.join(lines) 980 981 childlist = [self.BuildAttribute('NAME', name), 982 self.BuildAttribute('FORM', form)] 983 return self.BuildProduction(cls, p, index, childlist) 984 985 # 986 # BuildAttribute 987 # 988 # An ExtendedAttribute is a special production that results in a property 989 # which is applied to the adjacent item. Attributes have no children and 990 # instead represent key/value pairs. 991 # 992 def BuildAttribute(self, key, val): 993 return IDLAttribute(key, val) 994 995 996 # 997 # ParseData 998 # 999 # Attempts to parse the current data loaded in the lexer. 1000 # 1001 def ParseData(self, data, filename='<Internal>'): 1002 self.SetData(filename, data) 1003 try: 1004 self.parse_errors = 0 1005 self.parse_warnings = 0 1006 return self.yaccobj.parse(lexer=self) 1007 1008 except lex.LexError as le: 1009 ErrOut.Log(str(le)) 1010 return [] 1011 1012 # 1013 # ParseFile 1014 # 1015 # Loads a new file into the lexer and attemps to parse it. 1016 # 1017 def ParseFile(self, filename): 1018 date = time.ctime(os.path.getmtime(filename)) 1019 data = open(filename).read() 1020 if self.verbose: 1021 InfoOut.Log("Parsing %s" % filename) 1022 try: 1023 out = self.ParseData(data, filename) 1024 1025 # If we have a src root specified, remove it from the path 1026 srcroot = GetOption('srcroot') 1027 if srcroot and filename.find(srcroot) == 0: 1028 filename = filename[len(srcroot) + 1:] 1029 filenode = IDLFile(filename, out, self.parse_errors + self.lex_errors) 1030 filenode.SetProperty('DATETIME', date) 1031 return filenode 1032 1033 except Exception as e: 1034 ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos, 1035 'Internal parsing error - %s.' % str(e)) 1036 raise 1037 1038 1039 1040 # 1041 # Flatten Tree 1042 # 1043 # Flattens the tree of IDLNodes for use in testing. 1044 # 1045 def FlattenTree(node): 1046 add_self = False 1047 out = [] 1048 for child in node.GetChildren(): 1049 if child.IsA('Comment'): 1050 add_self = True 1051 else: 1052 out.extend(FlattenTree(child)) 1053 1054 if add_self: 1055 out = [str(node)] + out 1056 return out 1057 1058 1059 def TestErrors(filename, filenode): 1060 nodelist = filenode.GetChildren() 1061 1062 lexer = IDLLexer() 1063 data = open(filename).read() 1064 lexer.SetData(filename, data) 1065 1066 pass_comments = [] 1067 fail_comments = [] 1068 while True: 1069 tok = lexer.lexobj.token() 1070 if tok == None: break 1071 if tok.type == 'COMMENT': 1072 args = tok.value[3:-3].split() 1073 if args[0] == 'OK': 1074 pass_comments.append((tok.lineno, ' '.join(args[1:]))) 1075 else: 1076 if args[0] == 'FAIL': 1077 fail_comments.append((tok.lineno, ' '.join(args[1:]))) 1078 obj_list = [] 1079 for node in nodelist: 1080 obj_list.extend(FlattenTree(node)) 1081 1082 errors = 0 1083 1084 # 1085 # Check for expected successes 1086 # 1087 obj_cnt = len(obj_list) 1088 pass_cnt = len(pass_comments) 1089 if obj_cnt != pass_cnt: 1090 InfoOut.Log("Mismatched pass (%d) vs. nodes built (%d)." 1091 % (pass_cnt, obj_cnt)) 1092 InfoOut.Log("PASS: %s" % [x[1] for x in pass_comments]) 1093 InfoOut.Log("OBJS: %s" % obj_list) 1094 errors += 1 1095 if pass_cnt > obj_cnt: pass_cnt = obj_cnt 1096 1097 for i in range(pass_cnt): 1098 line, comment = pass_comments[i] 1099 if obj_list[i] != comment: 1100 ErrOut.LogLine(filename, line, None, "OBJ %s : EXPECTED %s\n" % 1101 (obj_list[i], comment)) 1102 errors += 1 1103 1104 # 1105 # Check for expected errors 1106 # 1107 err_list = ErrOut.DrainLog() 1108 err_cnt = len(err_list) 1109 fail_cnt = len(fail_comments) 1110 if err_cnt != fail_cnt: 1111 InfoOut.Log("Mismatched fail (%d) vs. errors seen (%d)." 1112 % (fail_cnt, err_cnt)) 1113 InfoOut.Log("FAIL: %s" % [x[1] for x in fail_comments]) 1114 InfoOut.Log("ERRS: %s" % err_list) 1115 errors += 1 1116 if fail_cnt > err_cnt: fail_cnt = err_cnt 1117 1118 for i in range(fail_cnt): 1119 line, comment = fail_comments[i] 1120 err = err_list[i].strip() 1121 1122 if err_list[i] != comment: 1123 ErrOut.Log("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % ( 1124 filename, line, err_list[i], comment)) 1125 errors += 1 1126 1127 # Clear the error list for the next run 1128 err_list = [] 1129 return errors 1130 1131 1132 def TestFile(parser, filename): 1133 # Capture errors instead of reporting them so we can compare them 1134 # with the expected errors. 1135 ErrOut.SetConsole(False) 1136 ErrOut.SetCapture(True) 1137 1138 filenode = parser.ParseFile(filename) 1139 1140 # Renable output 1141 ErrOut.SetConsole(True) 1142 ErrOut.SetCapture(False) 1143 1144 # Compare captured errors 1145 return TestErrors(filename, filenode) 1146 1147 1148 def TestErrorFiles(filter): 1149 idldir = os.path.split(sys.argv[0])[0] 1150 idldir = os.path.join(idldir, 'test_parser', '*.idl') 1151 filenames = glob.glob(idldir) 1152 parser = IDLParser() 1153 total_errs = 0 1154 for filename in filenames: 1155 if filter and filename not in filter: continue 1156 errs = TestFile(parser, filename) 1157 if errs: 1158 ErrOut.Log("%s test failed with %d error(s)." % (filename, errs)) 1159 total_errs += errs 1160 1161 if total_errs: 1162 ErrOut.Log("Failed parsing test.") 1163 else: 1164 InfoOut.Log("Passed parsing test.") 1165 return total_errs 1166 1167 1168 def TestNamespaceFiles(filter): 1169 idldir = os.path.split(sys.argv[0])[0] 1170 idldir = os.path.join(idldir, 'test_namespace', '*.idl') 1171 filenames = glob.glob(idldir) 1172 testnames = [] 1173 1174 for filename in filenames: 1175 if filter and filename not in filter: continue 1176 testnames.append(filename) 1177 1178 # If we have no files to test, then skip this test 1179 if not testnames: 1180 InfoOut.Log('No files to test for namespace.') 1181 return 0 1182 1183 InfoOut.SetConsole(False) 1184 ast = ParseFiles(testnames) 1185 InfoOut.SetConsole(True) 1186 1187 errs = ast.GetProperty('ERRORS') 1188 if errs: 1189 ErrOut.Log("Failed namespace test.") 1190 else: 1191 InfoOut.Log("Passed namespace test.") 1192 return errs 1193 1194 1195 1196 def FindVersionError(releases, node): 1197 err_cnt = 0 1198 if node.IsA('Interface', 'Struct'): 1199 comment_list = [] 1200 comment = node.GetOneOf('Comment') 1201 if comment and comment.GetName()[:4] == 'REL:': 1202 comment_list = comment.GetName()[5:].strip().split(' ') 1203 1204 first_list = [node.first_release[rel] for rel in releases] 1205 first_list = sorted(set(first_list)) 1206 if first_list != comment_list: 1207 node.Error("Mismatch in releases: %s vs %s." % ( 1208 comment_list, first_list)) 1209 err_cnt += 1 1210 1211 for child in node.GetChildren(): 1212 err_cnt += FindVersionError(releases, child) 1213 return err_cnt 1214 1215 1216 def TestVersionFiles(filter): 1217 idldir = os.path.split(sys.argv[0])[0] 1218 idldir = os.path.join(idldir, 'test_version', '*.idl') 1219 filenames = glob.glob(idldir) 1220 testnames = [] 1221 1222 for filename in filenames: 1223 if filter and filename not in filter: continue 1224 testnames.append(filename) 1225 1226 # If we have no files to test, then skip this test 1227 if not testnames: 1228 InfoOut.Log('No files to test for version.') 1229 return 0 1230 1231 ast = ParseFiles(testnames) 1232 errs = FindVersionError(ast.releases, ast) 1233 errs += ast.errors 1234 1235 if errs: 1236 ErrOut.Log("Failed version test.") 1237 else: 1238 InfoOut.Log("Passed version test.") 1239 return errs 1240 1241 1242 default_dirs = ['.', 'trusted', 'dev', 'private'] 1243 def ParseFiles(filenames): 1244 parser = IDLParser() 1245 filenodes = [] 1246 1247 if not filenames: 1248 filenames = [] 1249 srcroot = GetOption('srcroot') 1250 dirs = default_dirs 1251 if GetOption('include_private'): 1252 dirs += ['private'] 1253 for dirname in dirs: 1254 srcdir = os.path.join(srcroot, dirname, '*.idl') 1255 srcdir = os.path.normpath(srcdir) 1256 filenames += sorted(glob.glob(srcdir)) 1257 1258 if not filenames: 1259 ErrOut.Log('No sources provided.') 1260 1261 for filename in filenames: 1262 filenode = parser.ParseFile(filename) 1263 filenodes.append(filenode) 1264 1265 ast = IDLAst(filenodes) 1266 if GetOption('dump_tree'): ast.Dump(0) 1267 1268 Lint(ast) 1269 return ast 1270 1271 1272 def Main(args): 1273 filenames = ParseOptions(args) 1274 1275 # If testing... 1276 if GetOption('test'): 1277 errs = TestErrorFiles(filenames) 1278 errs = TestNamespaceFiles(filenames) 1279 errs = TestVersionFiles(filenames) 1280 if errs: 1281 ErrOut.Log("Parser failed with %d errors." % errs) 1282 return -1 1283 return 0 1284 1285 # Otherwise, build the AST 1286 ast = ParseFiles(filenames) 1287 errs = ast.GetProperty('ERRORS') 1288 if errs: 1289 ErrOut.Log('Found %d error(s).' % errs); 1290 InfoOut.Log("%d files processed." % len(filenames)) 1291 return errs 1292 1293 1294 if __name__ == '__main__': 1295 sys.exit(Main(sys.argv[1:])) 1296 1297
