1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """ Parser for PPAPI IDL """ 7 8 # 9 # IDL Parser 10 # 11 # The parser is uses the PLY yacc library to build a set of parsing rules based 12 # on WebIDL. 13 # 14 # WebIDL, and WebIDL regular expressions can be found at: 15 # http://dev.w3.org/2006/webapi/WebIDL/ 16 # PLY can be found at: 17 # http://www.dabeaz.com/ply/ 18 # 19 # The parser generates a tree by recursively matching sets of items against 20 # defined patterns. When a match is made, that set of items is reduced 21 # to a new item. The new item can provide a match for parent patterns. 22 # In this way an AST is built (reduced) depth first. 23 24 25 import getopt 26 import glob 27 import os.path 28 import re 29 import sys 30 import time 31 32 from idl_ast import IDLAst 33 from idl_log import ErrOut, InfoOut, WarnOut 34 from idl_lexer import IDLLexer 35 from idl_node import IDLAttribute, IDLFile, IDLNode 36 from idl_option import GetOption, Option, ParseOptions 37 from idl_lint import Lint 38 from idl_visitor import IDLVisitor 39 40 from ply import lex 41 from ply import yacc 42 43 Option('build_debug', 'Debug tree building.') 44 Option('parse_debug', 'Debug parse reduction steps.') 45 Option('token_debug', 'Debug token generation.') 46 Option('dump_tree', 'Dump the tree.') 47 Option('srcroot', 'Working directory.', default=os.path.join('..', 'api')) 48 Option('include_private', 'Include private IDL directory in default API paths.') 49 50 # 51 # ERROR_REMAP 52 # 53 # Maps the standard error formula into a more friendly error message. 54 # 55 ERROR_REMAP = { 56 'Unexpected ")" after "(".' : 'Empty argument list.', 57 'Unexpected ")" after ",".' : 'Missing argument.', 58 'Unexpected "}" after ",".' : 'Trailing comma in block.', 59 'Unexpected "}" after "{".' : 'Unexpected empty block.', 60 'Unexpected comment after "}".' : 'Unexpected trailing comment.', 61 'Unexpected "{" after keyword "enum".' : 'Enum missing name.', 62 'Unexpected "{" after keyword "struct".' : 'Struct missing name.', 63 'Unexpected "{" after keyword "interface".' : 'Interface missing name.', 64 } 65 66 # DumpReduction 67 # 68 # Prints out the set of items which matched a particular pattern and the 69 # new item or set it was reduced to. 70 def DumpReduction(cls, p): 71 if p[0] is None: 72 InfoOut.Log("OBJ: %s(%d) - None\n" % (cls, len(p))) 73 InfoOut.Log(" [%s]\n" % [str(x) for x in p[1:]]) 74 else: 75 out = "" 76 for index in range(len(p) - 1): 77 out += " >%s< " % str(p[index + 1]) 78 InfoOut.Log("OBJ: %s(%d) - %s : %s\n" % (cls, len(p), str(p[0]), out)) 79 80 81 # CopyToList 82 # 83 # Takes an input item, list, or None, and returns a new list of that set. 84 def CopyToList(item): 85 # If the item is 'Empty' make it an empty list 86 if not item: item = [] 87 88 # If the item is not a list 89 if type(item) is not type([]): item = [item] 90 91 # Make a copy we can modify 92 return list(item) 93 94 95 96 # ListFromConcat 97 # 98 # Generate a new List by joining of two sets of inputs which can be an 99 # individual item, a list of items, or None. 100 def ListFromConcat(*items): 101 itemsout = [] 102 for item in items: 103 itemlist = CopyToList(item) 104 itemsout.extend(itemlist) 105 106 return itemsout 107 108 109 # TokenTypeName 110 # 111 # Generate a string which has the type and value of the token. 112 def TokenTypeName(t): 113 if t.type == 'SYMBOL': return 'symbol %s' % t.value 114 if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']: 115 return 'value %s' % t.value 116 if t.type == 'STRING' : return 'string "%s"' % t.value 117 if t.type == 'COMMENT' : return 'comment' 118 if t.type == t.value: return '"%s"' % t.value 119 return 'keyword "%s"' % t.value 120 121 122 # 123 # IDL Parser 124 # 125 # The Parser inherits the from the Lexer to provide PLY with the tokenizing 126 # definitions. Parsing patterns are encoded as function where p_<name> is 127 # is called any time a patern matching the function documentation is found. 128 # Paterns are expressed in the form of: 129 # """ <new item> : <item> .... 130 # | <item> ....""" 131 # 132 # Where new item is the result of a match against one or more sets of items 133 # separated by the "|". 134 # 135 # The function is called with an object 'p' where p[0] is the output object 136 # and p[n] is the set of inputs for positive values of 'n'. Len(p) can be 137 # used to distinguish between multiple item sets in the pattern. 138 # 139 # For more details on parsing refer to the PLY documentation at 140 # http://www.dabeaz.com/ply/ 141 # 142 # 143 # The parser uses the following conventions: 144 # a <type>_block defines a block of <type> definitions in the form of: 145 # [comment] [ext_attr_block] <type> <name> '{' <type>_list '}' ';' 146 # A block is reduced by returning an object of <type> with a name of <name> 147 # which in turn has <type>_list as children. 148 # 149 # A [comment] is a optional C style comment block enclosed in /* ... */ which 150 # is appended to the adjacent node as a child. 151 # 152 # A [ext_attr_block] is an optional list of Extended Attributes which is 153 # appended to the adjacent node as a child. 154 # 155 # a <type>_list defines a list of <type> items which will be passed as a 156 # list of children to the parent pattern. A list is in the form of: 157 # [comment] [ext_attr_block] <...DEF...> ';' <type>_list | (empty) 158 # or 159 # [comment] [ext_attr_block] <...DEF...> <type>_cont 160 # 161 # In the first form, the list is reduced recursively, where the right side 162 # <type>_list is first reduced then joined with pattern currently being 163 # matched. The list is terminated with the (empty) pattern is matched. 164 # 165 # In the second form the list is reduced recursively, where the right side 166 # <type>_cont is first reduced then joined with the pattern currently being 167 # matched. The type_<cont> is in the form of: 168 # ',' <type>_list | (empty) 169 # The <type>_cont form is used to consume the ',' which only occurs when 170 # there is more than one object in the list. The <type>_cont also provides 171 # the terminating (empty) definition. 172 # 173 174 175 class IDLParser(IDLLexer): 176 # TOP 177 # 178 # This pattern defines the top of the parse tree. The parse tree is in the 179 # the form of: 180 # 181 # top 182 # *modifiers 183 # *comments 184 # *ext_attr_block 185 # ext_attr_list 186 # attr_arg_list 187 # *integer, value 188 # *param_list 189 # *typeref 190 # 191 # top_list 192 # describe_block 193 # describe_list 194 # enum_block 195 # enum_item 196 # interface_block 197 # member 198 # label_block 199 # label_item 200 # struct_block 201 # member 202 # typedef_decl 203 # typedef_data 204 # typedef_func 205 # 206 # (* sub matches found at multiple levels and are not truly children of top) 207 # 208 # We force all input files to start with two comments. The first comment is a 209 # Copyright notice followed by a set of file wide Extended Attributes, followed 210 # by the file comment and finally by file level patterns. 211 # 212 # Find the Copyright, File comment, and optional file wide attributes. We 213 # use a match with COMMENT instead of comments to force the token to be 214 # present. The extended attributes and the top_list become siblings which 215 # in turn are children of the file object created from the results of top. 216 def p_top(self, p): 217 """top : COMMENT COMMENT ext_attr_block top_list""" 218 219 Copyright = self.BuildComment('Copyright', p, 1) 220 Filedoc = self.BuildComment('Comment', p, 2) 221 222 p[0] = ListFromConcat(Copyright, Filedoc, p[3], p[4]) 223 if self.parse_debug: DumpReduction('top', p) 224 225 def p_top_short(self, p): 226 """top : COMMENT ext_attr_block top_list""" 227 Copyright = self.BuildComment('Copyright', p, 1) 228 Filedoc = IDLNode('Comment', self.lexobj.filename, p.lineno(2)-1, 229 p.lexpos(2)-1, [self.BuildAttribute('NAME', ''), 230 self.BuildAttribute('FORM', 'cc')]) 231 p[0] = ListFromConcat(Copyright, Filedoc, p[2], p[3]) 232 if self.parse_debug: DumpReduction('top', p) 233 234 # Build a list of top level items. 235 def p_top_list(self, p): 236 """top_list : callback_decl top_list 237 | describe_block top_list 238 | dictionary_block top_list 239 | enum_block top_list 240 | inline top_list 241 | interface_block top_list 242 | label_block top_list 243 | namespace top_list 244 | struct_block top_list 245 | typedef_decl top_list 246 | bad_decl top_list 247 | """ 248 if len(p) > 2: 249 p[0] = ListFromConcat(p[1], p[2]) 250 if self.parse_debug: DumpReduction('top_list', p) 251 252 # Recover from error and continue parsing at the next top match. 253 def p_top_error(self, p): 254 """top_list : error top_list""" 255 p[0] = p[2] 256 257 # Recover from error and continue parsing at the next top match. 258 def p_bad_decl(self, p): 259 """bad_decl : modifiers SYMBOL error '}' ';'""" 260 p[0] = [] 261 262 # 263 # Modifier List 264 # 265 # 266 def p_modifiers(self, p): 267 """modifiers : comments ext_attr_block""" 268 p[0] = ListFromConcat(p[1], p[2]) 269 if self.parse_debug: DumpReduction('modifiers', p) 270 271 # 272 # Comments 273 # 274 # Comments are optional list of C style comment objects. Comments are returned 275 # as a list or None. 276 # 277 def p_comments(self, p): 278 """comments : COMMENT comments 279 | """ 280 if len(p) > 1: 281 child = self.BuildComment('Comment', p, 1) 282 p[0] = ListFromConcat(child, p[2]) 283 if self.parse_debug: DumpReduction('comments', p) 284 else: 285 if self.parse_debug: DumpReduction('no comments', p) 286 287 288 # 289 # Namespace 290 # 291 # A namespace provides a named scope to an enclosed top_list. 292 # 293 def p_namespace(self, p): 294 """namespace : modifiers NAMESPACE namespace_name '{' top_list '}' ';'""" 295 children = ListFromConcat(p[1], p[5]) 296 p[0] = self.BuildNamed('Namespace', p, 3, children) 297 298 # We allow namespace names of the form foo.bar.baz. 299 def p_namespace_name(self, p): 300 """namespace_name : SYMBOL 301 | SYMBOL '.' namespace_name""" 302 p[0] = "".join(p[1:]) 303 304 305 # 306 # Dictionary 307 # 308 # A dictionary is a named list of optional and required members. 309 # 310 def p_dictionary_block(self, p): 311 """dictionary_block : modifiers DICTIONARY SYMBOL '{' struct_list '}' ';'""" 312 p[0] = self.BuildNamed('Dictionary', p, 3, ListFromConcat(p[1], p[5])) 313 314 # 315 # Callback 316 # 317 # A callback is essentially a single function declaration (outside of an 318 # Interface). 319 # 320 def p_callback_decl(self, p): 321 """callback_decl : modifiers CALLBACK SYMBOL '=' SYMBOL param_list ';'""" 322 children = ListFromConcat(p[1], p[6]) 323 p[0] = self.BuildNamed('Callback', p, 3, children) 324 325 326 # 327 # Inline 328 # 329 # Inline blocks define option code to be emitted based on language tag, 330 # in the form of: 331 # #inline <LANGUAGE> 332 # <CODE> 333 # #endinl 334 # 335 def p_inline(self, p): 336 """inline : modifiers INLINE""" 337 words = p[2].split() 338 name = self.BuildAttribute('NAME', words[1]) 339 lines = p[2].split('\n') 340 value = self.BuildAttribute('VALUE', '\n'.join(lines[1:-1]) + '\n') 341 children = ListFromConcat(name, value, p[1]) 342 p[0] = self.BuildProduction('Inline', p, 2, children) 343 if self.parse_debug: DumpReduction('inline', p) 344 345 # Extended Attributes 346 # 347 # Extended Attributes denote properties which will be applied to a node in the 348 # AST. A list of extended attributes are denoted by a brackets '[' ... ']' 349 # enclosing a comma separated list of extended attributes in the form of: 350 # 351 # Name 352 # Name=HEX | INT | OCT | FLOAT 353 # Name="STRING" 354 # Name=Function(arg ...) 355 # TODO(noelallen) -Not currently supported: 356 # ** Name(arg ...) ... 357 # ** Name=Scope::Value 358 # 359 # Extended Attributes are returned as a list or None. 360 361 def p_ext_attr_block(self, p): 362 """ext_attr_block : '[' ext_attr_list ']' 363 | """ 364 if len(p) > 1: 365 p[0] = p[2] 366 if self.parse_debug: DumpReduction('ext_attr_block', p) 367 else: 368 if self.parse_debug: DumpReduction('no ext_attr_block', p) 369 370 def p_ext_attr_list(self, p): 371 """ext_attr_list : SYMBOL '=' SYMBOL ext_attr_cont 372 | SYMBOL '=' value ext_attr_cont 373 | SYMBOL '=' SYMBOL param_list ext_attr_cont 374 | SYMBOL ext_attr_cont""" 375 # If there are 4 tokens plus a return slot, this must be in the form 376 # SYMBOL = SYMBOL|value ext_attr_cont 377 if len(p) == 5: 378 p[0] = ListFromConcat(self.BuildAttribute(p[1], p[3]), p[4]) 379 # If there are 5 tokens plus a return slot, this must be in the form 380 # SYMBOL = SYMBOL (param_list) ext_attr_cont 381 elif len(p) == 6: 382 member = self.BuildNamed('Member', p, 3, [p[4]]) 383 p[0] = ListFromConcat(self.BuildAttribute(p[1], member), p[5]) 384 # Otherwise, this must be: SYMBOL ext_attr_cont 385 else: 386 p[0] = ListFromConcat(self.BuildAttribute(p[1], 'True'), p[2]) 387 if self.parse_debug: DumpReduction('ext_attribute_list', p) 388 389 def p_ext_attr_list_values(self, p): 390 """ext_attr_list : SYMBOL '=' '(' values ')' ext_attr_cont 391 | SYMBOL '=' '(' symbols ')' ext_attr_cont""" 392 p[0] = ListFromConcat(self.BuildAttribute(p[1], p[4]), p[6]) 393 394 def p_values(self, p): 395 """values : value values_cont""" 396 p[0] = ListFromConcat(p[1], p[2]) 397 398 def p_symbols(self, p): 399 """symbols : SYMBOL symbols_cont""" 400 p[0] = ListFromConcat(p[1], p[2]) 401 402 def p_symbols_cont(self, p): 403 """symbols_cont : ',' SYMBOL symbols_cont 404 | """ 405 if len(p) > 1: p[0] = ListFromConcat(p[2], p[3]) 406 407 def p_values_cont(self, p): 408 """values_cont : ',' value values_cont 409 | """ 410 if len(p) > 1: p[0] = ListFromConcat(p[2], p[3]) 411 412 def p_ext_attr_cont(self, p): 413 """ext_attr_cont : ',' ext_attr_list 414 |""" 415 if len(p) > 1: p[0] = p[2] 416 if self.parse_debug: DumpReduction('ext_attribute_cont', p) 417 418 def p_ext_attr_func(self, p): 419 """ext_attr_list : SYMBOL '(' attr_arg_list ')' ext_attr_cont""" 420 p[0] = ListFromConcat(self.BuildAttribute(p[1] + '()', p[3]), p[5]) 421 if self.parse_debug: DumpReduction('attr_arg_func', p) 422 423 def p_ext_attr_arg_list(self, p): 424 """attr_arg_list : SYMBOL attr_arg_cont 425 | value attr_arg_cont""" 426 p[0] = ListFromConcat(p[1], p[2]) 427 428 def p_attr_arg_cont(self, p): 429 """attr_arg_cont : ',' attr_arg_list 430 | """ 431 if self.parse_debug: DumpReduction('attr_arg_cont', p) 432 if len(p) > 1: p[0] = p[2] 433 434 def p_attr_arg_error(self, p): 435 """attr_arg_cont : error attr_arg_cont""" 436 p[0] = p[2] 437 if self.parse_debug: DumpReduction('attr_arg_error', p) 438 439 440 # 441 # Describe 442 # 443 # A describe block is defined at the top level. It provides a mechanism for 444 # attributing a group of ext_attr to a describe_list. Members of the 445 # describe list are language specific 'Type' declarations 446 # 447 def p_describe_block(self, p): 448 """describe_block : modifiers DESCRIBE '{' describe_list '}' ';'""" 449 children = ListFromConcat(p[1], p[4]) 450 p[0] = self.BuildProduction('Describe', p, 2, children) 451 if self.parse_debug: DumpReduction('describe_block', p) 452 453 # Recover from describe error and continue parsing at the next top match. 454 def p_describe_error(self, p): 455 """describe_list : error describe_list""" 456 p[0] = [] 457 458 def p_describe_list(self, p): 459 """describe_list : modifiers SYMBOL ';' describe_list 460 | modifiers ENUM ';' describe_list 461 | modifiers STRUCT ';' describe_list 462 | modifiers TYPEDEF ';' describe_list 463 | """ 464 if len(p) > 1: 465 Type = self.BuildNamed('Type', p, 2, p[1]) 466 p[0] = ListFromConcat(Type, p[4]) 467 468 # 469 # Constant Values (integer, value) 470 # 471 # Constant values can be found at various levels. A Constant value is returns 472 # as the string value after validated against a FLOAT, HEX, INT, OCT or 473 # STRING pattern as appropriate. 474 # 475 def p_value(self, p): 476 """value : FLOAT 477 | HEX 478 | INT 479 | OCT 480 | STRING""" 481 p[0] = p[1] 482 if self.parse_debug: DumpReduction('value', p) 483 484 def p_value_lshift(self, p): 485 """value : integer LSHIFT INT""" 486 p[0] = "%s << %s" % (p[1], p[3]) 487 if self.parse_debug: DumpReduction('value', p) 488 489 # Integers are numbers which may not be floats used in cases like array sizes. 490 def p_integer(self, p): 491 """integer : HEX 492 | INT 493 | OCT""" 494 p[0] = p[1] 495 if self.parse_debug: DumpReduction('integer', p) 496 497 # 498 # Expression 499 # 500 # A simple arithmetic expression. 501 # 502 precedence = ( 503 ('left','|','&','^'), 504 ('left','LSHIFT','RSHIFT'), 505 ('left','+','-'), 506 ('left','*','/'), 507 ('right','UMINUS','~'), 508 ) 509 510 def p_expression_binop(self, p): 511 """expression : expression LSHIFT expression 512 | expression RSHIFT expression 513 | expression '|' expression 514 | expression '&' expression 515 | expression '^' expression 516 | expression '+' expression 517 | expression '-' expression 518 | expression '*' expression 519 | expression '/' expression""" 520 p[0] = "%s %s %s" % (str(p[1]), str(p[2]), str(p[3])) 521 if self.parse_debug: DumpReduction('expression_binop', p) 522 523 def p_expression_unop(self, p): 524 """expression : '-' expression %prec UMINUS 525 | '~' expression %prec '~'""" 526 p[0] = "%s%s" % (str(p[1]), str(p[2])) 527 if self.parse_debug: DumpReduction('expression_unop', p) 528 529 def p_expression_term(self, p): 530 "expression : '(' expression ')'" 531 p[0] = "%s%s%s" % (str(p[1]), str(p[2]), str(p[3])) 532 if self.parse_debug: DumpReduction('expression_term', p) 533 534 def p_expression_symbol(self, p): 535 "expression : SYMBOL" 536 p[0] = p[1] 537 if self.parse_debug: DumpReduction('expression_symbol', p) 538 539 def p_expression_integer(self, p): 540 "expression : integer" 541 p[0] = p[1] 542 if self.parse_debug: DumpReduction('expression_integer', p) 543 544 # 545 # Array List 546 # 547 # Defined a list of array sizes (if any). 548 # 549 def p_arrays(self, p): 550 """arrays : '[' ']' arrays 551 | '[' integer ']' arrays 552 | """ 553 # If there are 3 tokens plus a return slot it is an unsized array 554 if len(p) == 4: 555 array = self.BuildProduction('Array', p, 1) 556 p[0] = ListFromConcat(array, p[3]) 557 # If there are 4 tokens plus a return slot it is a fixed array 558 elif len(p) == 5: 559 count = self.BuildAttribute('FIXED', p[2]) 560 array = self.BuildProduction('Array', p, 2, [count]) 561 p[0] = ListFromConcat(array, p[4]) 562 # If there is only a return slot, do not fill it for this terminator. 563 elif len(p) == 1: return 564 if self.parse_debug: DumpReduction('arrays', p) 565 566 567 # An identifier is a legal value for a parameter or attribute name. Lots of 568 # existing IDL files use "callback" as a parameter/attribute name, so we allow 569 # a SYMBOL or the CALLBACK keyword. 570 def p_identifier(self, p): 571 """identifier : SYMBOL 572 | CALLBACK""" 573 p[0] = p[1] 574 # Save the line number of the underlying token (otherwise it gets 575 # discarded), since we use it in the productions with an identifier in 576 # them. 577 p.set_lineno(0, p.lineno(1)) 578 579 # 580 # Parameter List 581 # 582 # A parameter list is a collection of arguments which are passed to a 583 # function. 584 # 585 def p_param_list(self, p): 586 """param_list : '(' param_item param_cont ')' 587 | '(' ')' """ 588 if len(p) > 3: 589 args = ListFromConcat(p[2], p[3]) 590 else: 591 args = [] 592 p[0] = self.BuildProduction('Callspec', p, 1, args) 593 if self.parse_debug: DumpReduction('param_list', p) 594 595 def p_param_item(self, p): 596 """param_item : modifiers optional SYMBOL arrays identifier""" 597 typeref = self.BuildAttribute('TYPEREF', p[3]) 598 children = ListFromConcat(p[1], p[2], typeref, p[4]) 599 p[0] = self.BuildNamed('Param', p, 5, children) 600 if self.parse_debug: DumpReduction('param_item', p) 601 602 def p_optional(self, p): 603 """optional : OPTIONAL 604 | """ 605 if len(p) == 2: 606 p[0] = self.BuildAttribute('OPTIONAL', True) 607 608 609 def p_param_cont(self, p): 610 """param_cont : ',' param_item param_cont 611 | """ 612 if len(p) > 1: 613 p[0] = ListFromConcat(p[2], p[3]) 614 if self.parse_debug: DumpReduction('param_cont', p) 615 616 def p_param_error(self, p): 617 """param_cont : error param_cont""" 618 p[0] = p[2] 619 620 621 # 622 # Typedef 623 # 624 # A typedef creates a new referencable type. The typedef can specify an array 625 # definition as well as a function declaration. 626 # 627 def p_typedef_data(self, p): 628 """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL ';' """ 629 typeref = self.BuildAttribute('TYPEREF', p[3]) 630 children = ListFromConcat(p[1], typeref) 631 p[0] = self.BuildNamed('Typedef', p, 4, children) 632 if self.parse_debug: DumpReduction('typedef_data', p) 633 634 def p_typedef_array(self, p): 635 """typedef_decl : modifiers TYPEDEF SYMBOL arrays SYMBOL ';' """ 636 typeref = self.BuildAttribute('TYPEREF', p[3]) 637 children = ListFromConcat(p[1], typeref, p[4]) 638 p[0] = self.BuildNamed('Typedef', p, 5, children) 639 if self.parse_debug: DumpReduction('typedef_array', p) 640 641 def p_typedef_func(self, p): 642 """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL param_list ';' """ 643 typeref = self.BuildAttribute('TYPEREF', p[3]) 644 children = ListFromConcat(p[1], typeref, p[5]) 645 p[0] = self.BuildNamed('Typedef', p, 4, children) 646 if self.parse_debug: DumpReduction('typedef_func', p) 647 648 # 649 # Enumeration 650 # 651 # An enumeration is a set of named integer constants. An enumeration 652 # is valid type which can be referenced in other definitions. 653 # 654 def p_enum_block(self, p): 655 """enum_block : modifiers ENUM SYMBOL '{' enum_list '}' ';'""" 656 p[0] = self.BuildNamed('Enum', p, 3, ListFromConcat(p[1], p[5])) 657 if self.parse_debug: DumpReduction('enum_block', p) 658 659 # Recover from enum error and continue parsing at the next top match. 660 def p_enum_errorA(self, p): 661 """enum_block : modifiers ENUM error '{' enum_list '}' ';'""" 662 p[0] = [] 663 664 def p_enum_errorB(self, p): 665 """enum_block : modifiers ENUM error ';'""" 666 p[0] = [] 667 668 def p_enum_list(self, p): 669 """enum_list : modifiers SYMBOL '=' expression enum_cont 670 | modifiers SYMBOL enum_cont""" 671 if len(p) > 4: 672 val = self.BuildAttribute('VALUE', p[4]) 673 enum = self.BuildNamed('EnumItem', p, 2, ListFromConcat(val, p[1])) 674 p[0] = ListFromConcat(enum, p[5]) 675 else: 676 enum = self.BuildNamed('EnumItem', p, 2, p[1]) 677 p[0] = ListFromConcat(enum, p[3]) 678 if self.parse_debug: DumpReduction('enum_list', p) 679 680 def p_enum_cont(self, p): 681 """enum_cont : ',' enum_list 682 |""" 683 if len(p) > 1: p[0] = p[2] 684 if self.parse_debug: DumpReduction('enum_cont', p) 685 686 def p_enum_cont_error(self, p): 687 """enum_cont : error enum_cont""" 688 p[0] = p[2] 689 if self.parse_debug: DumpReduction('enum_error', p) 690 691 692 # 693 # Label 694 # 695 # A label is a special kind of enumeration which allows us to go from a 696 # set of labels 697 # 698 def p_label_block(self, p): 699 """label_block : modifiers LABEL SYMBOL '{' label_list '}' ';'""" 700 p[0] = self.BuildNamed('Label', p, 3, ListFromConcat(p[1], p[5])) 701 if self.parse_debug: DumpReduction('label_block', p) 702 703 def p_label_list(self, p): 704 """label_list : modifiers SYMBOL '=' FLOAT label_cont""" 705 val = self.BuildAttribute('VALUE', p[4]) 706 label = self.BuildNamed('LabelItem', p, 2, ListFromConcat(val, p[1])) 707 p[0] = ListFromConcat(label, p[5]) 708 if self.parse_debug: DumpReduction('label_list', p) 709 710 def p_label_cont(self, p): 711 """label_cont : ',' label_list 712 |""" 713 if len(p) > 1: p[0] = p[2] 714 if self.parse_debug: DumpReduction('label_cont', p) 715 716 def p_label_cont_error(self, p): 717 """label_cont : error label_cont""" 718 p[0] = p[2] 719 if self.parse_debug: DumpReduction('label_error', p) 720 721 722 # 723 # Members 724 # 725 # A member attribute or function of a struct or interface. 726 # 727 def p_member_attribute(self, p): 728 """member_attribute : modifiers SYMBOL arrays questionmark identifier""" 729 typeref = self.BuildAttribute('TYPEREF', p[2]) 730 children = ListFromConcat(p[1], typeref, p[3], p[4]) 731 p[0] = self.BuildNamed('Member', p, 5, children) 732 if self.parse_debug: DumpReduction('attribute', p) 733 734 def p_member_function(self, p): 735 """member_function : modifiers static SYMBOL SYMBOL param_list""" 736 typeref = self.BuildAttribute('TYPEREF', p[3]) 737 children = ListFromConcat(p[1], p[2], typeref, p[5]) 738 p[0] = self.BuildNamed('Member', p, 4, children) 739 if self.parse_debug: DumpReduction('function', p) 740 741 def p_static(self, p): 742 """static : STATIC 743 | """ 744 if len(p) == 2: 745 p[0] = self.BuildAttribute('STATIC', True) 746 747 def p_questionmark(self, p): 748 """questionmark : '?' 749 | """ 750 if len(p) == 2: 751 p[0] = self.BuildAttribute('OPTIONAL', True) 752 753 # 754 # Interface 755 # 756 # An interface is a named collection of functions. 757 # 758 def p_interface_block(self, p): 759 """interface_block : modifiers INTERFACE SYMBOL '{' interface_list '}' ';'""" 760 p[0] = self.BuildNamed('Interface', p, 3, ListFromConcat(p[1], p[5])) 761 if self.parse_debug: DumpReduction('interface_block', p) 762 763 def p_interface_error(self, p): 764 """interface_block : modifiers INTERFACE error '{' interface_list '}' ';'""" 765 p[0] = [] 766 767 def p_interface_list(self, p): 768 """interface_list : member_function ';' interface_list 769 | """ 770 if len(p) > 1 : 771 p[0] = ListFromConcat(p[1], p[3]) 772 if self.parse_debug: DumpReduction('interface_list', p) 773 774 775 # 776 # Struct 777 # 778 # A struct is a named collection of members which in turn reference other 779 # types. The struct is a referencable type. 780 # 781 def p_struct_block(self, p): 782 """struct_block : modifiers STRUCT SYMBOL '{' struct_list '}' ';'""" 783 children = ListFromConcat(p[1], p[5]) 784 p[0] = self.BuildNamed('Struct', p, 3, children) 785 if self.parse_debug: DumpReduction('struct_block', p) 786 787 # Recover from struct error and continue parsing at the next top match. 788 def p_struct_error(self, p): 789 """enum_block : modifiers STRUCT error '{' struct_list '}' ';'""" 790 p[0] = [] 791 792 def p_struct_list(self, p): 793 """struct_list : member_attribute ';' struct_list 794 | member_function ';' struct_list 795 |""" 796 if len(p) > 1: p[0] = ListFromConcat(p[1], p[3]) 797 798 799 # 800 # Parser Errors 801 # 802 # p_error is called whenever the parser can not find a pattern match for 803 # a set of items from the current state. The p_error function defined here 804 # is triggered logging an error, and parsing recover happens as the 805 # p_<type>_error functions defined above are called. This allows the parser 806 # to continue so as to capture more than one error per file. 807 # 808 def p_error(self, t): 809 filename = self.lexobj.filename 810 self.parse_errors += 1 811 if t: 812 lineno = t.lineno 813 pos = t.lexpos 814 prev = self.yaccobj.symstack[-1] 815 if type(prev) == lex.LexToken: 816 msg = "Unexpected %s after %s." % ( 817 TokenTypeName(t), TokenTypeName(prev)) 818 else: 819 msg = "Unexpected %s." % (t.value) 820 else: 821 lineno = self.last.lineno 822 pos = self.last.lexpos 823 msg = "Unexpected end of file after %s." % TokenTypeName(self.last) 824 self.yaccobj.restart() 825 826 # Attempt to remap the error to a friendlier form 827 if msg in ERROR_REMAP: 828 msg = ERROR_REMAP[msg] 829 830 # Log the error 831 ErrOut.LogLine(filename, lineno, pos, msg) 832 833 def Warn(self, node, msg): 834 WarnOut.LogLine(node.filename, node.lineno, node.pos, msg) 835 self.parse_warnings += 1 836 837 def __init__(self): 838 IDLLexer.__init__(self) 839 self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False, 840 optimize=0, write_tables=0) 841 842 self.build_debug = GetOption('build_debug') 843 self.parse_debug = GetOption('parse_debug') 844 self.token_debug = GetOption('token_debug') 845 self.verbose = GetOption('verbose') 846 self.parse_errors = 0 847 848 # 849 # Tokenizer 850 # 851 # The token function returns the next token provided by IDLLexer for matching 852 # against the leaf paterns. 853 # 854 def token(self): 855 tok = self.lexobj.token() 856 if tok: 857 self.last = tok 858 if self.token_debug: 859 InfoOut.Log("TOKEN %s(%s)" % (tok.type, tok.value)) 860 return tok 861 862 # 863 # BuildProduction 864 # 865 # Production is the set of items sent to a grammar rule resulting in a new 866 # item being returned. 867 # 868 # p - Is the Yacc production object containing the stack of items 869 # index - Index into the production of the name for the item being produced. 870 # cls - The type of item being producted 871 # childlist - The children of the new item 872 def BuildProduction(self, cls, p, index, childlist=None): 873 if not childlist: childlist = [] 874 filename = self.lexobj.filename 875 lineno = p.lineno(index) 876 pos = p.lexpos(index) 877 out = IDLNode(cls, filename, lineno, pos, childlist) 878 if self.build_debug: 879 InfoOut.Log("Building %s" % out) 880 return out 881 882 def BuildNamed(self, cls, p, index, childlist=None): 883 if not childlist: childlist = [] 884 childlist.append(self.BuildAttribute('NAME', p[index])) 885 return self.BuildProduction(cls, p, index, childlist) 886 887 def BuildComment(self, cls, p, index): 888 name = p[index] 889 890 # Remove comment markers 891 lines = [] 892 if name[:2] == '//': 893 # For C++ style, remove any leading whitespace and the '//' marker from 894 # each line. 895 form = 'cc' 896 for line in name.split('\n'): 897 start = line.find('//') 898 lines.append(line[start+2:]) 899 else: 900 # For C style, remove ending '*/'' 901 form = 'c' 902 for line in name[:-2].split('\n'): 903 # Remove characters until start marker for this line '*' if found 904 # otherwise it should be blank. 905 offs = line.find('*') 906 if offs >= 0: 907 line = line[offs + 1:].rstrip() 908 else: 909 line = '' 910 lines.append(line) 911 name = '\n'.join(lines) 912 913 childlist = [self.BuildAttribute('NAME', name), 914 self.BuildAttribute('FORM', form)] 915 return self.BuildProduction(cls, p, index, childlist) 916 917 # 918 # BuildAttribute 919 # 920 # An ExtendedAttribute is a special production that results in a property 921 # which is applied to the adjacent item. Attributes have no children and 922 # instead represent key/value pairs. 923 # 924 def BuildAttribute(self, key, val): 925 return IDLAttribute(key, val) 926 927 928 # 929 # ParseData 930 # 931 # Attempts to parse the current data loaded in the lexer. 932 # 933 def ParseData(self, data, filename='<Internal>'): 934 self.SetData(filename, data) 935 try: 936 self.parse_errors = 0 937 self.parse_warnings = 0 938 return self.yaccobj.parse(lexer=self) 939 940 except lex.LexError as le: 941 ErrOut.Log(str(le)) 942 return [] 943 944 # 945 # ParseFile 946 # 947 # Loads a new file into the lexer and attemps to parse it. 948 # 949 def ParseFile(self, filename): 950 date = time.ctime(os.path.getmtime(filename)) 951 data = open(filename).read() 952 if self.verbose: 953 InfoOut.Log("Parsing %s" % filename) 954 try: 955 out = self.ParseData(data, filename) 956 957 # If we have a src root specified, remove it from the path 958 srcroot = GetOption('srcroot') 959 if srcroot and filename.find(srcroot) == 0: 960 filename = filename[len(srcroot) + 1:] 961 filenode = IDLFile(filename, out, self.parse_errors + self.lex_errors) 962 filenode.SetProperty('DATETIME', date) 963 return filenode 964 965 except Exception as e: 966 ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos, 967 'Internal parsing error - %s.' % str(e)) 968 raise 969 970 971 972 # 973 # Flatten Tree 974 # 975 # Flattens the tree of IDLNodes for use in testing. 976 # 977 def FlattenTree(node): 978 add_self = False 979 out = [] 980 for child in node.children: 981 if child.IsA('Comment'): 982 add_self = True 983 else: 984 out.extend(FlattenTree(child)) 985 986 if add_self: 987 out = [str(node)] + out 988 return out 989 990 991 def TestErrors(filename, filenode): 992 nodelist = filenode.GetChildren() 993 994 lexer = IDLLexer() 995 data = open(filename).read() 996 lexer.SetData(filename, data) 997 998 pass_comments = [] 999 fail_comments = [] 1000 while True: 1001 tok = lexer.lexobj.token() 1002 if tok == None: break 1003 if tok.type == 'COMMENT': 1004 args = tok.value[3:-3].split() 1005 if args[0] == 'OK': 1006 pass_comments.append((tok.lineno, ' '.join(args[1:]))) 1007 else: 1008 if args[0] == 'FAIL': 1009 fail_comments.append((tok.lineno, ' '.join(args[1:]))) 1010 obj_list = [] 1011 for node in nodelist: 1012 obj_list.extend(FlattenTree(node)) 1013 1014 errors = 0 1015 1016 # 1017 # Check for expected successes 1018 # 1019 obj_cnt = len(obj_list) 1020 pass_cnt = len(pass_comments) 1021 if obj_cnt != pass_cnt: 1022 InfoOut.Log("Mismatched pass (%d) vs. nodes built (%d)." 1023 % (pass_cnt, obj_cnt)) 1024 InfoOut.Log("PASS: %s" % [x[1] for x in pass_comments]) 1025 InfoOut.Log("OBJS: %s" % obj_list) 1026 errors += 1 1027 if pass_cnt > obj_cnt: pass_cnt = obj_cnt 1028 1029 for i in range(pass_cnt): 1030 line, comment = pass_comments[i] 1031 if obj_list[i] != comment: 1032 ErrOut.LogLine(filename, line, None, "OBJ %s : EXPECTED %s\n" % 1033 (obj_list[i], comment)) 1034 errors += 1 1035 1036 # 1037 # Check for expected errors 1038 # 1039 err_list = ErrOut.DrainLog() 1040 err_cnt = len(err_list) 1041 fail_cnt = len(fail_comments) 1042 if err_cnt != fail_cnt: 1043 InfoOut.Log("Mismatched fail (%d) vs. errors seen (%d)." 1044 % (fail_cnt, err_cnt)) 1045 InfoOut.Log("FAIL: %s" % [x[1] for x in fail_comments]) 1046 InfoOut.Log("ERRS: %s" % err_list) 1047 errors += 1 1048 if fail_cnt > err_cnt: fail_cnt = err_cnt 1049 1050 for i in range(fail_cnt): 1051 line, comment = fail_comments[i] 1052 err = err_list[i].strip() 1053 1054 if err_list[i] != comment: 1055 ErrOut.Log("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % ( 1056 filename, line, err_list[i], comment)) 1057 errors += 1 1058 1059 # Clear the error list for the next run 1060 err_list = [] 1061 return errors 1062 1063 1064 def TestFile(parser, filename): 1065 # Capture errors instead of reporting them so we can compare them 1066 # with the expected errors. 1067 ErrOut.SetConsole(False) 1068 ErrOut.SetCapture(True) 1069 1070 filenode = parser.ParseFile(filename) 1071 1072 # Renable output 1073 ErrOut.SetConsole(True) 1074 ErrOut.SetCapture(False) 1075 1076 # Compare captured errors 1077 return TestErrors(filename, filenode) 1078 1079 1080 def TestErrorFiles(filter): 1081 idldir = os.path.split(sys.argv[0])[0] 1082 idldir = os.path.join(idldir, 'test_parser', '*.idl') 1083 filenames = glob.glob(idldir) 1084 parser = IDLParser() 1085 total_errs = 0 1086 for filename in filenames: 1087 if filter and filename not in filter: continue 1088 errs = TestFile(parser, filename) 1089 if errs: 1090 ErrOut.Log("%s test failed with %d error(s)." % (filename, errs)) 1091 total_errs += errs 1092 1093 if total_errs: 1094 ErrOut.Log("Failed parsing test.") 1095 else: 1096 InfoOut.Log("Passed parsing test.") 1097 return total_errs 1098 1099 1100 def TestNamespaceFiles(filter): 1101 idldir = os.path.split(sys.argv[0])[0] 1102 idldir = os.path.join(idldir, 'test_namespace', '*.idl') 1103 filenames = glob.glob(idldir) 1104 testnames = [] 1105 1106 for filename in filenames: 1107 if filter and filename not in filter: continue 1108 testnames.append(filename) 1109 1110 # If we have no files to test, then skip this test 1111 if not testnames: 1112 InfoOut.Log('No files to test for namespace.') 1113 return 0 1114 1115 InfoOut.SetConsole(False) 1116 ast = ParseFiles(testnames) 1117 InfoOut.SetConsole(True) 1118 1119 errs = ast.GetProperty('ERRORS') 1120 if errs: 1121 ErrOut.Log("Failed namespace test.") 1122 else: 1123 InfoOut.Log("Passed namespace test.") 1124 return errs 1125 1126 1127 1128 def FindVersionError(releases, node): 1129 err_cnt = 0 1130 if node.IsA('Interface', 'Struct'): 1131 comment_list = [] 1132 comment = node.GetOneOf('Comment') 1133 if comment and comment.GetName()[:4] == 'REL:': 1134 comment_list = comment.GetName()[5:].strip().split(' ') 1135 1136 first_list = [node.first_release[rel] for rel in releases] 1137 first_list = sorted(set(first_list)) 1138 if first_list != comment_list: 1139 node.Error("Mismatch in releases: %s vs %s." % ( 1140 comment_list, first_list)) 1141 err_cnt += 1 1142 1143 for child in node.GetChildren(): 1144 err_cnt += FindVersionError(releases, child) 1145 return err_cnt 1146 1147 1148 def TestVersionFiles(filter): 1149 idldir = os.path.split(sys.argv[0])[0] 1150 idldir = os.path.join(idldir, 'test_version', '*.idl') 1151 filenames = glob.glob(idldir) 1152 testnames = [] 1153 1154 for filename in filenames: 1155 if filter and filename not in filter: continue 1156 testnames.append(filename) 1157 1158 # If we have no files to test, then skip this test 1159 if not testnames: 1160 InfoOut.Log('No files to test for version.') 1161 return 0 1162 1163 ast = ParseFiles(testnames) 1164 errs = FindVersionError(ast.releases, ast) 1165 errs += ast.errors 1166 1167 if errs: 1168 ErrOut.Log("Failed version test.") 1169 else: 1170 InfoOut.Log("Passed version test.") 1171 return errs 1172 1173 1174 default_dirs = ['.', 'trusted', 'dev', 'private', 'extensions', 1175 'extensions/dev'] 1176 def ParseFiles(filenames): 1177 parser = IDLParser() 1178 filenodes = [] 1179 1180 if not filenames: 1181 filenames = [] 1182 srcroot = GetOption('srcroot') 1183 dirs = default_dirs 1184 if GetOption('include_private'): 1185 dirs += ['private'] 1186 for dirname in dirs: 1187 srcdir = os.path.join(srcroot, dirname, '*.idl') 1188 srcdir = os.path.normpath(srcdir) 1189 filenames += sorted(glob.glob(srcdir)) 1190 1191 if not filenames: 1192 ErrOut.Log('No sources provided.') 1193 1194 for filename in filenames: 1195 filenode = parser.ParseFile(filename) 1196 filenodes.append(filenode) 1197 1198 ast = IDLAst(filenodes) 1199 if GetOption('dump_tree'): ast.Dump(0) 1200 1201 Lint(ast) 1202 return ast 1203 1204 1205 def Main(args): 1206 filenames = ParseOptions(args) 1207 1208 # If testing... 1209 if GetOption('test'): 1210 errs = TestErrorFiles(filenames) 1211 errs = TestNamespaceFiles(filenames) 1212 errs = TestVersionFiles(filenames) 1213 if errs: 1214 ErrOut.Log("Parser failed with %d errors." % errs) 1215 return -1 1216 return 0 1217 1218 # Otherwise, build the AST 1219 ast = ParseFiles(filenames) 1220 errs = ast.GetProperty('ERRORS') 1221 if errs: 1222 ErrOut.Log('Found %d error(s).' % errs); 1223 InfoOut.Log("%d files processed." % len(filenames)) 1224 return errs 1225 1226 1227 if __name__ == '__main__': 1228 sys.exit(Main(sys.argv[1:])) 1229 1230