Home | History | Annotate | Download | only in idl_parser 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """ Parser for PPAPI IDL """
      7 
      8 #
      9 # IDL Parser
     10 #
     11 # The parser is uses the PLY yacc library to build a set of parsing rules based
     12 # on WebIDL.
     13 #
     14 # WebIDL, and WebIDL grammar can be found at:
     15 #   http://heycam.github.io/webidl/
     16 # PLY can be found at:
     17 #   http://www.dabeaz.com/ply/
     18 #
     19 # The parser generates a tree by recursively matching sets of items against
     20 # defined patterns.  When a match is made, that set of items is reduced
     21 # to a new item.   The new item can provide a match for parent patterns.
     22 # In this way an AST is built (reduced) depth first.
     23 #
     24 
     25 #
     26 # Disable check for line length and Member as Function due to how grammar rules
     27 # are defined with PLY
     28 #
     29 # pylint: disable=R0201
     30 # pylint: disable=C0301
     31 
     32 import os.path
     33 import sys
     34 import time
     35 
     36 from idl_lexer import IDLLexer
     37 from idl_node import IDLAttribute, IDLNode
     38 
     39 #
     40 # Try to load the ply module, if not, then assume it is in the third_party
     41 # directory.
     42 #
     43 try:
     44   # Disable lint check which fails to find the ply module.
     45   # pylint: disable=F0401
     46   from ply import lex
     47   from ply import yacc
     48 except ImportError:
     49   module_path, module_name = os.path.split(__file__)
     50   third_party = os.path.join(module_path, os.par, os.par, 'third_party')
     51   sys.path.append(third_party)
     52   # pylint: disable=F0401
     53   from ply import lex
     54   from ply import yacc
     55 
     56 #
     57 # ERROR_REMAP
     58 #
     59 # Maps the standard error formula into a more friendly error message.
     60 #
     61 ERROR_REMAP = {
     62   'Unexpected ")" after "(".' : 'Empty argument list.',
     63   'Unexpected ")" after ",".' : 'Missing argument.',
     64   'Unexpected "}" after ",".' : 'Trailing comma in block.',
     65   'Unexpected "}" after "{".' : 'Unexpected empty block.',
     66   'Unexpected comment after "}".' : 'Unexpected trailing comment.',
     67   'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
     68   'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
     69   'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
     70 }
     71 
     72 
     73 def Boolean(val):
     74   """Convert to strict boolean type."""
     75   if val:
     76     return True
     77   return False
     78 
     79 
     80 def ListFromConcat(*items):
     81   """Generate list by concatenating inputs"""
     82   itemsout = []
     83   for item in items:
     84     if item is None:
     85       continue
     86     if type(item) is not type([]):
     87       itemsout.append(item)
     88     else:
     89       itemsout.extend(item)
     90 
     91   return itemsout
     92 
     93 def ExpandProduction(p):
     94   if type(p) == list:
     95     return '[' + ', '.join([ExpandProduction(x) for x in p]) + ']'
     96   if type(p) == IDLNode:
     97     return 'Node:' + str(p)
     98   if type(p) == IDLAttribute:
     99     return 'Attr:' + str(p)
    100   if type(p) == str:
    101     return 'str:' + p
    102   return '%s:%s' % (p.__class__.__name__, str(p))
    103 
    104 # TokenTypeName
    105 #
    106 # Generate a string which has the type and value of the token.
    107 #
    108 def TokenTypeName(t):
    109   if t.type == 'SYMBOL':
    110     return 'symbol %s' % t.value
    111   if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']:
    112     return 'value %s' % t.value
    113   if t.type == 'string' :
    114     return 'string "%s"' % t.value
    115   if t.type == 'COMMENT' :
    116     return 'comment'
    117   if t.type == t.value:
    118     return '"%s"' % t.value
    119   if t.type == ',':
    120     return 'Comma'
    121   if t.type == 'identifier':
    122     return 'identifier "%s"' % t.value
    123   return 'keyword "%s"' % t.value
    124 
    125 
    126 #
    127 # IDL Parser
    128 #
    129 # The Parser inherits the from the Lexer to provide PLY with the tokenizing
    130 # definitions.  Parsing patterns are encoded as functions where p_<name> is
    131 # is called any time a patern matching the function documentation is found.
    132 # Paterns are expressed in the form of:
    133 # """ <new item> : <item> ....
    134 #                | <item> ...."""
    135 #
    136 # Where new item is the result of a match against one or more sets of items
    137 # separated by the "|".
    138 #
    139 # The function is called with an object 'p' where p[0] is the output object
    140 # and p[n] is the set of inputs for positive values of 'n'.  Len(p) can be
    141 # used to distinguish between multiple item sets in the pattern.
    142 #
    143 # For more details on parsing refer to the PLY documentation at
    144 #    http://www.dabeaz.com/ply/
    145 #
    146 # The parser is based on the WebIDL standard.  See:
    147 #    http://heycam.github.io/webidl/#idl-grammar
    148 #
    149 # The various productions are annotated so that the WHOLE number greater than
    150 # zero in the comment denotes the matching WebIDL grammar definition.
    151 #
    152 # Productions with a fractional component in the comment denote additions to
    153 # the WebIDL spec, such as comments.
    154 #
    155 
    156 
    157 class IDLParser(object):
    158 #
    159 # We force all input files to start with two comments.  The first comment is a
    160 # Copyright notice followed by a file comment and finally by file level
    161 # productions.
    162 #
    163   # [0] Insert a TOP definition for Copyright and Comments
    164   def p_Top(self, p):
    165     """Top : COMMENT COMMENT Definitions"""
    166     Copyright = self.BuildComment('Copyright', p, 1)
    167     Filedoc = self.BuildComment('Comment', p, 2)
    168     p[0] = ListFromConcat(Copyright, Filedoc, p[3])
    169 
    170   # [0.1] Add support for Multiple COMMENTS
    171   def p_Comments(self, p):
    172     """Comments : CommentsRest"""
    173     if len(p) > 1:
    174       p[0] = p[1]
    175 
    176   # [0.2] Produce a COMMENT and aggregate sibling comments
    177   def p_CommentsRest(self, p):
    178     """CommentsRest : COMMENT CommentsRest
    179                     | """
    180     if len(p) > 1:
    181       p[0] = ListFromConcat(self.BuildComment('Comment', p, 1), p[2])
    182 
    183 
    184 #
    185 #The parser is based on the WebIDL standard.  See:
    186 # http://heycam.github.io/webidl/#idl-grammar
    187 #
    188   # [1]
    189   def p_Definitions(self, p):
    190     """Definitions : ExtendedAttributeList Definition Definitions
    191            | """
    192     if len(p) > 1:
    193       p[2].AddChildren(p[1])
    194       p[0] = ListFromConcat(p[2], p[3])
    195 
    196   # [2]
    197   def p_Definition(self, p):
    198     """Definition : CallbackOrInterface
    199                   | Partial
    200                   | Dictionary
    201                   | Exception
    202                   | Enum
    203                   | Typedef
    204                   | ImplementsStatement"""
    205     p[0] = p[1]
    206 
    207   # [2.1] Error recovery for definition
    208   def p_DefinitionError(self, p):
    209     """Definition : error ';'"""
    210     p[0] = self.BuildError(p, 'Definition')
    211 
    212   # [3]
    213   def p_CallbackOrInterface(self, p):
    214     """CallbackOrInterface : CALLBACK CallbackRestOrInterface
    215                            | Interface"""
    216     if len(p) > 2:
    217       p[0] = p[2]
    218     else:
    219       p[0] = p[1]
    220 
    221   # [4]
    222   def p_CallbackRestOrInterface(self, p):
    223     """CallbackRestOrInterface : CallbackRest
    224                                | Interface"""
    225     p[0] = p[1]
    226 
    227   # [5]
    228   def p_Interface(self, p):
    229     """Interface : INTERFACE identifier Inheritance '{' InterfaceMembers '}' ';'"""
    230     p[0] = self.BuildNamed('Interface', p, 2, ListFromConcat(p[3], p[5]))
    231 
    232   # [6]
    233   def p_Partial(self, p):
    234     """Partial : PARTIAL PartialDefinition"""
    235     p[2].AddChildren(self.BuildTrue('Partial'))
    236     p[0] = p[2]
    237 
    238   # [6.1] Error recovery for Partial
    239   def p_PartialError(self, p):
    240     """Partial : PARTIAL error"""
    241     p[0] = self.BuildError(p, 'Partial')
    242 
    243   # [7]
    244   def p_PartialDefinition(self, p):
    245     """PartialDefinition : PartialDictionary
    246                          | PartialInterface"""
    247     p[0] = p[1]
    248 
    249   # [8]
    250   def p_PartialInterface(self, p):
    251     """PartialInterface : INTERFACE identifier '{' InterfaceMembers '}' ';'"""
    252     p[0] = self.BuildNamed('Interface', p, 2, p[4])
    253 
    254   # [9]
    255   def p_InterfaceMembers(self, p):
    256     """InterfaceMembers : ExtendedAttributeList InterfaceMember InterfaceMembers
    257                         |"""
    258     if len(p) > 1:
    259       p[2].AddChildren(p[1])
    260       p[0] = ListFromConcat(p[2], p[3])
    261 
    262   # [10]
    263   def p_InterfaceMember(self, p):
    264     """InterfaceMember : Const
    265                        | AttributeOrOperationOrIterator"""
    266     p[0] = p[1]
    267 
    268   # [11]
    269   def p_Dictionary(self, p):
    270     """Dictionary : DICTIONARY identifier Inheritance '{' DictionaryMembers '}' ';'"""
    271     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[3], p[5]))
    272 
    273   # [11.1] Error recovery for regular Dictionary
    274   def p_DictionaryError(self, p):
    275     """Dictionary : DICTIONARY error ';'"""
    276     p[0] = self.BuildError(p, 'Dictionary')
    277 
    278   # [12]
    279   def p_DictionaryMembers(self, p):
    280     """DictionaryMembers : ExtendedAttributeList DictionaryMember DictionaryMembers
    281                          |"""
    282     if len(p) > 1:
    283       p[2].AddChildren(p[1])
    284       p[0] = ListFromConcat(p[2], p[3])
    285 
    286   # [13]
    287   def p_DictionaryMember(self, p):
    288     """DictionaryMember : Type identifier Default ';'"""
    289     p[0] = self.BuildNamed('Key', p, 2, ListFromConcat(p[1], p[3]))
    290 
    291   # [14]
    292   def p_PartialDictionary(self, p):
    293     """PartialDictionary : DICTIONARY identifier '{' DictionaryMembers '}' ';'"""
    294     partial = self.BuildTrue('Partial')
    295     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[4], partial))
    296 
    297   # [14.1] Error recovery for Partial Dictionary
    298   def p_PartialDictionaryError(self, p):
    299     """PartialDictionary : DICTIONARY error ';'"""
    300     p[0] = self.BuildError(p, 'PartialDictionary')
    301 
    302   # [15]
    303   def p_Default(self, p):
    304     """Default : '=' DefaultValue
    305                |"""
    306     if len(p) > 1:
    307       p[0] = self.BuildProduction('Default', p, 2, p[2])
    308 
    309   # [16]
    310   def p_DefaultValue(self, p):
    311     """DefaultValue : ConstValue
    312                     | string"""
    313     if type(p[1]) == str:
    314       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'DOMString'),
    315                             self.BuildAttribute('NAME', p[1]))
    316     else:
    317       p[0] = p[1]
    318 
    319   # [17]
    320   def p_Exception(self, p):
    321     """Exception : EXCEPTION identifier Inheritance '{' ExceptionMembers '}' ';'"""
    322     p[0] = self.BuildNamed('Exception', p, 2, ListFromConcat(p[3], p[5]))
    323 
    324   # [18]
    325   def p_ExceptionMembers(self, p):
    326     """ExceptionMembers : ExtendedAttributeList ExceptionMember ExceptionMembers
    327                         |"""
    328     if len(p) > 1:
    329       p[2].AddChildren(p[1])
    330       p[0] = ListFromConcat(p[2], p[3])
    331 
    332   # [18.1] Error recovery for ExceptionMembers
    333   def p_ExceptionMembersError(self, p):
    334     """ExceptionMembers : error"""
    335     p[0] = self.BuildError(p, 'ExceptionMembers')
    336 
    337   # [19]
    338   def p_Inheritance(self, p):
    339     """Inheritance : ':' identifier
    340                    |"""
    341     if len(p) > 1:
    342       p[0] = self.BuildNamed('Inherit', p, 2)
    343 
    344   # [20]
    345   def p_Enum(self, p):
    346     """Enum : ENUM identifier '{' EnumValueList '}' ';'"""
    347     p[0] = self.BuildNamed('Enum', p, 2, p[4])
    348 
    349   # [20.1] Error recovery for Enums
    350   def p_EnumError(self, p):
    351     """Enum : ENUM error ';'"""
    352     p[0] = self.BuildError(p, 'Enum')
    353 
    354   # [21]
    355   def p_EnumValueList(self, p):
    356     """EnumValueList : ExtendedAttributeList string EnumValueListComma"""
    357     enum = self.BuildNamed('EnumItem', p, 2, p[1])
    358     p[0] = ListFromConcat(enum, p[3])
    359 
    360   # [22]
    361   def p_EnumValueListComma(self, p):
    362     """EnumValueListComma : ',' EnumValueListString
    363                           |"""
    364     if len(p) > 1:
    365       p[0] = p[2]
    366 
    367   # [23]
    368   def p_EnumValueListString(self, p):
    369     """EnumValueListString : ExtendedAttributeList string EnumValueListComma
    370                            |"""
    371     if len(p) > 1:
    372       enum = self.BuildNamed('EnumItem', p, 2, p[1])
    373       p[0] = ListFromConcat(enum, p[3])
    374 
    375   # [24]
    376   def p_CallbackRest(self, p):
    377     """CallbackRest : identifier '=' ReturnType '(' ArgumentList ')' ';'"""
    378     arguments = self.BuildProduction('Arguments', p, 4, p[5])
    379     p[0] = self.BuildNamed('Callback', p, 1, ListFromConcat(p[3], arguments))
    380 
    381   # [25]
    382   def p_Typedef(self, p):
    383     """Typedef : TYPEDEF ExtendedAttributeListNoComments Type identifier ';'"""
    384     p[0] = self.BuildNamed('Typedef', p, 4, ListFromConcat(p[2], p[3]))
    385 
    386   # [25.1] Error recovery for Typedefs
    387   def p_TypedefError(self, p):
    388     """Typedef : TYPEDEF error ';'"""
    389     p[0] = self.BuildError(p, 'Typedef')
    390 
    391   # [26]
    392   def p_ImplementsStatement(self, p):
    393     """ImplementsStatement : identifier IMPLEMENTS identifier ';'"""
    394     name = self.BuildAttribute('REFERENCE', p[3])
    395     p[0] = self.BuildNamed('Implements', p, 1, name)
    396 
    397   # [27]
    398   def p_Const(self,  p):
    399     """Const : CONST ConstType identifier '=' ConstValue ';'"""
    400     value = self.BuildProduction('Value', p, 5, p[5])
    401     p[0] = self.BuildNamed('Const', p, 3, ListFromConcat(p[2], value))
    402 
    403   # [28]
    404   def p_ConstValue(self, p):
    405     """ConstValue : BooleanLiteral
    406                   | FloatLiteral
    407                   | integer
    408                   | null"""
    409     if type(p[1]) == str:
    410       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'integer'),
    411                             self.BuildAttribute('NAME', p[1]))
    412     else:
    413       p[0] = p[1]
    414 
    415   # [28.1] Add definition for NULL
    416   def p_null(self, p):
    417     """null : NULL"""
    418     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'NULL'),
    419                           self.BuildAttribute('NAME', 'NULL'))
    420 
    421   # [29]
    422   def p_BooleanLiteral(self, p):
    423     """BooleanLiteral : TRUE
    424                       | FALSE"""
    425     value = self.BuildAttribute('VALUE', Boolean(p[1] == 'true'))
    426     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'boolean'), value)
    427 
    428   # [30]
    429   def p_FloatLiteral(self, p):
    430     """FloatLiteral : float
    431                     | '-' INFINITY
    432                     | INFINITY
    433                     | NAN """
    434     if len(p) > 2:
    435       val = '-Infinity'
    436     else:
    437       val = p[1]
    438     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'float'),
    439                           self.BuildAttribute('VALUE', val))
    440 
    441   # [31] Removed unsupported: Serializer
    442   def p_AttributeOrOperationOrIterator(self, p):
    443     """AttributeOrOperationOrIterator : Stringifier
    444                                       | StaticMember
    445                                       | Attribute
    446                                       | OperationOrIterator"""
    447     p[0] = p[1]
    448 
    449   # [32-37] NOT IMPLEMENTED (Serializer)
    450 
    451   # [38]
    452   def p_Stringifier(self, p):
    453     """Stringifier : STRINGIFIER StringifierRest"""
    454     p[0] = self.BuildProduction('Stringifier', p, 1, p[2])
    455 
    456   # [39]
    457   def p_StringifierRest(self, p):
    458     """StringifierRest : AttributeRest
    459                        | ReturnType OperationRest
    460                        | ';'"""
    461     if len(p) == 3:
    462       p[2].AddChildren(p[1])
    463       p[0] = p[2]
    464     elif p[1] != ';':
    465       p[0] = p[1]
    466 
    467   # [40]
    468   def p_StaticMember(self, p):
    469     """StaticMember : STATIC StaticMemberRest"""
    470     p[2].AddChildren(self.BuildTrue('STATIC'))
    471     p[0] = p[2]
    472 
    473   # [41]
    474   def p_StaticMemberRest(self, p):
    475     """StaticMemberRest : AttributeRest
    476                         | ReturnType OperationRest"""
    477     if len(p) == 2:
    478       p[0] = p[1]
    479     else:
    480       p[2].AddChildren(p[1])
    481       p[0] = p[2]
    482 
    483   # [42]
    484   def p_Attribute(self, p):
    485     """Attribute : Inherit AttributeRest"""
    486     p[2].AddChildren(ListFromConcat(p[1]))
    487     p[0] = p[2]
    488 
    489   # [43]
    490   def p_AttributeRest(self, p):
    491     """AttributeRest : ReadOnly ATTRIBUTE Type identifier ';'"""
    492     p[0] = self.BuildNamed('Attribute', p, 4,
    493                            ListFromConcat(p[1], p[3]))
    494 
    495   # [44]
    496   def p_Inherit(self, p):
    497     """Inherit : INHERIT
    498                |"""
    499     if len(p) > 1:
    500       p[0] = self.BuildTrue('INHERIT')
    501 
    502   # [45]
    503   def p_ReadOnly(self, p):
    504     """ReadOnly : READONLY
    505                 |"""
    506     if len(p) > 1:
    507       p[0] = self.BuildTrue('READONLY')
    508 
    509   # [46]
    510   def p_OperationOrIterator(self, p):
    511     """OperationOrIterator : ReturnType OperationOrIteratorRest
    512                            | SpecialOperation"""
    513     if len(p) == 3:
    514       p[2].AddChildren(p[1])
    515       p[0] = p[2]
    516     else:
    517       p[0] = p[1]
    518 
    519   # [47]
    520   def p_SpecialOperation(self, p):
    521     """SpecialOperation : Special Specials ReturnType OperationRest"""
    522     p[4].AddChildren(ListFromConcat(p[1], p[2], p[3]))
    523     p[0] = p[4]
    524 
    525   # [48]
    526   def p_Specials(self, p):
    527     """Specials : Special Specials
    528                 | """
    529     if len(p) > 1:
    530       p[0] = ListFromConcat(p[1], p[2])
    531 
    532   # [49]
    533   def p_Special(self, p):
    534     """Special : GETTER
    535                | SETTER
    536                | CREATOR
    537                | DELETER
    538                | LEGACYCALLER"""
    539     p[0] = self.BuildTrue(p[1].upper())
    540 
    541   # [50] Removed unsupported: IteratorRest
    542   def p_OperationOrIteratorRest(self, p):
    543     """OperationOrIteratorRest : OperationRest"""
    544     p[0] = p[1]
    545 
    546   # [51-53] NOT IMPLEMENTED (IteratorRest)
    547 
    548   # [54]
    549   def p_OperationRest(self, p):
    550     """OperationRest : OptionalIdentifier '(' ArgumentList ')' ';'"""
    551     arguments = self.BuildProduction('Arguments', p, 2, p[3])
    552     p[0] = self.BuildNamed('Operation', p, 1, arguments)
    553 
    554   # [55]
    555   def p_OptionalIdentifier(self, p):
    556     """OptionalIdentifier : identifier
    557                           |"""
    558     if len(p) > 1:
    559       p[0] = p[1]
    560     else:
    561       p[0] = '_unnamed_'
    562 
    563   # [56]
    564   def p_ArgumentList(self, p):
    565     """ArgumentList : Argument Arguments
    566                     |"""
    567     if len(p) > 1:
    568       p[0] = ListFromConcat(p[1], p[2])
    569 
    570   # [56.1] ArgumentList error recovery
    571   def p_ArgumentListError(self, p):
    572     """ArgumentList : error """
    573     p[0] = self.BuildError(p, 'ArgumentList')
    574 
    575   # [57]
    576   def p_Arguments(self, p):
    577     """Arguments : ',' Argument Arguments
    578                  |"""
    579     if len(p) > 1:
    580       p[0] = ListFromConcat(p[2], p[3])
    581 
    582   # [58]
    583   def p_Argument(self, p):
    584     """Argument : ExtendedAttributeList OptionalOrRequiredArgument"""
    585     p[2].AddChildren(p[1])
    586     p[0] = p[2]
    587 
    588   # [59]
    589   def p_OptionalOrRequiredArgument(self, p):
    590     """OptionalOrRequiredArgument : OPTIONAL Type ArgumentName Default
    591                                   | Type Ellipsis ArgumentName"""
    592     if len(p) > 4:
    593       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[2], p[4]))
    594       arg.AddChildren(self.BuildTrue('OPTIONAL'))
    595     else:
    596       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[1], p[2]))
    597     p[0] = arg
    598 
    599   # [60]
    600   def p_ArgumentName(self, p):
    601     """ArgumentName : ArgumentNameKeyword
    602                     | identifier"""
    603     p[0] = p[1]
    604 
    605   # [61]
    606   def p_Ellipsis(self, p):
    607     """Ellipsis : ELLIPSIS
    608                 |"""
    609     if len(p) > 1:
    610       p[0] = self.BuildNamed('Argument', p, 1)
    611       p[0].AddChildren(self.BuildTrue('ELLIPSIS'))
    612 
    613   # [62]
    614   def p_ExceptionMember(self, p):
    615     """ExceptionMember : Const
    616                        | ExceptionField"""
    617     p[0] = p[1]
    618 
    619   # [63]
    620   def p_ExceptionField(self, p):
    621     """ExceptionField : Type identifier ';'"""
    622     p[0] = self.BuildNamed('ExceptionField', p, 2, p[1])
    623 
    624   # [63.1] Error recovery for ExceptionMembers
    625   def p_ExceptionFieldError(self, p):
    626     """ExceptionField : error"""
    627     p[0] = self.BuildError(p, 'ExceptionField')
    628 
    629   # [64] No comment version for mid statement attributes.
    630   def p_ExtendedAttributeListNoComments(self, p):
    631     """ExtendedAttributeListNoComments : '[' ExtendedAttribute ExtendedAttributes ']'
    632                                        | """
    633     if len(p) > 2:
    634       items = ListFromConcat(p[2], p[3])
    635       p[0] = self.BuildProduction('ExtAttributes', p, 1, items)
    636 
    637   # [64.1] Add optional comment field for start of statements.
    638   def p_ExtendedAttributeList(self, p):
    639     """ExtendedAttributeList : Comments '[' ExtendedAttribute ExtendedAttributes ']'
    640                              | Comments """
    641     if len(p) > 2:
    642       items = ListFromConcat(p[3], p[4])
    643       attribs = self.BuildProduction('ExtAttributes', p, 2, items)
    644       p[0] = ListFromConcat(p[1], attribs)
    645     else:
    646       p[0] = p[1]
    647 
    648   # [65]
    649   def p_ExtendedAttributes(self, p):
    650     """ExtendedAttributes : ',' ExtendedAttribute ExtendedAttributes
    651                           |"""
    652     if len(p) > 1:
    653       p[0] = ListFromConcat(p[2], p[3])
    654 
    655   # We only support:
    656   #    [ identifier ]
    657   #    [ identifier ( ArgumentList ) ]
    658   #    [ identifier = identifier ]
    659   #    [ identifier = ( IdentifierList ) ]
    660   #    [ identifier = identifier ( ArgumentList ) ]
    661   # [66] map directly to [91-93, 95]
    662   # [67-69, 71] are unsupported
    663   def p_ExtendedAttribute(self, p):
    664     """ExtendedAttribute : ExtendedAttributeNoArgs
    665                          | ExtendedAttributeArgList
    666                          | ExtendedAttributeIdent
    667                          | ExtendedAttributeIdentList
    668                          | ExtendedAttributeNamedArgList"""
    669     p[0] = p[1]
    670 
    671   # [70]
    672   def p_ArgumentNameKeyword(self, p):
    673     """ArgumentNameKeyword : ATTRIBUTE
    674                            | CALLBACK
    675                            | CONST
    676                            | CREATOR
    677                            | DELETER
    678                            | DICTIONARY
    679                            | ENUM
    680                            | EXCEPTION
    681                            | GETTER
    682                            | IMPLEMENTS
    683                            | INHERIT
    684                            | LEGACYCALLER
    685                            | PARTIAL
    686                            | SERIALIZER
    687                            | SETTER
    688                            | STATIC
    689                            | STRINGIFIER
    690                            | TYPEDEF
    691                            | UNRESTRICTED"""
    692     p[0] = p[1]
    693 
    694   # [72]
    695   def p_Type(self, p):
    696     """Type : SingleType
    697             | UnionType TypeSuffix"""
    698     if len(p) == 2:
    699       p[0] = self.BuildProduction('Type', p, 1, p[1])
    700     else:
    701       p[0] = self.BuildProduction('Type', p, 1, ListFromConcat(p[1], p[2]))
    702 
    703   # [73]
    704   def p_SingleType(self, p):
    705     """SingleType : NonAnyType
    706                   | ANY TypeSuffixStartingWithArray"""
    707     if len(p) == 2:
    708       p[0] = p[1]
    709     else:
    710       p[0] = ListFromConcat(self.BuildProduction('Any', p, 1), p[2])
    711 
    712   # [74]
    713   def p_UnionType(self, p):
    714     """UnionType : '(' UnionMemberType OR UnionMemberType UnionMemberTypes ')'"""
    715 
    716   # [75]
    717   def p_UnionMemberType(self, p):
    718     """UnionMemberType : NonAnyType
    719                        | UnionType TypeSuffix
    720                        | ANY '[' ']' TypeSuffix"""
    721   # [76]
    722   def p_UnionMemberTypes(self, p):
    723     """UnionMemberTypes : OR UnionMemberType UnionMemberTypes
    724                         |"""
    725 
    726   # [77] Moved BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP to PrimitiveType
    727   # Moving all built-in types into PrimitiveType makes it easier to
    728   # differentiate between them and 'identifier', since p[1] would be a string in
    729   # both cases.
    730   def p_NonAnyType(self, p):
    731     """NonAnyType : PrimitiveType TypeSuffix
    732                   | identifier TypeSuffix
    733                   | SEQUENCE '<' Type '>' Null"""
    734     if len(p) == 3:
    735       if type(p[1]) == str:
    736         typeref = self.BuildNamed('Typeref', p, 1)
    737       else:
    738         typeref = p[1]
    739       p[0] = ListFromConcat(typeref, p[2])
    740 
    741     if len(p) == 6:
    742       p[0] = self.BuildProduction('Sequence', p, 1, ListFromConcat(p[3], p[5]))
    743 
    744 
    745   # [78]
    746   def p_ConstType(self,  p):
    747     """ConstType : PrimitiveType Null
    748                  | identifier Null"""
    749     if type(p[1]) == str:
    750       p[0] = self.BuildNamed('Typeref', p, 1, p[2])
    751     else:
    752       p[1].AddChildren(p[2])
    753       p[0] = p[1]
    754 
    755 
    756   # [79] Added BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP
    757   def p_PrimitiveType(self, p):
    758     """PrimitiveType : UnsignedIntegerType
    759                      | UnrestrictedFloatType
    760                      | BOOLEAN
    761                      | BYTE
    762                      | OCTET
    763                      | BYTESTRING
    764                      | DOMSTRING
    765                      | OBJECT
    766                      | DATE
    767                      | REGEXP"""
    768     if type(p[1]) == str:
    769       p[0] = self.BuildNamed('PrimitiveType', p, 1)
    770     else:
    771       p[0] = p[1]
    772 
    773 
    774   # [80]
    775   def p_UnrestrictedFloatType(self, p):
    776     """UnrestrictedFloatType : UNRESTRICTED FloatType
    777                              | FloatType"""
    778     if len(p) == 2:
    779       typeref = self.BuildNamed('PrimitiveType', p, 1)
    780     else:
    781       typeref = self.BuildNamed('PrimitiveType', p, 2)
    782       typeref.AddChildren(self.BuildTrue('UNRESTRICTED'))
    783     p[0] = typeref
    784 
    785 
    786   # [81]
    787   def p_FloatType(self, p):
    788     """FloatType : FLOAT
    789                  | DOUBLE"""
    790     p[0] = p[1]
    791 
    792   # [82]
    793   def p_UnsignedIntegerType(self, p):
    794     """UnsignedIntegerType : UNSIGNED IntegerType
    795                            | IntegerType"""
    796     if len(p) == 2:
    797       p[0] = p[1]
    798     else:
    799       p[0] = 'unsigned ' + p[2]
    800 
    801   # [83]
    802   def p_IntegerType(self, p):
    803     """IntegerType : SHORT
    804                    | LONG OptionalLong"""
    805     if len(p) == 2:
    806       p[0] = p[1]
    807     else:
    808       p[0] = p[1] + p[2]
    809 
    810   # [84]
    811   def p_OptionalLong(self, p):
    812     """OptionalLong : LONG
    813                     | """
    814     if len(p) > 1:
    815       p[0] = ' ' + p[1]
    816     else:
    817       p[0] = ''
    818 
    819 
    820   # [85] Add support for sized array
    821   def p_TypeSuffix(self, p):
    822     """TypeSuffix : '[' integer ']' TypeSuffix
    823                   | '[' ']' TypeSuffix
    824                   | '?' TypeSuffixStartingWithArray
    825                   | """
    826     if len(p) == 5:
    827       p[0] = self.BuildNamed('Array', p, 2, p[4])
    828 
    829     if len(p) == 4:
    830       p[0] = self.BuildProduction('Array', p, 1, p[3])
    831 
    832     if len(p) == 3:
    833       p[0] = ListFromConcat(self.BuildTrue('NULLABLE'), p[2])
    834 
    835 
    836   # [86]
    837   def p_TypeSuffixStartingWithArray(self, p):
    838     """TypeSuffixStartingWithArray : '[' ']' TypeSuffix
    839                                    | """
    840     if len(p) > 1:
    841       p[0] = self.BuildProduction('Array', p, 0, p[3])
    842 
    843   # [87]
    844   def p_Null(self, p):
    845     """Null : '?'
    846             |"""
    847     if len(p) > 1:
    848       p[0] = self.BuildTrue('NULLABLE')
    849 
    850   # [88]
    851   def p_ReturnType(self, p):
    852     """ReturnType : Type
    853                   | VOID"""
    854     if p[1] == 'void':
    855       p[0] = self.BuildProduction('Type', p, 1)
    856       p[0].AddChildren(self.BuildNamed('PrimitiveType', p, 1))
    857     else:
    858       p[0] = p[1]
    859 
    860   # [89]
    861   def p_IdentifierList(self, p):
    862     """IdentifierList : identifier Identifiers"""
    863     p[0] = ListFromConcat(p[1], p[2])
    864 
    865   # [90]
    866   def p_Identifiers(self, p):
    867     """Identifiers : ',' identifier Identifiers
    868                    |"""
    869     if len(p) > 1:
    870       p[0] = ListFromConcat(p[2], p[3])
    871 
    872   # [91]
    873   def p_ExtendedAttributeNoArgs(self, p):
    874     """ExtendedAttributeNoArgs : identifier"""
    875     p[0] = self.BuildNamed('ExtAttribute', p, 1)
    876 
    877   # [92]
    878   def p_ExtendedAttributeArgList(self, p):
    879     """ExtendedAttributeArgList : identifier '(' ArgumentList ')'"""
    880     arguments = self.BuildProduction('Arguments', p, 2, p[3])
    881     p[0] = self.BuildNamed('ExtAttribute', p, 1, arguments)
    882 
    883   # [93]
    884   def p_ExtendedAttributeIdent(self, p):
    885     """ExtendedAttributeIdent : identifier '=' identifier"""
    886     value = self.BuildAttribute('VALUE', p[3])
    887     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
    888 
    889   # [94]
    890   def p_ExtendedAttributeIdentList(self, p):
    891     """ExtendedAttributeIdentList : identifier '=' '(' IdentifierList ')'"""
    892     value = self.BuildAttribute('VALUE', p[4])
    893     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
    894 
    895   # [95]
    896   def p_ExtendedAttributeNamedArgList(self, p):
    897     """ExtendedAttributeNamedArgList : identifier '=' identifier '(' ArgumentList ')'"""
    898     args = self.BuildProduction('Arguments', p, 4, p[5])
    899     value = self.BuildNamed('Call', p, 3, args)
    900     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
    901 
    902   # [96] NOT IMPLEMENTED (ExtendedAttributeTypePair)
    903 
    904 #
    905 # Parser Errors
    906 #
    907 # p_error is called whenever the parser can not find a pattern match for
    908 # a set of items from the current state.  The p_error function defined here
    909 # is triggered logging an error, and parsing recovery happens as the
    910 # p_<type>_error functions defined above are called.  This allows the parser
    911 # to continue so as to capture more than one error per file.
    912 #
    913   def p_error(self, t):
    914     if t:
    915       lineno = t.lineno
    916       pos = t.lexpos
    917       prev = self.yaccobj.symstack[-1]
    918       if type(prev) == lex.LexToken:
    919         msg = "Unexpected %s after %s." % (
    920             TokenTypeName(t), TokenTypeName(prev))
    921       else:
    922         msg = "Unexpected %s." % (t.value)
    923     else:
    924       last = self.LastToken()
    925       lineno = last.lineno
    926       pos = last.lexpos
    927       msg = "Unexpected end of file after %s." % TokenTypeName(last)
    928       self.yaccobj.restart()
    929 
    930     # Attempt to remap the error to a friendlier form
    931     if msg in ERROR_REMAP:
    932       msg = ERROR_REMAP[msg]
    933 
    934     self._last_error_msg = msg
    935     self._last_error_lineno = lineno
    936     self._last_error_pos = pos
    937 
    938   def Warn(self, node, msg):
    939     sys.stdout.write(node.GetLogLine(msg))
    940     self.parse_warnings += 1
    941 
    942   def LastToken(self):
    943     return self.lexer.last
    944 
    945   def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
    946     self.lexer = lexer
    947     self.tokens = lexer.KnownTokens()
    948     self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=debug,
    949                              optimize=0, write_tables=0)
    950     self.parse_debug = debug
    951     self.verbose = verbose
    952     self.mute_error = mute_error
    953     self._parse_errors = 0
    954     self._parse_warnings = 0
    955     self._last_error_msg = None
    956     self._last_error_lineno = 0
    957     self._last_error_pos = 0
    958 
    959 
    960 #
    961 # BuildProduction
    962 #
    963 # Production is the set of items sent to a grammar rule resulting in a new
    964 # item being returned.
    965 #
    966 # p - Is the Yacc production object containing the stack of items
    967 # index - Index into the production of the name for the item being produced.
    968 # cls - The type of item being producted
    969 # childlist - The children of the new item
    970   def BuildProduction(self, cls, p, index, childlist=None):
    971     try:
    972       if not childlist:
    973         childlist = []
    974 
    975       filename = self.lexer.Lexer().filename
    976       lineno = p.lineno(index)
    977       pos = p.lexpos(index)
    978       out = IDLNode(cls, filename, lineno, pos, childlist)
    979       return out
    980     except:
    981       print 'Exception while parsing:'
    982       for num, item in enumerate(p):
    983         print '  [%d] %s' % (num, ExpandProduction(item))
    984       if self.LastToken():
    985         print 'Last token: %s' % str(self.LastToken())
    986       raise
    987 
    988   def BuildNamed(self, cls, p, index, childlist=None):
    989     childlist = ListFromConcat(childlist)
    990     childlist.append(self.BuildAttribute('NAME', p[index]))
    991     return self.BuildProduction(cls, p, index, childlist)
    992 
    993   def BuildComment(self, cls, p, index):
    994     name = p[index]
    995 
    996     # Remove comment markers
    997     lines = []
    998     if name[:2] == '//':
    999       # For C++ style, remove any leading whitespace and the '//' marker from
   1000       # each line.
   1001       form = 'cc'
   1002       for line in name.split('\n'):
   1003         start = line.find('//')
   1004         lines.append(line[start+2:])
   1005     else:
   1006       # For C style, remove ending '*/''
   1007       form = 'c'
   1008       for line in name[:-2].split('\n'):
   1009         # Remove characters until start marker for this line '*' if found
   1010         # otherwise it should be blank.
   1011         offs = line.find('*')
   1012         if offs >= 0:
   1013           line = line[offs + 1:].rstrip()
   1014         else:
   1015           line = ''
   1016         lines.append(line)
   1017     name = '\n'.join(lines)
   1018     childlist = [self.BuildAttribute('NAME', name),
   1019                  self.BuildAttribute('FORM', form)]
   1020     return self.BuildProduction(cls, p, index, childlist)
   1021 
   1022 #
   1023 # BuildError
   1024 #
   1025 # Build and Errror node as part of the recovery process.
   1026 #
   1027 #
   1028   def BuildError(self, p, prod):
   1029     self._parse_errors += 1
   1030     name = self.BuildAttribute('NAME', self._last_error_msg)
   1031     line = self.BuildAttribute('LINE', self._last_error_lineno)
   1032     pos = self.BuildAttribute('POS', self._last_error_pos)
   1033     prod = self.BuildAttribute('PROD', prod)
   1034 
   1035     node = self.BuildProduction('Error', p, 1,
   1036                                 ListFromConcat(name, line, pos, prod))
   1037     if not self.mute_error:
   1038       node.Error(self._last_error_msg)
   1039 
   1040     return node
   1041 
   1042 #
   1043 # BuildAttribute
   1044 #
   1045 # An ExtendedAttribute is a special production that results in a property
   1046 # which is applied to the adjacent item.  Attributes have no children and
   1047 # instead represent key/value pairs.
   1048 #
   1049   def BuildAttribute(self, key, val):
   1050     return IDLAttribute(key, val)
   1051 
   1052   def BuildFalse(self, key):
   1053     return IDLAttribute(key, Boolean(False))
   1054 
   1055   def BuildTrue(self, key):
   1056     return IDLAttribute(key, Boolean(True))
   1057 
   1058   def GetErrors(self):
   1059     # Access lexer errors, despite being private
   1060     # pylint: disable=W0212
   1061     return self._parse_errors + self.lexer._lex_errors
   1062 
   1063 #
   1064 # ParseData
   1065 #
   1066 # Attempts to parse the current data loaded in the lexer.
   1067 #
   1068   def ParseText(self, filename, data):
   1069     self._parse_errors = 0
   1070     self._parse_warnings = 0
   1071     self._last_error_msg = None
   1072     self._last_error_lineno = 0
   1073     self._last_error_pos = 0
   1074 
   1075     try:
   1076       self.lexer.Tokenize(data, filename)
   1077       nodes = self.yaccobj.parse(lexer=self.lexer) or []
   1078       name = self.BuildAttribute('NAME', filename)
   1079       return IDLNode('File', filename, 0, 0, nodes + [name])
   1080 
   1081     except lex.LexError as lexError:
   1082       sys.stderr.write('Error in token: %s\n' % str(lexError))
   1083     return None
   1084 
   1085 
   1086 
   1087 def ParseFile(parser, filename):
   1088   """Parse a file and return a File type of node."""
   1089   with open(filename) as fileobject:
   1090     try:
   1091       out = parser.ParseText(filename, fileobject.read())
   1092       out.SetProperty('DATETIME', time.ctime(os.path.getmtime(filename)))
   1093       out.SetProperty('ERRORS', parser.GetErrors())
   1094       return out
   1095 
   1096     except Exception as e:
   1097       last = parser.LastToken()
   1098       sys.stderr.write('%s(%d) : Internal parsing error\n\t%s.\n' % (
   1099                        filename, last.lineno, str(e)))
   1100 
   1101 
   1102 def main(argv):
   1103   nodes = []
   1104   parser = IDLParser(IDLLexer())
   1105   errors = 0
   1106   for filename in argv:
   1107     filenode = ParseFile(parser, filename)
   1108     if (filenode):
   1109       errors += filenode.GetProperty('ERRORS')
   1110       nodes.append(filenode)
   1111 
   1112   ast = IDLNode('AST', '__AST__', 0, 0, nodes)
   1113 
   1114   print '\n'.join(ast.Tree(accept_props=['PROD']))
   1115   if errors:
   1116     print '\nFound %d errors.\n' % errors
   1117 
   1118   return errors
   1119 
   1120 
   1121 if __name__ == '__main__':
   1122   sys.exit(main(sys.argv[1:]))
   1123