1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """ Lexer for PPAPI IDL 7 8 The lexer uses the PLY library to build a tokenizer which understands both 9 WebIDL and Pepper tokens. 10 11 WebIDL, and WebIDL regular expressions can be found at: 12 http://www.w3.org/TR/2012/CR-WebIDL-20120419/ 13 PLY can be found at: 14 http://www.dabeaz.com/ply/ 15 """ 16 17 from idl_lexer import IDLLexer 18 import optparse 19 import os.path 20 import sys 21 22 23 # 24 # IDL PPAPI Lexer 25 # 26 class IDLPPAPILexer(IDLLexer): 27 # Special multi-character operators 28 def t_LSHIFT(self, t): 29 r'<<' 30 return t; 31 32 def t_RSHIFT(self, t): 33 r'>>' 34 return t; 35 36 def t_INLINE(self, t): 37 r'\#inline (.|\n)*?\#endinl.*' 38 self.AddLines(t.value.count('\n')) 39 return t 40 41 # Return a "preprocessor" inline block 42 def __init__(self): 43 IDLLexer.__init__(self) 44 self._AddTokens(['INLINE', 'LSHIFT', 'RSHIFT']) 45 self._AddKeywords(['label', 'struct']) 46 47 # Add number types 48 self._AddKeywords(['char', 'int8_t', 'int16_t', 'int32_t', 'int64_t']); 49 self._AddKeywords(['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t']); 50 self._AddKeywords(['double_t', 'float_t']); 51 52 # Add handle types 53 self._AddKeywords(['handle_t', 'PP_FileHandle']); 54 55 # Add pointer types (void*, char*, const char*, const void*) 56 self._AddKeywords(['mem_t', 'str_t', 'cstr_t', 'interface_t']); 57 58 # Remove JS types 59 self._DelKeywords(['boolean', 'byte', 'Date', 'DOMString', 'double', 60 'float', 'long', 'object', 'octet', 'short', 'unsigned']) 61 62 63 # If run by itself, attempt to build the lexer 64 if __name__ == '__main__': 65 lexer = IDLPPAPILexer() 66 lexer.Tokenize(open('test_parser/inline_ppapi.idl').read()) 67 for tok in lexer.GetTokens(): 68 print '\n' + str(tok)
