1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """ Lexer for PPAPI IDL """ 7 8 # 9 # IDL Lexer 10 # 11 # The lexer is uses the PLY lex library to build a tokenizer which understands 12 # WebIDL tokens. 13 # 14 # WebIDL, and WebIDL regular expressions can be found at: 15 # http://dev.w3.org/2006/webapi/WebIDL/ 16 # PLY can be found at: 17 # http://www.dabeaz.com/ply/ 18 19 import os.path 20 import re 21 import sys 22 23 # 24 # Try to load the ply module, if not, then assume it is in the third_party 25 # directory, relative to ppapi 26 # 27 try: 28 from ply import lex 29 except: 30 module_path, module_name = os.path.split(__file__) 31 third_party = os.path.join(module_path, '..', '..', 'third_party') 32 sys.path.append(third_party) 33 from ply import lex 34 35 from idl_option import GetOption, Option, ParseOptions 36 37 38 Option('output', 'Generate output.') 39 40 # 41 # IDL Lexer 42 # 43 class IDLLexer(object): 44 # 'tokens' is a value required by lex which specifies the complete list 45 # of valid token types. 46 tokens = [ 47 # Symbol and keywords types 48 'COMMENT', 49 'DESCRIBE', 50 'ENUM', 51 'LABEL', 52 'SYMBOL', 53 'INLINE', 54 'INTERFACE', 55 'STRUCT', 56 'TYPEDEF', 57 58 # Extra WebIDL keywords 59 'CALLBACK', 60 'DICTIONARY', 61 'OPTIONAL', 62 'STATIC', 63 64 # Invented for apps use 65 'NAMESPACE', 66 67 # Data types 68 'FLOAT', 69 'OCT', 70 'INT', 71 'HEX', 72 'STRING', 73 74 # Operators 75 'LSHIFT', 76 'RSHIFT' 77 ] 78 79 # 'keywords' is a map of string to token type. All SYMBOL tokens are 80 # matched against keywords, to determine if the token is actually a keyword. 81 keywords = { 82 'describe' : 'DESCRIBE', 83 'enum' : 'ENUM', 84 'label' : 'LABEL', 85 'interface' : 'INTERFACE', 86 'readonly' : 'READONLY', 87 'struct' : 'STRUCT', 88 'typedef' : 'TYPEDEF', 89 90 'callback' : 'CALLBACK', 91 'dictionary' : 'DICTIONARY', 92 'optional' : 'OPTIONAL', 93 'static' : 'STATIC', 94 'namespace' : 'NAMESPACE', 95 } 96 97 # 'literals' is a value expected by lex which specifies a list of valid 98 # literal tokens, meaning the token type and token value are identical. 99 literals = '"*.(){}[],;:=+-/~|&^?' 100 101 # Token definitions 102 # 103 # Lex assumes any value or function in the form of 't_<TYPE>' represents a 104 # regular expression where a match will emit a token of type <TYPE>. In the 105 # case of a function, the function is called when a match is made. These 106 # definitions come from WebIDL. 107 108 # 't_ignore' is a special match of items to ignore 109 t_ignore = ' \t' 110 111 # Constant values 112 t_FLOAT = r'-?(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?|-?\d+[Ee][+-]?\d+' 113 t_INT = r'-?[0-9]+[uU]?' 114 t_OCT = r'-?0[0-7]+' 115 t_HEX = r'-?0[Xx][0-9A-Fa-f]+' 116 t_LSHIFT = r'<<' 117 t_RSHIFT = r'>>' 118 119 # A line ending '\n', we use this to increment the line number 120 def t_LINE_END(self, t): 121 r'\n+' 122 self.AddLines(len(t.value)) 123 124 # We do not process escapes in the IDL strings. Strings are exclusively 125 # used for attributes, and not used as typical 'C' constants. 126 def t_STRING(self, t): 127 r'"[^"]*"' 128 t.value = t.value[1:-1] 129 self.AddLines(t.value.count('\n')) 130 return t 131 132 # A C or C++ style comment: /* xxx */ or // 133 def t_COMMENT(self, t): 134 r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' 135 self.AddLines(t.value.count('\n')) 136 return t 137 138 # Return a "preprocessor" inline block 139 def t_INLINE(self, t): 140 r'\#inline (.|\n)*?\#endinl.*' 141 self.AddLines(t.value.count('\n')) 142 return t 143 144 # A symbol or keyword. 145 def t_KEYWORD_SYMBOL(self, t): 146 r'_?[A-Za-z][A-Za-z_0-9]*' 147 148 # All non-keywords are assumed to be symbols 149 t.type = self.keywords.get(t.value, 'SYMBOL') 150 151 # We strip leading underscores so that you can specify symbols with the same 152 # value as a keywords (E.g. a dictionary named 'interface'). 153 if t.value[0] == '_': 154 t.value = t.value[1:] 155 return t 156 157 def t_ANY_error(self, t): 158 msg = "Unrecognized input" 159 line = self.lexobj.lineno 160 161 # If that line has not been accounted for, then we must have hit 162 # EoF, so compute the beginning of the line that caused the problem. 163 if line >= len(self.index): 164 # Find the offset in the line of the first word causing the issue 165 word = t.value.split()[0] 166 offs = self.lines[line - 1].find(word) 167 # Add the computed line's starting position 168 self.index.append(self.lexobj.lexpos - offs) 169 msg = "Unexpected EoF reached after" 170 171 pos = self.lexobj.lexpos - self.index[line] 172 file = self.lexobj.filename 173 out = self.ErrorMessage(file, line, pos, msg) 174 sys.stderr.write(out + '\n') 175 self.lex_errors += 1 176 177 178 def AddLines(self, count): 179 # Set the lexer position for the beginning of the next line. In the case 180 # of multiple lines, tokens can not exist on any of the lines except the 181 # last one, so the recorded value for previous lines are unused. We still 182 # fill the array however, to make sure the line count is correct. 183 self.lexobj.lineno += count 184 for i in range(count): 185 self.index.append(self.lexobj.lexpos) 186 187 def FileLineMsg(self, file, line, msg): 188 if file: return "%s(%d) : %s" % (file, line + 1, msg) 189 return "<BuiltIn> : %s" % msg 190 191 def SourceLine(self, file, line, pos): 192 caret = '\t^'.expandtabs(pos) 193 # We decrement the line number since the array is 0 based while the 194 # line numbers are 1 based. 195 return "%s\n%s" % (self.lines[line - 1], caret) 196 197 def ErrorMessage(self, file, line, pos, msg): 198 return "\n%s\n%s" % ( 199 self.FileLineMsg(file, line, msg), 200 self.SourceLine(file, line, pos)) 201 202 def SetData(self, filename, data): 203 # Start with line 1, not zero 204 self.lexobj.lineno = 1 205 self.lexobj.filename = filename 206 self.lines = data.split('\n') 207 self.index = [0] 208 self.lexobj.input(data) 209 self.lex_errors = 0 210 211 def __init__(self): 212 self.lexobj = lex.lex(object=self, lextab=None, optimize=0) 213 214 215 216 # 217 # FilesToTokens 218 # 219 # From a set of source file names, generate a list of tokens. 220 # 221 def FilesToTokens(filenames, verbose=False): 222 lexer = IDLLexer() 223 outlist = [] 224 for filename in filenames: 225 data = open(filename).read() 226 lexer.SetData(filename, data) 227 if verbose: sys.stdout.write(' Loaded %s...\n' % filename) 228 while 1: 229 t = lexer.lexobj.token() 230 if t is None: break 231 outlist.append(t) 232 return outlist 233 234 235 def TokensFromText(text): 236 lexer = IDLLexer() 237 lexer.SetData('unknown', text) 238 outlist = [] 239 while 1: 240 t = lexer.lexobj.token() 241 if t is None: break 242 outlist.append(t.value) 243 return outlist 244 245 # 246 # TextToTokens 247 # 248 # From a block of text, generate a list of tokens 249 # 250 def TextToTokens(source): 251 lexer = IDLLexer() 252 outlist = [] 253 lexer.SetData('AUTO', source) 254 while 1: 255 t = lexer.lexobj.token() 256 if t is None: break 257 outlist.append(t.value) 258 return outlist 259 260 261 # 262 # TestSame 263 # 264 # From a set of token values, generate a new source text by joining with a 265 # single space. The new source is then tokenized and compared against the 266 # old set. 267 # 268 def TestSame(values1): 269 # Recreate the source from the tokens. We use newline instead of whitespace 270 # since the '//' and #inline regex are line sensitive. 271 text = '\n'.join(values1) 272 values2 = TextToTokens(text) 273 274 count1 = len(values1) 275 count2 = len(values2) 276 if count1 != count2: 277 print "Size mismatch original %d vs %d\n" % (count1, count2) 278 if count1 > count2: count1 = count2 279 280 for i in range(count1): 281 if values1[i] != values2[i]: 282 print "%d >>%s<< >>%s<<" % (i, values1[i], values2[i]) 283 284 if GetOption('output'): 285 sys.stdout.write('Generating original.txt and tokenized.txt\n') 286 open('original.txt', 'w').write(src1) 287 open('tokenized.txt', 'w').write(src2) 288 289 if values1 == values2: 290 sys.stdout.write('Same: Pass\n') 291 return 0 292 293 print "****************\n%s\n%s***************\n" % (src1, src2) 294 sys.stdout.write('Same: Failed\n') 295 return -1 296 297 298 # 299 # TestExpect 300 # 301 # From a set of tokens pairs, verify the type field of the second matches 302 # the value of the first, so that: 303 # INT 123 FLOAT 1.1 304 # will generate a passing test, where the first token is the SYMBOL INT, 305 # and the second token is the INT 123, third token is the SYMBOL FLOAT and 306 # the fourth is the FLOAT 1.1, etc... 307 def TestExpect(tokens): 308 count = len(tokens) 309 index = 0 310 errors = 0 311 while index < count: 312 type = tokens[index].value 313 token = tokens[index + 1] 314 index += 2 315 316 if type != token.type: 317 sys.stderr.write('Mismatch: Expected %s, but got %s = %s.\n' % 318 (type, token.type, token.value)) 319 errors += 1 320 321 if not errors: 322 sys.stdout.write('Expect: Pass\n') 323 return 0 324 325 sys.stdout.write('Expect: Failed\n') 326 return -1 327 328 329 def Main(args): 330 filenames = ParseOptions(args) 331 332 try: 333 tokens = FilesToTokens(filenames, GetOption('verbose')) 334 values = [tok.value for tok in tokens] 335 if GetOption('output'): sys.stdout.write(' <> '.join(values) + '\n') 336 if GetOption('test'): 337 if TestSame(values): 338 return -1 339 if TestExpect(tokens): 340 return -1 341 return 0 342 343 except lex.LexError as le: 344 sys.stderr.write('%s\n' % str(le)) 345 return -1 346 347 348 if __name__ == '__main__': 349 sys.exit(Main(sys.argv[1:])) 350
