1 import unittest 2 import imp 3 import os 4 import errno 5 import sys 6 import glob 7 import re 8 import tempfile 9 import shutil 10 import inspect 11 import hashlib 12 from distutils.errors import * 13 import antlr3 14 15 def unlink(path): 16 try: 17 os.unlink(path) 18 except OSError, exc: 19 if exc.errno != errno.ENOENT: 20 raise 21 22 23 class GrammarCompileError(Exception): 24 """Grammar failed to compile.""" 25 pass 26 27 28 # At least on MacOSX tempdir (/tmp) is a symlink. It's sometimes dereferences, 29 # sometimes not, breaking the inspect.getmodule() function. 30 testbasedir = os.path.join( 31 os.path.realpath(tempfile.gettempdir()), 32 'antlr3-test') 33 34 35 class BrokenTest(unittest.TestCase.failureException): 36 def __repr__(self): 37 name, reason = self.args 38 return '%s: %s: %s works now' % ( 39 (self.__class__.__name__, name, reason)) 40 41 42 def broken(reason, *exceptions): 43 '''Indicates a failing (or erroneous) test case fails that should succeed. 44 If the test fails with an exception, list the exception type in args''' 45 def wrapper(test_method): 46 def replacement(*args, **kwargs): 47 try: 48 test_method(*args, **kwargs) 49 except exceptions or unittest.TestCase.failureException: 50 pass 51 else: 52 raise BrokenTest(test_method.__name__, reason) 53 replacement.__doc__ = test_method.__doc__ 54 replacement.__name__ = 'XXX_' + test_method.__name__ 55 replacement.todo = reason 56 return replacement 57 return wrapper 58 59 60 dependencyCache = {} 61 compileErrorCache = {} 62 63 # setup java CLASSPATH 64 if 'CLASSPATH' not in os.environ: 65 cp = [] 66 67 baseDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 68 libDir = os.path.join(baseDir, 'lib') 69 70 jar = os.path.join(libDir, 'ST-4.0.1.jar') 71 if not os.path.isfile(jar): 72 raise DistutilsFileError( 73 "Missing file '%s'. Grap it from a distribution package." 74 % jar, 75 ) 76 cp.append(jar) 77 78 jar = os.path.join(libDir, 'antlr-2.7.7.jar') 79 if not os.path.isfile(jar): 80 raise DistutilsFileError( 81 "Missing file '%s'. Grap it from a distribution package." 82 % jar, 83 ) 84 cp.append(jar) 85 86 jar = os.path.join(libDir, 'junit-4.2.jar') 87 if not os.path.isfile(jar): 88 raise DistutilsFileError( 89 "Missing file '%s'. Grap it from a distribution package." 90 % jar, 91 ) 92 cp.append(jar) 93 94 cp.append(os.path.join(baseDir, 'runtime', 'Python', 'build')) 95 96 classpath = '-cp "' + ':'.join([os.path.abspath(p) for p in cp]) + '"' 97 98 else: 99 classpath = '' 100 101 102 class ANTLRTest(unittest.TestCase): 103 def __init__(self, *args, **kwargs): 104 unittest.TestCase.__init__(self, *args, **kwargs) 105 106 self.moduleName = os.path.splitext(os.path.basename(sys.modules[self.__module__].__file__))[0] 107 self.className = self.__class__.__name__ 108 self._baseDir = None 109 110 self.lexerModule = None 111 self.parserModule = None 112 113 self.grammarName = None 114 self.grammarType = None 115 116 117 def assertListEqual(self, a, b): 118 if a == b: 119 return 120 121 import difflib 122 a = [str(l) + '\n' for l in a] 123 b = [str(l) + '\n' for l in b] 124 125 raise AssertionError(''.join(difflib.unified_diff(a, b))) 126 127 128 @property 129 def baseDir(self): 130 if self._baseDir is None: 131 testName = 'unknownTest' 132 for frame in inspect.stack(): 133 code = frame[0].f_code 134 codeMod = inspect.getmodule(code) 135 if codeMod is None: 136 continue 137 138 # skip frames not in requested module 139 if codeMod is not sys.modules[self.__module__]: 140 continue 141 142 # skip some unwanted names 143 if code.co_name in ('nextToken', '<module>'): 144 continue 145 146 if code.co_name.startswith('test'): 147 testName = code.co_name 148 break 149 150 self._baseDir = os.path.join( 151 testbasedir, 152 self.moduleName, self.className, testName) 153 if not os.path.isdir(self._baseDir): 154 os.makedirs(self._baseDir) 155 156 return self._baseDir 157 158 159 def _invokeantlr(self, dir, file, options, javaOptions=''): 160 cmd = 'cd %s; java %s %s org.antlr.Tool -o . %s %s 2>&1' % ( 161 dir, javaOptions, classpath, options, file 162 ) 163 fp = os.popen(cmd) 164 output = '' 165 failed = False 166 for line in fp: 167 output += line 168 169 if line.startswith('error('): 170 failed = True 171 172 rc = fp.close() 173 if rc is not None: 174 failed = True 175 176 if failed: 177 raise GrammarCompileError( 178 "Failed to compile grammar '%s':\n%s\n\n" % (file, cmd) 179 + output 180 ) 181 182 183 def compileGrammar(self, grammarName=None, options='', javaOptions=''): 184 if grammarName is None: 185 grammarName = self.moduleName + '.g' 186 187 self._baseDir = os.path.join( 188 testbasedir, 189 self.moduleName) 190 if not os.path.isdir(self._baseDir): 191 os.makedirs(self._baseDir) 192 193 if self.grammarName is None: 194 self.grammarName = os.path.splitext(grammarName)[0] 195 196 grammarPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), grammarName) 197 198 # get type and name from first grammar line 199 grammar = open(grammarPath, 'r').read() 200 m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE) 201 assert m is not None, grammar 202 self.grammarType = m.group(2) 203 if self.grammarType is None: 204 self.grammarType = 'combined' 205 206 if self.grammarType is None: 207 assert self.grammarType in ('lexer', 'parser', 'tree', 'combined'), self.grammarType 208 209 # don't try to rebuild grammar, if it already failed 210 if grammarName in compileErrorCache: 211 return 212 213 try: 214 # # get dependencies from antlr 215 # if grammarName in dependencyCache: 216 # dependencies = dependencyCache[grammarName] 217 218 # else: 219 # dependencies = [] 220 # cmd = ('cd %s; java %s %s org.antlr.Tool -o . -depend %s 2>&1' 221 # % (self.baseDir, javaOptions, classpath, grammarPath)) 222 223 # output = "" 224 # failed = False 225 226 # fp = os.popen(cmd) 227 # for line in fp: 228 # output += line 229 230 # if line.startswith('error('): 231 # failed = True 232 # elif ':' in line: 233 # a, b = line.strip().split(':', 1) 234 # dependencies.append( 235 # (os.path.join(self.baseDir, a.strip()), 236 # [os.path.join(self.baseDir, b.strip())]) 237 # ) 238 239 # rc = fp.close() 240 # if rc is not None: 241 # failed = True 242 243 # if failed: 244 # raise GrammarCompileError( 245 # "antlr -depend failed with code %s on grammar '%s':\n\n" 246 # % (rc, grammarName) 247 # + cmd 248 # + "\n" 249 # + output 250 # ) 251 252 # # add dependencies to my .stg files 253 # templateDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tool', 'src', 'main', 'resources', 'org', 'antlr', 'codegen', 'templates', 'Python')) 254 # templates = glob.glob(os.path.join(templateDir, '*.stg')) 255 256 # for dst, src in dependencies: 257 # src.extend(templates) 258 259 # dependencyCache[grammarName] = dependencies 260 261 # rebuild = False 262 # for dest, sources in dependencies: 263 # if not os.path.isfile(dest): 264 # rebuild = True 265 # break 266 267 # for source in sources: 268 # if os.path.getmtime(source) > os.path.getmtime(dest): 269 # rebuild = True 270 # break 271 272 273 # if rebuild: 274 # self._invokeantlr(self.baseDir, grammarPath, options, javaOptions) 275 276 self._invokeantlr(self.baseDir, grammarPath, options, javaOptions) 277 278 except: 279 # mark grammar as broken 280 compileErrorCache[grammarName] = True 281 raise 282 283 284 def lexerClass(self, base): 285 """Optionally build a subclass of generated lexer class""" 286 287 return base 288 289 290 def parserClass(self, base): 291 """Optionally build a subclass of generated parser class""" 292 293 return base 294 295 296 def walkerClass(self, base): 297 """Optionally build a subclass of generated walker class""" 298 299 return base 300 301 302 def __load_module(self, name): 303 modFile, modPathname, modDescription \ 304 = imp.find_module(name, [self.baseDir]) 305 306 return imp.load_module( 307 name, modFile, modPathname, modDescription 308 ) 309 310 311 def getLexer(self, *args, **kwargs): 312 """Build lexer instance. Arguments are passed to lexer.__init__().""" 313 314 if self.grammarType == 'lexer': 315 self.lexerModule = self.__load_module(self.grammarName) 316 cls = getattr(self.lexerModule, self.grammarName) 317 else: 318 self.lexerModule = self.__load_module(self.grammarName + 'Lexer') 319 cls = getattr(self.lexerModule, self.grammarName + 'Lexer') 320 321 cls = self.lexerClass(cls) 322 323 lexer = cls(*args, **kwargs) 324 325 return lexer 326 327 328 def getParser(self, *args, **kwargs): 329 """Build parser instance. Arguments are passed to parser.__init__().""" 330 331 if self.grammarType == 'parser': 332 self.lexerModule = self.__load_module(self.grammarName) 333 cls = getattr(self.lexerModule, self.grammarName) 334 else: 335 self.parserModule = self.__load_module(self.grammarName + 'Parser') 336 cls = getattr(self.parserModule, self.grammarName + 'Parser') 337 cls = self.parserClass(cls) 338 339 parser = cls(*args, **kwargs) 340 341 return parser 342 343 344 def getWalker(self, *args, **kwargs): 345 """Build walker instance. Arguments are passed to walker.__init__().""" 346 347 self.walkerModule = self.__load_module(self.grammarName + 'Walker') 348 cls = getattr(self.walkerModule, self.grammarName + 'Walker') 349 cls = self.walkerClass(cls) 350 351 walker = cls(*args, **kwargs) 352 353 return walker 354 355 356 def writeInlineGrammar(self, grammar): 357 # Create a unique ID for this test and use it as the grammar name, 358 # to avoid class name reuse. This kinda sucks. Need to find a way so 359 # tests can use the same grammar name without messing up the namespace. 360 # Well, first I should figure out what the exact problem is... 361 id = hashlib.md5(self.baseDir).hexdigest()[-8:] 362 grammar = grammar.replace('$TP', 'TP' + id) 363 grammar = grammar.replace('$T', 'T' + id) 364 365 # get type and name from first grammar line 366 m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE) 367 assert m is not None, grammar 368 grammarType = m.group(2) 369 if grammarType is None: 370 grammarType = 'combined' 371 grammarName = m.group(3) 372 373 assert grammarType in ('lexer', 'parser', 'tree', 'combined'), grammarType 374 375 grammarPath = os.path.join(self.baseDir, grammarName + '.g') 376 377 # dump temp grammar file 378 fp = open(grammarPath, 'w') 379 fp.write(grammar) 380 fp.close() 381 382 return grammarName, grammarPath, grammarType 383 384 385 def writeFile(self, name, contents): 386 testDir = os.path.dirname(os.path.abspath(__file__)) 387 path = os.path.join(self.baseDir, name) 388 389 fp = open(path, 'w') 390 fp.write(contents) 391 fp.close() 392 393 return path 394 395 396 def compileInlineGrammar(self, grammar, options='', javaOptions='', 397 returnModule=False): 398 # write grammar file 399 grammarName, grammarPath, grammarType = self.writeInlineGrammar(grammar) 400 401 # compile it 402 self._invokeantlr( 403 os.path.dirname(grammarPath), 404 os.path.basename(grammarPath), 405 options, 406 javaOptions 407 ) 408 409 if grammarType == 'combined': 410 lexerMod = self.__load_module(grammarName + 'Lexer') 411 parserMod = self.__load_module(grammarName + 'Parser') 412 if returnModule: 413 return lexerMod, parserMod 414 415 lexerCls = getattr(lexerMod, grammarName + 'Lexer') 416 lexerCls = self.lexerClass(lexerCls) 417 parserCls = getattr(parserMod, grammarName + 'Parser') 418 parserCls = self.parserClass(parserCls) 419 420 return lexerCls, parserCls 421 422 if grammarType == 'lexer': 423 lexerMod = self.__load_module(grammarName) 424 if returnModule: 425 return lexerMod 426 427 lexerCls = getattr(lexerMod, grammarName) 428 lexerCls = self.lexerClass(lexerCls) 429 430 return lexerCls 431 432 if grammarType == 'parser': 433 parserMod = self.__load_module(grammarName) 434 if returnModule: 435 return parserMod 436 437 parserCls = getattr(parserMod, grammarName) 438 parserCls = self.parserClass(parserCls) 439 440 return parserCls 441 442 if grammarType == 'tree': 443 walkerMod = self.__load_module(grammarName) 444 if returnModule: 445 return walkerMod 446 447 walkerCls = getattr(walkerMod, grammarName) 448 walkerCls = self.walkerClass(walkerCls) 449 450 return walkerCls 451
