Home | History | Annotate | Download | only in tests
      1 import unittest
      2 import imp
      3 import os
      4 import errno
      5 import sys
      6 import glob
      7 import re
      8 import tempfile
      9 import shutil
     10 import inspect
     11 import hashlib
     12 from distutils.errors import *
     13 import antlr3
     14 
     15 def unlink(path):
     16     try:
     17         os.unlink(path)
     18     except OSError, exc:
     19         if exc.errno != errno.ENOENT:
     20             raise
     21 
     22 
     23 class GrammarCompileError(Exception):
     24   """Grammar failed to compile."""
     25   pass
     26 
     27 
     28 # At least on MacOSX tempdir (/tmp) is a symlink. It's sometimes dereferences,
     29 # sometimes not, breaking the inspect.getmodule() function.
     30 testbasedir = os.path.join(
     31     os.path.realpath(tempfile.gettempdir()),
     32     'antlr3-test')
     33 
     34 
     35 class BrokenTest(unittest.TestCase.failureException):
     36     def __repr__(self):
     37         name, reason = self.args
     38         return '%s: %s: %s works now' % (
     39             (self.__class__.__name__, name, reason))
     40 
     41 
     42 def broken(reason, *exceptions):
     43     '''Indicates a failing (or erroneous) test case fails that should succeed.
     44     If the test fails with an exception, list the exception type in args'''
     45     def wrapper(test_method):
     46         def replacement(*args, **kwargs):
     47             try:
     48                 test_method(*args, **kwargs)
     49             except exceptions or unittest.TestCase.failureException:
     50                 pass
     51             else:
     52                 raise BrokenTest(test_method.__name__, reason)
     53         replacement.__doc__ = test_method.__doc__
     54         replacement.__name__ = 'XXX_' + test_method.__name__
     55         replacement.todo = reason
     56         return replacement
     57     return wrapper
     58 
     59 
     60 dependencyCache = {}
     61 compileErrorCache = {}
     62 
     63 # setup java CLASSPATH
     64 if 'CLASSPATH' not in os.environ:
     65     cp = []
     66 
     67     baseDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
     68     libDir = os.path.join(baseDir, 'lib')
     69 
     70     jar = os.path.join(libDir, 'ST-4.0.1.jar')
     71     if not os.path.isfile(jar):
     72         raise DistutilsFileError(
     73             "Missing file '%s'. Grap it from a distribution package."
     74             % jar,
     75             )
     76     cp.append(jar)
     77 
     78     jar = os.path.join(libDir, 'antlr-2.7.7.jar')
     79     if not os.path.isfile(jar):
     80         raise DistutilsFileError(
     81             "Missing file '%s'. Grap it from a distribution package."
     82             % jar,
     83             )
     84     cp.append(jar)
     85 
     86     jar = os.path.join(libDir, 'junit-4.2.jar')
     87     if not os.path.isfile(jar):
     88         raise DistutilsFileError(
     89             "Missing file '%s'. Grap it from a distribution package."
     90             % jar,
     91             )
     92     cp.append(jar)
     93 
     94     cp.append(os.path.join(baseDir, 'runtime', 'Python', 'build'))
     95 
     96     classpath = '-cp "' + ':'.join([os.path.abspath(p) for p in cp]) + '"'
     97 
     98 else:
     99     classpath = ''
    100 
    101 
    102 class ANTLRTest(unittest.TestCase):
    103     def __init__(self, *args, **kwargs):
    104         unittest.TestCase.__init__(self, *args, **kwargs)
    105 
    106         self.moduleName = os.path.splitext(os.path.basename(sys.modules[self.__module__].__file__))[0]
    107         self.className = self.__class__.__name__
    108         self._baseDir = None
    109 
    110         self.lexerModule = None
    111         self.parserModule = None
    112 
    113         self.grammarName = None
    114         self.grammarType = None
    115 
    116 
    117     def assertListEqual(self, a, b):
    118         if a == b:
    119             return
    120 
    121         import difflib
    122         a = [str(l) + '\n' for l in a]
    123         b = [str(l) + '\n' for l in b]
    124 
    125         raise AssertionError(''.join(difflib.unified_diff(a, b)))
    126 
    127 
    128     @property
    129     def baseDir(self):
    130         if self._baseDir is None:
    131             testName = 'unknownTest'
    132             for frame in inspect.stack():
    133                 code = frame[0].f_code
    134                 codeMod = inspect.getmodule(code)
    135                 if codeMod is None:
    136                     continue
    137 
    138                 # skip frames not in requested module
    139                 if codeMod is not sys.modules[self.__module__]:
    140                     continue
    141 
    142                 # skip some unwanted names
    143                 if code.co_name in ('nextToken', '<module>'):
    144                     continue
    145 
    146                 if code.co_name.startswith('test'):
    147                     testName = code.co_name
    148                     break
    149 
    150             self._baseDir = os.path.join(
    151                 testbasedir,
    152                 self.moduleName, self.className, testName)
    153             if not os.path.isdir(self._baseDir):
    154                 os.makedirs(self._baseDir)
    155 
    156         return self._baseDir
    157 
    158 
    159     def _invokeantlr(self, dir, file, options, javaOptions=''):
    160         cmd = 'cd %s; java %s %s org.antlr.Tool -o . %s %s 2>&1' % (
    161             dir, javaOptions, classpath, options, file
    162             )
    163         fp = os.popen(cmd)
    164         output = ''
    165         failed = False
    166         for line in fp:
    167             output += line
    168 
    169             if line.startswith('error('):
    170                 failed = True
    171 
    172         rc = fp.close()
    173         if rc is not None:
    174             failed = True
    175 
    176         if failed:
    177             raise GrammarCompileError(
    178                 "Failed to compile grammar '%s':\n%s\n\n" % (file, cmd)
    179                 + output
    180                 )
    181 
    182 
    183     def compileGrammar(self, grammarName=None, options='', javaOptions=''):
    184         if grammarName is None:
    185             grammarName = self.moduleName + '.g'
    186 
    187         self._baseDir = os.path.join(
    188             testbasedir,
    189             self.moduleName)
    190         if not os.path.isdir(self._baseDir):
    191             os.makedirs(self._baseDir)
    192 
    193         if self.grammarName is None:
    194             self.grammarName = os.path.splitext(grammarName)[0]
    195 
    196         grammarPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), grammarName)
    197 
    198         # get type and name from first grammar line
    199         grammar = open(grammarPath, 'r').read()
    200         m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE)
    201         assert m is not None, grammar
    202         self.grammarType = m.group(2)
    203         if self.grammarType is None:
    204             self.grammarType = 'combined'
    205 
    206         if self.grammarType is None:
    207             assert self.grammarType in ('lexer', 'parser', 'tree', 'combined'), self.grammarType
    208 
    209         # don't try to rebuild grammar, if it already failed
    210         if grammarName in compileErrorCache:
    211             return
    212 
    213         try:
    214         #     # get dependencies from antlr
    215         #     if grammarName in dependencyCache:
    216         #         dependencies = dependencyCache[grammarName]
    217 
    218         #     else:
    219         #         dependencies = []
    220         #         cmd = ('cd %s; java %s %s org.antlr.Tool -o . -depend %s 2>&1'
    221         #                % (self.baseDir, javaOptions, classpath, grammarPath))
    222 
    223         #         output = ""
    224         #         failed = False
    225 
    226         #         fp = os.popen(cmd)
    227         #         for line in fp:
    228         #             output += line
    229 
    230         #             if line.startswith('error('):
    231         #                 failed = True
    232         #             elif ':' in line:
    233         #                 a, b = line.strip().split(':', 1)
    234         #                 dependencies.append(
    235         #                     (os.path.join(self.baseDir, a.strip()),
    236         #                      [os.path.join(self.baseDir, b.strip())])
    237         #                     )
    238 
    239         #         rc = fp.close()
    240         #         if rc is not None:
    241         #             failed = True
    242 
    243         #         if failed:
    244         #             raise GrammarCompileError(
    245         #                 "antlr -depend failed with code %s on grammar '%s':\n\n"
    246         #                 % (rc, grammarName)
    247         #                 + cmd
    248         #                 + "\n"
    249         #                 + output
    250         #                 )
    251 
    252         #         # add dependencies to my .stg files
    253         #         templateDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tool', 'src', 'main', 'resources', 'org', 'antlr', 'codegen', 'templates', 'Python'))
    254         #         templates = glob.glob(os.path.join(templateDir, '*.stg'))
    255 
    256         #         for dst, src in dependencies:
    257         #             src.extend(templates)
    258 
    259         #         dependencyCache[grammarName] = dependencies
    260 
    261         #     rebuild = False
    262         #     for dest, sources in dependencies:
    263         #         if not os.path.isfile(dest):
    264         #             rebuild = True
    265         #             break
    266 
    267         #         for source in sources:
    268         #             if os.path.getmtime(source) > os.path.getmtime(dest):
    269         #                 rebuild = True
    270         #                 break
    271 
    272 
    273         #     if rebuild:
    274         #         self._invokeantlr(self.baseDir, grammarPath, options, javaOptions)
    275 
    276             self._invokeantlr(self.baseDir, grammarPath, options, javaOptions)
    277 
    278         except:
    279             # mark grammar as broken
    280             compileErrorCache[grammarName] = True
    281             raise
    282 
    283 
    284     def lexerClass(self, base):
    285         """Optionally build a subclass of generated lexer class"""
    286 
    287         return base
    288 
    289 
    290     def parserClass(self, base):
    291         """Optionally build a subclass of generated parser class"""
    292 
    293         return base
    294 
    295 
    296     def walkerClass(self, base):
    297         """Optionally build a subclass of generated walker class"""
    298 
    299         return base
    300 
    301 
    302     def __load_module(self, name):
    303         modFile, modPathname, modDescription \
    304                  = imp.find_module(name, [self.baseDir])
    305 
    306         return imp.load_module(
    307             name, modFile, modPathname, modDescription
    308             )
    309 
    310 
    311     def getLexer(self, *args, **kwargs):
    312         """Build lexer instance. Arguments are passed to lexer.__init__()."""
    313 
    314         if self.grammarType == 'lexer':
    315             self.lexerModule = self.__load_module(self.grammarName)
    316             cls = getattr(self.lexerModule, self.grammarName)
    317         else:
    318             self.lexerModule = self.__load_module(self.grammarName + 'Lexer')
    319             cls = getattr(self.lexerModule, self.grammarName + 'Lexer')
    320 
    321         cls = self.lexerClass(cls)
    322 
    323         lexer = cls(*args, **kwargs)
    324 
    325         return lexer
    326 
    327 
    328     def getParser(self, *args, **kwargs):
    329         """Build parser instance. Arguments are passed to parser.__init__()."""
    330 
    331         if self.grammarType == 'parser':
    332             self.lexerModule = self.__load_module(self.grammarName)
    333             cls = getattr(self.lexerModule, self.grammarName)
    334         else:
    335             self.parserModule = self.__load_module(self.grammarName + 'Parser')
    336             cls = getattr(self.parserModule, self.grammarName + 'Parser')
    337         cls = self.parserClass(cls)
    338 
    339         parser = cls(*args, **kwargs)
    340 
    341         return parser
    342 
    343 
    344     def getWalker(self, *args, **kwargs):
    345         """Build walker instance. Arguments are passed to walker.__init__()."""
    346 
    347         self.walkerModule = self.__load_module(self.grammarName + 'Walker')
    348         cls = getattr(self.walkerModule, self.grammarName + 'Walker')
    349         cls = self.walkerClass(cls)
    350 
    351         walker = cls(*args, **kwargs)
    352 
    353         return walker
    354 
    355 
    356     def writeInlineGrammar(self, grammar):
    357         # Create a unique ID for this test and use it as the grammar name,
    358         # to avoid class name reuse. This kinda sucks. Need to find a way so
    359         # tests can use the same grammar name without messing up the namespace.
    360         # Well, first I should figure out what the exact problem is...
    361         id = hashlib.md5(self.baseDir).hexdigest()[-8:]
    362         grammar = grammar.replace('$TP', 'TP' + id)
    363         grammar = grammar.replace('$T', 'T' + id)
    364 
    365         # get type and name from first grammar line
    366         m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE)
    367         assert m is not None, grammar
    368         grammarType = m.group(2)
    369         if grammarType is None:
    370             grammarType = 'combined'
    371         grammarName = m.group(3)
    372 
    373         assert grammarType in ('lexer', 'parser', 'tree', 'combined'), grammarType
    374 
    375         grammarPath = os.path.join(self.baseDir, grammarName + '.g')
    376 
    377         # dump temp grammar file
    378         fp = open(grammarPath, 'w')
    379         fp.write(grammar)
    380         fp.close()
    381 
    382         return grammarName, grammarPath, grammarType
    383 
    384 
    385     def writeFile(self, name, contents):
    386         testDir = os.path.dirname(os.path.abspath(__file__))
    387         path = os.path.join(self.baseDir, name)
    388 
    389         fp = open(path, 'w')
    390         fp.write(contents)
    391         fp.close()
    392 
    393         return path
    394 
    395 
    396     def compileInlineGrammar(self, grammar, options='', javaOptions='',
    397                              returnModule=False):
    398         # write grammar file
    399         grammarName, grammarPath, grammarType = self.writeInlineGrammar(grammar)
    400 
    401         # compile it
    402         self._invokeantlr(
    403             os.path.dirname(grammarPath),
    404             os.path.basename(grammarPath),
    405             options,
    406             javaOptions
    407             )
    408 
    409         if grammarType == 'combined':
    410             lexerMod = self.__load_module(grammarName + 'Lexer')
    411             parserMod = self.__load_module(grammarName + 'Parser')
    412             if returnModule:
    413                 return lexerMod, parserMod
    414 
    415             lexerCls = getattr(lexerMod, grammarName + 'Lexer')
    416             lexerCls = self.lexerClass(lexerCls)
    417             parserCls = getattr(parserMod, grammarName + 'Parser')
    418             parserCls = self.parserClass(parserCls)
    419 
    420             return lexerCls, parserCls
    421 
    422         if grammarType == 'lexer':
    423             lexerMod = self.__load_module(grammarName)
    424             if returnModule:
    425                 return lexerMod
    426 
    427             lexerCls = getattr(lexerMod, grammarName)
    428             lexerCls = self.lexerClass(lexerCls)
    429 
    430             return lexerCls
    431 
    432         if grammarType == 'parser':
    433             parserMod = self.__load_module(grammarName)
    434             if returnModule:
    435                 return parserMod
    436 
    437             parserCls = getattr(parserMod, grammarName)
    438             parserCls = self.parserClass(parserCls)
    439 
    440             return parserCls
    441 
    442         if grammarType == 'tree':
    443             walkerMod = self.__load_module(grammarName)
    444             if returnModule:
    445                 return walkerMod
    446 
    447             walkerCls = getattr(walkerMod, grammarName)
    448             walkerCls = self.walkerClass(walkerCls)
    449 
    450             return walkerCls
    451