Home | History | Annotate | Download | only in test 
      1 # -*- Python -*-
      2 
      3 import os
      4 import platform
      5 import re
      6 import subprocess
      7 import tempfile
      8 
      9 import lit.formats
     10 import lit.util
     11 
     12 # Configuration file for the 'lit' test runner.
     13 
     14 # name: The name of this test suite.
     15 config.name = 'Clang'
     16 
     17 # Tweak PATH for Win32
     18 if platform.system() == 'Windows':
     19     # Seek sane tools in directories and set to $PATH.
     20     path = getattr(config, 'lit_tools_dir', None)
     21     path = lit_config.getToolsPath(path,
     22                                    config.environment['PATH'],
     23                                    ['cmp.exe', 'grep.exe', 'sed.exe'])
     24     if path is not None:
     25         path = os.path.pathsep.join((path,
     26                                      config.environment['PATH']))
     27         config.environment['PATH'] = path
     28 
     29 # Choose between lit's internal shell pipeline runner and a real shell.  If
     30 # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
     31 use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
     32 if use_lit_shell:
     33     # 0 is external, "" is default, and everything else is internal.
     34     execute_external = (use_lit_shell == "0")
     35 else:
     36     # Otherwise we default to internal on Windows and external elsewhere, as
     37     # bash on Windows is usually very slow.
     38     execute_external = (not sys.platform in ['win32'])
     39 
     40 # testFormat: The test format to use to interpret tests.
     41 #
     42 # For now we require '&&' between commands, until they get globally killed and
     43 # the test runner updated.
     44 config.test_format = lit.formats.ShTest(execute_external)
     45 
     46 # suffixes: A list of file extensions to treat as test files.
     47 config.suffixes = ['.c', '.cpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s', '.S', '.modulemap']
     48 
     49 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
     50 # subdirectories contain auxiliary inputs for various tests in their parent
     51 # directories.
     52 config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
     53 
     54 # test_source_root: The root path where tests are located.
     55 config.test_source_root = os.path.dirname(__file__)
     56 
     57 # test_exec_root: The root path where tests should be run.
     58 clang_obj_root = getattr(config, 'clang_obj_root', None)
     59 if clang_obj_root is not None:
     60     config.test_exec_root = os.path.join(clang_obj_root, 'test')
     61 
     62 # Set llvm_{src,obj}_root for use by others.
     63 config.llvm_src_root = getattr(config, 'llvm_src_root', None)
     64 config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)
     65 
     66 # Clear some environment variables that might affect Clang.
     67 #
     68 # This first set of vars are read by Clang, but shouldn't affect tests
     69 # that aren't specifically looking for these features, or are required
     70 # simply to run the tests at all.
     71 #
     72 # FIXME: Should we have a tool that enforces this?
     73 
     74 # safe_env_vars = ('TMPDIR', 'TEMP', 'TMP', 'USERPROFILE', 'PWD',
     75 #                  'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET',
     76 #                  'VCINSTALLDIR', 'VC100COMNTOOLS', 'VC90COMNTOOLS',
     77 #                  'VC80COMNTOOLS')
     78 possibly_dangerous_env_vars = ['COMPILER_PATH', 'RC_DEBUG_OPTIONS',
     79                                'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH',
     80                                'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH',
     81                                'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH',
     82                                'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING',
     83                                'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX',
     84                                'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS',
     85                                'LIBCLANG_RESOURCE_USAGE',
     86                                'LIBCLANG_CODE_COMPLETION_LOGGING']
     87 # Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it.
     88 if platform.system() != 'Windows':
     89     possibly_dangerous_env_vars.append('INCLUDE')
     90 for name in possibly_dangerous_env_vars:
     91   if name in config.environment:
     92     del config.environment[name]
     93 
     94 # Tweak the PATH to include the tools dir and the scripts dir.
     95 if clang_obj_root is not None:
     96     clang_tools_dir = getattr(config, 'clang_tools_dir', None)
     97     if not clang_tools_dir:
     98         lit_config.fatal('No Clang tools dir set!')
     99     llvm_tools_dir = getattr(config, 'llvm_tools_dir', None)
    100     if not llvm_tools_dir:
    101         lit_config.fatal('No LLVM tools dir set!')
    102     path = os.path.pathsep.join((
    103             clang_tools_dir, llvm_tools_dir, config.environment['PATH']))
    104     config.environment['PATH'] = path
    105     llvm_libs_dir = getattr(config, 'llvm_libs_dir', None)
    106     if not llvm_libs_dir:
    107         lit_config.fatal('No LLVM libs dir set!')
    108     path = os.path.pathsep.join((llvm_libs_dir,
    109                                  config.environment.get('LD_LIBRARY_PATH','')))
    110     config.environment['LD_LIBRARY_PATH'] = path
    111 
    112 # Propagate path to symbolizer for ASan/MSan.
    113 for symbolizer in ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']:
    114     if symbolizer in os.environ:
    115         config.environment[symbolizer] = os.environ[symbolizer]
    116 
    117 ###
    118 
    119 # Check that the object root is known.
    120 if config.test_exec_root is None:
    121     # Otherwise, we haven't loaded the site specific configuration (the user is
    122     # probably trying to run on a test file directly, and either the site
    123     # configuration hasn't been created by the build system, or we are in an
    124     # out-of-tree build situation).
    125 
    126     # Check for 'clang_site_config' user parameter, and use that if available.
    127     site_cfg = lit_config.params.get('clang_site_config', None)
    128     if site_cfg and os.path.exists(site_cfg):
    129         lit_config.load_config(config, site_cfg)
    130         raise SystemExit
    131 
    132     # Try to detect the situation where we are using an out-of-tree build by
    133     # looking for 'llvm-config'.
    134     #
    135     # FIXME: I debated (i.e., wrote and threw away) adding logic to
    136     # automagically generate the lit.site.cfg if we are in some kind of fresh
    137     # build situation. This means knowing how to invoke the build system though,
    138     # and I decided it was too much magic. We should solve this by just having
    139     # the .cfg files generated during the configuration step.
    140 
    141     llvm_config = lit.util.which('llvm-config', config.environment['PATH'])
    142     if not llvm_config:
    143         lit_config.fatal('No site specific configuration available!')
    144 
    145     # Get the source and object roots.
    146     llvm_src_root = lit.util.capture(['llvm-config', '--src-root']).strip()
    147     llvm_obj_root = lit.util.capture(['llvm-config', '--obj-root']).strip()
    148     clang_src_root = os.path.join(llvm_src_root, "tools", "clang")
    149     clang_obj_root = os.path.join(llvm_obj_root, "tools", "clang")
    150 
    151     # Validate that we got a tree which points to here, using the standard
    152     # tools/clang layout.
    153     this_src_root = os.path.dirname(config.test_source_root)
    154     if os.path.realpath(clang_src_root) != os.path.realpath(this_src_root):
    155         lit_config.fatal('No site specific configuration available!')
    156 
    157     # Check that the site specific configuration exists.
    158     site_cfg = os.path.join(clang_obj_root, 'test', 'lit.site.cfg')
    159     if not os.path.exists(site_cfg):
    160         lit_config.fatal(
    161             'No site specific configuration available! You may need to '
    162             'run "make test" in your Clang build directory.')
    163 
    164     # Okay, that worked. Notify the user of the automagic, and reconfigure.
    165     lit_config.note('using out-of-tree build at %r' % clang_obj_root)
    166     lit_config.load_config(config, site_cfg)
    167     raise SystemExit
    168 
    169 ###
    170 
    171 # Discover the 'clang' and 'clangcc' to use.
    172 
    173 import os
    174 
    175 def inferClang(PATH):
    176     # Determine which clang to use.
    177     clang = os.getenv('CLANG')
    178 
    179     # If the user set clang in the environment, definitely use that and don't
    180     # try to validate.
    181     if clang:
    182         return clang
    183 
    184     # Otherwise look in the path.
    185     clang = lit.util.which('clang', PATH)
    186 
    187     if not clang:
    188         lit_config.fatal("couldn't find 'clang' program, try setting "
    189                          "CLANG in your environment")
    190 
    191     return clang
    192 
    193 config.clang = inferClang(config.environment['PATH']).replace('\\', '/')
    194 if not lit_config.quiet:
    195     lit_config.note('using clang: %r' % config.clang)
    196 
    197 # Plugins (loadable modules)
    198 # TODO: This should be supplied by Makefile or autoconf.
    199 if sys.platform in ['win32', 'cygwin']:
    200     has_plugins = (config.enable_shared == 1)
    201 else:
    202     has_plugins = True
    203 
    204 if has_plugins and config.llvm_plugin_ext:
    205     config.available_features.add('plugins')
    206 
    207 config.substitutions.append( ('%llvmshlibdir', config.llvm_shlib_dir) )
    208 config.substitutions.append( ('%pluginext', config.llvm_plugin_ext) )
    209 config.substitutions.append( ('%PATH%', config.environment['PATH']) )
    210 
    211 if config.clang_examples:
    212     config.available_features.add('examples')
    213 
    214 # Note that when substituting %clang_cc1 also fill in the include directory of
    215 # the builtin headers. Those are part of even a freestanding environment, but
    216 # Clang relies on the driver to locate them.
    217 def getClangBuiltinIncludeDir(clang):
    218     # FIXME: Rather than just getting the version, we should have clang print
    219     # out its resource dir here in an easy to scrape form.
    220     cmd = subprocess.Popen([clang, '-print-file-name=include'],
    221                            stdout=subprocess.PIPE,
    222                            env=config.environment)
    223     if not cmd.stdout:
    224       lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
    225     dir = cmd.stdout.read().strip()
    226     if sys.platform in ['win32'] and execute_external:
    227         # Don't pass dosish path separator to msys bash.exe.
    228         dir = dir.replace('\\', '/')
    229     # Ensure the result is an ascii string, across Python2.5+ - Python3.
    230     return str(dir.decode('ascii'))
    231 
    232 def makeItaniumABITriple(triple):
    233     m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
    234     if not m:
    235       lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
    236     if m.group(3).lower() != 'win32':
    237       # All non-win32 triples use the Itanium ABI.
    238       return triple
    239     return m.group(1) + '-' + m.group(2) + '-mingw32'
    240 
    241 def makeMSABITriple(triple):
    242     m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
    243     if not m:
    244       lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
    245     isa = m.group(1).lower()
    246     vendor = m.group(2).lower()
    247     os = m.group(3).lower()
    248     if os == 'win32':
    249       # If the OS is win32, we're done.
    250       return triple
    251     if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa): 
    252       # For x86 ISAs, adjust the OS.
    253       return isa + '-' + vendor + '-win32'
    254     # -win32 is not supported for non-x86 targets; use a default.
    255     return 'i686-pc-win32'
    256 
    257 config.substitutions.append( ('%clang_cc1',
    258                               '%s -cc1 -internal-isystem %s -nostdsysteminc'
    259                               % (config.clang,
    260                                  getClangBuiltinIncludeDir(config.clang))) )
    261 config.substitutions.append( ('%clang_cpp', ' ' + config.clang +
    262                               ' --driver-mode=cpp '))
    263 config.substitutions.append( ('%clang_cl', ' ' + config.clang +
    264                               ' --driver-mode=cl '))
    265 config.substitutions.append( ('%clangxx', ' ' + config.clang +
    266                               ' --driver-mode=g++ '))
    267 config.substitutions.append( ('%clang', ' ' + config.clang + ' ') )
    268 config.substitutions.append( ('%test_debuginfo', ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') )
    269 config.substitutions.append( ('%itanium_abi_triple', makeItaniumABITriple(config.target_triple)) )
    270 config.substitutions.append( ('%ms_abi_triple', makeMSABITriple(config.target_triple)) )
    271 
    272 # The host triple might not be set, at least if we're compiling clang from
    273 # an already installed llvm.
    274 if config.host_triple and config.host_triple != '@LLVM_HOST_TRIPLE@':
    275     config.substitutions.append( ('%target_itanium_abi_host_triple', '--target=%s' % makeItaniumABITriple(config.host_triple)) )
    276 else:
    277     config.substitutions.append( ('%target_itanium_abi_host_triple', '') )
    278 
    279 # FIXME: Find nicer way to prohibit this.
    280 config.substitutions.append(
    281     (' clang ', """*** Do not use 'clang' in tests, use '%clang'. ***""") )
    282 config.substitutions.append(
    283     (' clang\+\+ ', """*** Do not use 'clang++' in tests, use '%clangxx'. ***"""))
    284 config.substitutions.append(
    285     (' clang-cc ',
    286      """*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***""") )
    287 config.substitutions.append(
    288     (' clang -cc1 ',
    289      """*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***""") )
    290 config.substitutions.append(
    291     (' %clang-cc1 ',
    292      """*** invalid substitution, use '%clang_cc1'. ***""") )
    293 config.substitutions.append(
    294     (' %clang-cpp ',
    295      """*** invalid substitution, use '%clang_cpp'. ***""") )
    296 config.substitutions.append(
    297     (' %clang-cl ',
    298      """*** invalid substitution, use '%clang_cl'. ***""") )
    299 
    300 # For each occurrence of a clang tool name as its own word, replace it
    301 # with the full path to the build directory holding that tool.  This
    302 # ensures that we are testing the tools just built and not some random
    303 # tools that might happen to be in the user's PATH.
    304 tool_dirs = os.path.pathsep.join((clang_tools_dir, llvm_tools_dir))
    305 
    306 # Regex assertions to reject neighbor hyphens/dots (seen in some tests).
    307 # For example, don't match 'clang-check-' or '.clang-format'.
    308 NoPreHyphenDot = r"(?<!(-|\.))"
    309 NoPostHyphenDot = r"(?!(-|\.))"
    310 NoPostBar = r"(?!(/|\\))"
    311 
    312 tool_patterns = [r"\bFileCheck\b",
    313                  r"\bc-index-test\b",
    314                  NoPreHyphenDot + r"\bclang-check\b" + NoPostHyphenDot,
    315                  NoPreHyphenDot + r"\bclang-format\b" + NoPostHyphenDot,
    316                  # FIXME: Some clang test uses opt?
    317                  NoPreHyphenDot + r"\bopt\b" + NoPostBar + NoPostHyphenDot,
    318                  # Handle these specially as they are strings searched
    319                  # for during testing.
    320                  r"\| \bcount\b",
    321                  r"\| \bnot\b"]
    322 
    323 if config.clang_examples:
    324     tool_patterns.append(NoPreHyphenDot + r"\bclang-interpreter\b" + NoPostHyphenDot)
    325 
    326 for pattern in tool_patterns:
    327     # Extract the tool name from the pattern.  This relies on the tool
    328     # name being surrounded by \b word match operators.  If the
    329     # pattern starts with "| ", include it in the string to be
    330     # substituted.
    331     tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
    332                           pattern)
    333     tool_pipe = tool_match.group(2)
    334     tool_name = tool_match.group(4)
    335     tool_path = lit.util.which(tool_name, tool_dirs)
    336     if not tool_path:
    337         # Warn, but still provide a substitution.
    338         lit_config.note('Did not find ' + tool_name + ' in ' + tool_dirs)
    339         tool_path = clang_tools_dir + '/' + tool_name
    340     config.substitutions.append((pattern, tool_pipe + tool_path))
    341 
    342 ###
    343 
    344 # Set available features we allow tests to conditionalize on.
    345 #
    346 # Enabled/disabled features
    347 if config.clang_staticanalyzer != 0:
    348     config.available_features.add("staticanalyzer")
    349 
    350 # As of 2011.08, crash-recovery tests still do not pass on FreeBSD.
    351 if platform.system() not in ['FreeBSD']:
    352     config.available_features.add('crash-recovery')
    353 
    354 # Shell execution
    355 if execute_external:
    356     config.available_features.add('shell')
    357 
    358 # For tests that require Darwin to run.
    359 # This is used by debuginfo-tests/*block*.m and debuginfo-tests/foreach.m.
    360 if platform.system() in ['Darwin']:
    361     config.available_features.add('system-darwin')
    362 elif platform.system() in ['Windows']:
    363     # For tests that require Windows to run.
    364     config.available_features.add('system-windows')
    365 
    366 # ANSI escape sequences in non-dumb terminal
    367 if platform.system() not in ['Windows']:
    368     config.available_features.add('ansi-escape-sequences')
    369 
    370 # Capability to print utf8 to the terminal.
    371 # Windows expects codepage, unless Wide API.
    372 if platform.system() not in ['Windows']:
    373     config.available_features.add('utf8-capable-terminal')
    374 
    375 # Native compilation: Check if triples match.
    376 # FIXME: Consider cases that target can be executed
    377 # even if host_triple were different from target_triple.
    378 if config.host_triple == config.target_triple:
    379     config.available_features.add("native")
    380 
    381 # Case-insensitive file system
    382 def is_filesystem_case_insensitive():
    383     handle, path = tempfile.mkstemp(prefix='case-test', dir=config.test_exec_root)
    384     isInsensitive = os.path.exists(
    385         os.path.join(
    386             os.path.dirname(path),
    387             os.path.basename(path).upper()
    388             ))
    389     os.close(handle)
    390     os.remove(path)
    391     return isInsensitive
    392 
    393 if is_filesystem_case_insensitive():
    394     config.available_features.add('case-insensitive-filesystem')
    395 
    396 # Tests that require the /dev/fd filesystem.
    397 if os.path.exists("/dev/fd/0") and sys.platform not in ['cygwin']:
    398     config.available_features.add('dev-fd-fs')
    399 
    400 # DW2 Target
    401 if not re.match(r'.*-win32$', config.target_triple):
    402     config.available_features.add('dw2')
    403 
    404 # Not set on native MS environment.
    405 if not re.match(r'.*-win32$', config.target_triple):
    406     config.available_features.add('non-ms-sdk')
    407 
    408 # Not set on native PS4 environment.
    409 if not re.match(r'.*-scei-ps4', config.target_triple):
    410     config.available_features.add('non-ps4-sdk')
    411 
    412 # [PR8833] LLP64-incompatible tests
    413 if not re.match(r'^x86_64.*-(win32|mingw32|windows-gnu)$', config.target_triple):
    414     config.available_features.add('LP64')
    415 
    416 # [PR12920] "clang-driver" -- set if gcc driver is not used.
    417 if not re.match(r'.*-(cygwin)$', config.target_triple):
    418     config.available_features.add('clang-driver')
    419 
    420 # [PR18856] Depends to remove opened file. On win32, a file could be removed
    421 # only if all handles were closed.
    422 if platform.system() not in ['Windows']:
    423     config.available_features.add('can-remove-opened-file')
    424 
    425 # Not set for targeting tls-incapable targets.
    426 if not re.match(r'.*-cygwin$', config.target_triple):
    427     config.available_features.add('tls')
    428 
    429 # Returns set of available features, registered-target(s) and asserts.
    430 def get_llvm_config_props():
    431     set_of_features = set()
    432 
    433     cmd = subprocess.Popen(
    434         [
    435             os.path.join(llvm_tools_dir, 'llvm-config'),
    436             '--assertion-mode',
    437             '--targets-built',
    438             ],
    439         stdout=subprocess.PIPE,
    440         env=config.environment
    441         )
    442     # 1st line corresponds to --assertion-mode, "ON" or "OFF".
    443     line = cmd.stdout.readline().strip().decode('ascii')
    444     if line == "ON":
    445         set_of_features.add('asserts')
    446 
    447     # 2nd line corresponds to --targets-built, like;
    448     # AArch64 ARM CppBackend X86
    449     for arch in cmd.stdout.readline().decode('ascii').split():
    450         set_of_features.add(arch.lower() + '-registered-target')
    451 
    452     return set_of_features
    453 
    454 config.available_features.update(get_llvm_config_props())
    455 
    456 if lit.util.which('xmllint'):
    457     config.available_features.add('xmllint')
    458 
    459 # Sanitizers.
    460 if 'Address' in config.llvm_use_sanitizer:
    461     config.available_features.add("asan")
    462 else:
    463     config.available_features.add("not_asan")
    464 if 'Memory' in config.llvm_use_sanitizer:
    465     config.available_features.add("msan")
    466 if 'Undefined' in config.llvm_use_sanitizer:
    467     config.available_features.add("ubsan")
    468 else:
    469     config.available_features.add("not_ubsan")
    470 
    471 if config.enable_backtrace == "1":
    472     config.available_features.add("backtrace")
    473 
    474 # Check if we should run long running tests.
    475 if lit_config.params.get("run_long_tests", None) == "true":
    476     config.available_features.add("long_tests")
    477 
    478 # Check if we should use gmalloc.
    479 use_gmalloc_str = lit_config.params.get('use_gmalloc', None)
    480 if use_gmalloc_str is not None:
    481     if use_gmalloc_str.lower() in ('1', 'true'):
    482         use_gmalloc = True
    483     elif use_gmalloc_str.lower() in ('', '0', 'false'):
    484         use_gmalloc = False
    485     else:
    486         lit_config.fatal('user parameter use_gmalloc should be 0 or 1')
    487 else:
    488     # Default to not using gmalloc
    489     use_gmalloc = False
    490 
    491 # Allow use of an explicit path for gmalloc library.
    492 # Will default to '/usr/lib/libgmalloc.dylib' if not set.
    493 gmalloc_path_str = lit_config.params.get('gmalloc_path',
    494                                          '/usr/lib/libgmalloc.dylib')
    495 if use_gmalloc:
    496      config.environment.update({'DYLD_INSERT_LIBRARIES' : gmalloc_path_str})
    497 
    498 lit.util.usePlatformSdkOnDarwin(config, lit_config)
    499