1 #===----------------------------------------------------------------------===## 2 # 3 # The LLVM Compiler Infrastructure 4 # 5 # This file is dual licensed under the MIT and the University of Illinois Open 6 # Source Licenses. See LICENSE.TXT for details. 7 # 8 #===----------------------------------------------------------------------===## 9 10 import copy 11 import errno 12 import os 13 import time 14 import random 15 16 import lit.Test # pylint: disable=import-error 17 import lit.TestRunner # pylint: disable=import-error 18 from lit.TestRunner import ParserKind, IntegratedTestKeywordParser \ 19 # pylint: disable=import-error 20 21 from libcxx.test.executor import LocalExecutor as LocalExecutor 22 import libcxx.util 23 24 25 class LibcxxTestFormat(object): 26 """ 27 Custom test format handler for use with the test format use by libc++. 28 29 Tests fall into two categories: 30 FOO.pass.cpp - Executable test which should compile, run, and exit with 31 code 0. 32 FOO.fail.cpp - Negative test case which is expected to fail compilation. 33 FOO.sh.cpp - A test that uses LIT's ShTest format. 34 """ 35 36 def __init__(self, cxx, use_verify_for_fail, execute_external, 37 executor, exec_env): 38 self.cxx = copy.deepcopy(cxx) 39 self.use_verify_for_fail = use_verify_for_fail 40 self.execute_external = execute_external 41 self.executor = executor 42 self.exec_env = dict(exec_env) 43 44 @staticmethod 45 def _make_custom_parsers(): 46 return [ 47 IntegratedTestKeywordParser('FLAKY_TEST.', ParserKind.TAG, 48 initial_value=False), 49 IntegratedTestKeywordParser('MODULES_DEFINES:', ParserKind.LIST, 50 initial_value=[]) 51 ] 52 53 @staticmethod 54 def _get_parser(key, parsers): 55 for p in parsers: 56 if p.keyword == key: 57 return p 58 assert False and "parser not found" 59 60 # TODO: Move this into lit's FileBasedTest 61 def getTestsInDirectory(self, testSuite, path_in_suite, 62 litConfig, localConfig): 63 source_path = testSuite.getSourcePath(path_in_suite) 64 for filename in os.listdir(source_path): 65 # Ignore dot files and excluded tests. 66 if filename.startswith('.') or filename in localConfig.excludes: 67 continue 68 69 filepath = os.path.join(source_path, filename) 70 if not os.path.isdir(filepath): 71 if any([filename.endswith(ext) 72 for ext in localConfig.suffixes]): 73 yield lit.Test.Test(testSuite, path_in_suite + (filename,), 74 localConfig) 75 76 def execute(self, test, lit_config): 77 while True: 78 try: 79 return self._execute(test, lit_config) 80 except OSError as oe: 81 if oe.errno != errno.ETXTBSY: 82 raise 83 time.sleep(0.1) 84 85 def _execute(self, test, lit_config): 86 name = test.path_in_suite[-1] 87 name_root, name_ext = os.path.splitext(name) 88 is_libcxx_test = test.path_in_suite[0] == 'libcxx' 89 is_sh_test = name_root.endswith('.sh') 90 is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm') 91 is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm') 92 is_objcxx_test = name.endswith('.mm') 93 is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \ 94 name.endswith('.arc.fail.mm') 95 assert is_sh_test or name_ext == '.cpp' or name_ext == '.mm', \ 96 'non-cpp file must be sh test' 97 98 if test.config.unsupported: 99 return (lit.Test.UNSUPPORTED, 100 "A lit.local.cfg marked this unsupported") 101 102 if is_objcxx_test and not \ 103 'objective-c++' in test.config.available_features: 104 return (lit.Test.UNSUPPORTED, "Objective-C++ is not supported") 105 106 parsers = self._make_custom_parsers() 107 script = lit.TestRunner.parseIntegratedTestScript( 108 test, additional_parsers=parsers, require_script=is_sh_test) 109 # Check if a result for the test was returned. If so return that 110 # result. 111 if isinstance(script, lit.Test.Result): 112 return script 113 if lit_config.noExecute: 114 return lit.Test.Result(lit.Test.PASS) 115 116 # Check that we don't have run lines on tests that don't support them. 117 if not is_sh_test and len(script) != 0: 118 lit_config.fatal('Unsupported RUN line found in test %s' % name) 119 120 tmpDir, tmpBase = lit.TestRunner.getTempPaths(test) 121 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, 122 tmpBase) 123 script = lit.TestRunner.applySubstitutions(script, substitutions) 124 125 test_cxx = copy.deepcopy(self.cxx) 126 if is_fail_test: 127 test_cxx.useCCache(False) 128 test_cxx.useWarnings(False) 129 extra_modules_defines = self._get_parser('MODULES_DEFINES:', 130 parsers).getValue() 131 if '-fmodules' in test.config.available_features: 132 test_cxx.compile_flags += [('-D%s' % mdef.strip()) for 133 mdef in extra_modules_defines] 134 test_cxx.addWarningFlagIfSupported('-Wno-macro-redefined') 135 # FIXME: libc++ debug tests #define _LIBCPP_ASSERT to override it 136 # If we see this we need to build the test against uniquely built 137 # modules. 138 if is_libcxx_test: 139 with open(test.getSourcePath(), 'r') as f: 140 contents = f.read() 141 if '#define _LIBCPP_ASSERT' in contents: 142 test_cxx.useModules(False) 143 144 if is_objcxx_test: 145 test_cxx.source_lang = 'objective-c++' 146 if is_objcxx_arc_test: 147 test_cxx.compile_flags += ['-fobjc-arc'] 148 else: 149 test_cxx.compile_flags += ['-fno-objc-arc'] 150 test_cxx.link_flags += ['-framework', 'Foundation'] 151 152 # Dispatch the test based on its suffix. 153 if is_sh_test: 154 if not isinstance(self.executor, LocalExecutor): 155 # We can't run ShTest tests with a executor yet. 156 # For now, bail on trying to run them 157 return lit.Test.UNSUPPORTED, 'ShTest format not yet supported' 158 test.config.environment = dict(self.exec_env) 159 return lit.TestRunner._runShTest(test, lit_config, 160 self.execute_external, script, 161 tmpBase) 162 elif is_fail_test: 163 return self._evaluate_fail_test(test, test_cxx, parsers) 164 elif is_pass_test: 165 return self._evaluate_pass_test(test, tmpBase, lit_config, 166 test_cxx, parsers) 167 else: 168 # No other test type is supported 169 assert False 170 171 def _clean(self, exec_path): # pylint: disable=no-self-use 172 libcxx.util.cleanFile(exec_path) 173 174 def _evaluate_pass_test(self, test, tmpBase, lit_config, 175 test_cxx, parsers): 176 execDir = os.path.dirname(test.getExecPath()) 177 source_path = test.getSourcePath() 178 exec_path = tmpBase + '.exe' 179 object_path = tmpBase + '.o' 180 # Create the output directory if it does not already exist. 181 libcxx.util.mkdir_p(os.path.dirname(tmpBase)) 182 try: 183 # Compile the test 184 cmd, out, err, rc = test_cxx.compileLinkTwoSteps( 185 source_path, out=exec_path, object_file=object_path, 186 cwd=execDir) 187 compile_cmd = cmd 188 if rc != 0: 189 report = libcxx.util.makeReport(cmd, out, err, rc) 190 report += "Compilation failed unexpectedly!" 191 return lit.Test.FAIL, report 192 # Run the test 193 local_cwd = os.path.dirname(source_path) 194 env = None 195 if self.exec_env: 196 env = self.exec_env 197 # TODO: Only list actually needed files in file_deps. 198 # Right now we just mark all of the .dat files in the same 199 # directory as dependencies, but it's likely less than that. We 200 # should add a `// FILE-DEP: foo.dat` to each test to track this. 201 data_files = [os.path.join(local_cwd, f) 202 for f in os.listdir(local_cwd) if f.endswith('.dat')] 203 is_flaky = self._get_parser('FLAKY_TEST.', parsers).getValue() 204 max_retry = 3 if is_flaky else 1 205 for retry_count in range(max_retry): 206 cmd, out, err, rc = self.executor.run(exec_path, [exec_path], 207 local_cwd, data_files, 208 env) 209 if rc == 0: 210 res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS 211 return res, '' 212 elif rc != 0 and retry_count + 1 == max_retry: 213 report = libcxx.util.makeReport(cmd, out, err, rc) 214 report = "Compiled With: %s\n%s" % (compile_cmd, report) 215 report += "Compiled test failed unexpectedly!" 216 return lit.Test.FAIL, report 217 218 assert False # Unreachable 219 finally: 220 # Note that cleanup of exec_file happens in `_clean()`. If you 221 # override this, cleanup is your reponsibility. 222 libcxx.util.cleanFile(object_path) 223 self._clean(exec_path) 224 225 def _evaluate_fail_test(self, test, test_cxx, parsers): 226 source_path = test.getSourcePath() 227 # FIXME: lift this detection into LLVM/LIT. 228 with open(source_path, 'r') as f: 229 contents = f.read() 230 verify_tags = ['expected-note', 'expected-remark', 'expected-warning', 231 'expected-error', 'expected-no-diagnostics'] 232 use_verify = self.use_verify_for_fail and \ 233 any([tag in contents for tag in verify_tags]) 234 # FIXME(EricWF): GCC 5 does not evaluate static assertions that 235 # are dependant on a template parameter when '-fsyntax-only' is passed. 236 # This is fixed in GCC 6. However for now we only pass "-fsyntax-only" 237 # when using Clang. 238 if test_cxx.type != 'gcc': 239 test_cxx.flags += ['-fsyntax-only'] 240 if use_verify: 241 test_cxx.useVerify() 242 test_cxx.useWarnings() 243 if '-Wuser-defined-warnings' in test_cxx.warning_flags: 244 test_cxx.warning_flags += ['-Wno-error=user-defined-warnings'] 245 else: 246 # We still need to enable certain warnings on .fail.cpp test when 247 # -verify isn't enabled. Such as -Werror=unused-result. However, 248 # we don't want it enabled too liberally, which might incorrectly 249 # allow unrelated failure tests to 'pass'. 250 # 251 # Therefore, we check if the test was expected to fail because of 252 # nodiscard before enabling it 253 test_str = "ignoring return value of function declared with " \ 254 + "'nodiscard' attribute" 255 if test_str in contents: 256 test_cxx.flags += ['-Werror=unused-result'] 257 cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull) 258 expected_rc = 0 if use_verify else 1 259 if rc == expected_rc: 260 return lit.Test.PASS, '' 261 else: 262 report = libcxx.util.makeReport(cmd, out, err, rc) 263 report_msg = ('Expected compilation to fail!' if not use_verify else 264 'Expected compilation using verify to pass!') 265 return lit.Test.FAIL, report + report_msg + '\n' 266
