1 # Copyright 2012 the V8 project authors. All rights reserved. 2 # Redistribution and use in source and binary forms, with or without 3 # modification, are permitted provided that the following conditions are 4 # met: 5 # 6 # * Redistributions of source code must retain the above copyright 7 # notice, this list of conditions and the following disclaimer. 8 # * Redistributions in binary form must reproduce the above 9 # copyright notice, this list of conditions and the following 10 # disclaimer in the documentation and/or other materials provided 11 # with the distribution. 12 # * Neither the name of Google Inc. nor the names of its 13 # contributors may be used to endorse or promote products derived 14 # from this software without specific prior written permission. 15 # 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 import copy 29 import os 30 import re 31 import shlex 32 33 from ..outproc import base as outproc 34 from ..local import command 35 from ..local import statusfile 36 from ..local import utils 37 38 FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") 39 40 41 42 class TestCase(object): 43 def __init__(self, suite, path, name, test_config): 44 self.suite = suite # TestSuite object 45 46 self.path = path # string, e.g. 'div-mod', 'test-api/foo' 47 self.name = name # string that identifies test in the status file 48 49 self.variant = None # name of the used testing variant 50 self.variant_flags = [] # list of strings, flags specific to this test 51 52 # Fields used by the test processors. 53 self.origin = None # Test that this test is subtest of. 54 self.processor = None # Processor that created this subtest. 55 self.procid = '%s/%s' % (self.suite.name, self.name) # unique id 56 self.keep_output = False # Can output of this test be dropped 57 58 # Test config contains information needed to build the command. 59 self._test_config = test_config 60 self._random_seed = None # Overrides test config value if not None 61 62 # Outcomes 63 self._statusfile_outcomes = None 64 self.expected_outcomes = None 65 self._statusfile_flags = None 66 67 self._prepare_outcomes() 68 69 def create_subtest(self, processor, subtest_id, variant=None, flags=None, 70 keep_output=False, random_seed=None): 71 subtest = copy.copy(self) 72 subtest.origin = self 73 subtest.processor = processor 74 subtest.procid += '.%s' % subtest_id 75 subtest.keep_output |= keep_output 76 if random_seed: 77 subtest._random_seed = random_seed 78 if flags: 79 subtest.variant_flags = subtest.variant_flags + flags 80 if variant is not None: 81 assert self.variant is None 82 subtest.variant = variant 83 subtest._prepare_outcomes() 84 return subtest 85 86 def _prepare_outcomes(self, force_update=True): 87 if force_update or self._statusfile_outcomes is None: 88 def is_flag(outcome): 89 return outcome.startswith('--') 90 def not_flag(outcome): 91 return not is_flag(outcome) 92 93 outcomes = self.suite.statusfile.get_outcomes(self.name, self.variant) 94 self._statusfile_outcomes = filter(not_flag, outcomes) 95 self._statusfile_flags = filter(is_flag, outcomes) 96 self.expected_outcomes = ( 97 self._parse_status_file_outcomes(self._statusfile_outcomes)) 98 99 def _parse_status_file_outcomes(self, outcomes): 100 if (statusfile.FAIL_SLOPPY in outcomes and 101 '--use-strict' not in self.variant_flags): 102 return outproc.OUTCOMES_FAIL 103 104 expected_outcomes = [] 105 if (statusfile.FAIL in outcomes or 106 statusfile.FAIL_OK in outcomes): 107 expected_outcomes.append(statusfile.FAIL) 108 if statusfile.CRASH in outcomes: 109 expected_outcomes.append(statusfile.CRASH) 110 111 # Do not add PASS if there is nothing else. Empty outcomes are converted to 112 # the global [PASS]. 113 if expected_outcomes and statusfile.PASS in outcomes: 114 expected_outcomes.append(statusfile.PASS) 115 116 # Avoid creating multiple instances of a list with a single FAIL. 117 if expected_outcomes == outproc.OUTCOMES_FAIL: 118 return outproc.OUTCOMES_FAIL 119 return expected_outcomes or outproc.OUTCOMES_PASS 120 121 @property 122 def do_skip(self): 123 return statusfile.SKIP in self._statusfile_outcomes 124 125 @property 126 def is_slow(self): 127 return statusfile.SLOW in self._statusfile_outcomes 128 129 @property 130 def is_fail_ok(self): 131 return statusfile.FAIL_OK in self._statusfile_outcomes 132 133 @property 134 def is_pass_or_fail(self): 135 return (statusfile.PASS in self._statusfile_outcomes and 136 statusfile.FAIL in self._statusfile_outcomes and 137 statusfile.CRASH not in self._statusfile_outcomes) 138 139 @property 140 def only_standard_variant(self): 141 return statusfile.NO_VARIANTS in self._statusfile_outcomes 142 143 def get_command(self): 144 params = self._get_cmd_params() 145 env = self._get_cmd_env() 146 shell, shell_flags = self._get_shell_with_flags() 147 timeout = self._get_timeout(params) 148 return self._create_cmd(shell, shell_flags + params, env, timeout) 149 150 def _get_cmd_params(self): 151 """Gets command parameters and combines them in the following order: 152 - files [empty by default] 153 - random seed 154 - extra flags (from command line) 155 - user flags (variant/fuzzer flags) 156 - mode flags (based on chosen mode) 157 - source flags (from source code) [empty by default] 158 - test-suite flags 159 - statusfile flags 160 161 The best way to modify how parameters are created is to only override 162 methods for getting partial parameters. 163 """ 164 return ( 165 self._get_files_params() + 166 self._get_random_seed_flags() + 167 self._get_extra_flags() + 168 self._get_variant_flags() + 169 self._get_mode_flags() + 170 self._get_source_flags() + 171 self._get_suite_flags() + 172 self._get_statusfile_flags() 173 ) 174 175 def _get_cmd_env(self): 176 return {} 177 178 def _get_files_params(self): 179 return [] 180 181 def _get_random_seed_flags(self): 182 return ['--random-seed=%d' % self.random_seed] 183 184 @property 185 def random_seed(self): 186 return self._random_seed or self._test_config.random_seed 187 188 def _get_extra_flags(self): 189 return self._test_config.extra_flags 190 191 def _get_variant_flags(self): 192 return self.variant_flags 193 194 def _get_statusfile_flags(self): 195 """Gets runtime flags from a status file. 196 197 Every outcome that starts with "--" is a flag. 198 """ 199 return self._statusfile_flags 200 201 def _get_mode_flags(self): 202 return self._test_config.mode_flags 203 204 def _get_source_flags(self): 205 return [] 206 207 def _get_suite_flags(self): 208 return [] 209 210 def _get_shell_with_flags(self): 211 shell = self.get_shell() 212 shell_flags = [] 213 if shell == 'd8': 214 shell_flags.append('--test') 215 if utils.IsWindows(): 216 shell += '.exe' 217 return shell, shell_flags 218 219 def _get_timeout(self, params): 220 timeout = self._test_config.timeout 221 if "--stress-opt" in params: 222 timeout *= 4 223 if "--noenable-vfp3" in params: 224 timeout *= 2 225 226 # TODO(majeski): make it slow outcome dependent. 227 timeout *= 2 228 return timeout 229 230 def get_shell(self): 231 return 'd8' 232 233 def _get_suffix(self): 234 return '.js' 235 236 def _create_cmd(self, shell, params, env, timeout): 237 return command.Command( 238 cmd_prefix=self._test_config.command_prefix, 239 shell=os.path.abspath(os.path.join(self._test_config.shell_dir, shell)), 240 args=params, 241 env=env, 242 timeout=timeout, 243 verbose=self._test_config.verbose, 244 resources_func=self._get_resources, 245 ) 246 247 def _parse_source_flags(self, source=None): 248 source = source or self.get_source() 249 flags = [] 250 for match in re.findall(FLAGS_PATTERN, source): 251 flags += shlex.split(match.strip()) 252 return flags 253 254 def is_source_available(self): 255 return self._get_source_path() is not None 256 257 def get_source(self): 258 with open(self._get_source_path()) as f: 259 return f.read() 260 261 def _get_source_path(self): 262 return None 263 264 def _get_resources(self): 265 """Returns a list of absolute paths with additional files needed by the 266 test case. 267 268 Used to push additional files to Android devices. 269 """ 270 return [] 271 272 @property 273 def output_proc(self): 274 if self.expected_outcomes is outproc.OUTCOMES_PASS: 275 return outproc.DEFAULT 276 return outproc.OutProc(self.expected_outcomes) 277 278 def __cmp__(self, other): 279 # Make sure that test cases are sorted correctly if sorted without 280 # key function. But using a key function is preferred for speed. 281 return cmp( 282 (self.suite.name, self.name, self.variant), 283 (other.suite.name, other.name, other.variant) 284 ) 285 286 def __str__(self): 287 return self.suite.name + '/' + self.name 288
