1 # Copyright 2012 the V8 project authors. All rights reserved. 2 # Redistribution and use in source and binary forms, with or without 3 # modification, are permitted provided that the following conditions are 4 # met: 5 # 6 # * Redistributions of source code must retain the above copyright 7 # notice, this list of conditions and the following disclaimer. 8 # * Redistributions in binary form must reproduce the above 9 # copyright notice, this list of conditions and the following 10 # disclaimer in the documentation and/or other materials provided 11 # with the distribution. 12 # * Neither the name of Google Inc. nor the names of its 13 # contributors may be used to endorse or promote products derived 14 # from this software without specific prior written permission. 15 # 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 29 from functools import wraps 30 import json 31 import os 32 import sys 33 import time 34 35 from . import execution 36 from . import junit_output 37 from . import statusfile 38 39 40 ABS_PATH_PREFIX = os.getcwd() + os.sep 41 42 43 class ProgressIndicator(object): 44 45 def __init__(self): 46 self.runner = None 47 48 def SetRunner(self, runner): 49 self.runner = runner 50 51 def Starting(self): 52 pass 53 54 def Done(self): 55 pass 56 57 def HasRun(self, test, has_unexpected_output): 58 pass 59 60 def Heartbeat(self): 61 pass 62 63 def PrintFailureHeader(self, test): 64 if test.suite.IsNegativeTest(test): 65 negative_marker = '[negative] ' 66 else: 67 negative_marker = '' 68 print "=== %(label)s %(negative)s===" % { 69 'label': test.GetLabel(), 70 'negative': negative_marker 71 } 72 73 def _EscapeCommand(self, test): 74 command = execution.GetCommand(test, self.runner.context) 75 parts = [] 76 for part in command: 77 if ' ' in part: 78 # Escape spaces. We may need to escape more characters for this 79 # to work properly. 80 parts.append('"%s"' % part) 81 else: 82 parts.append(part) 83 return " ".join(parts) 84 85 86 class IndicatorNotifier(object): 87 """Holds a list of progress indicators and notifies them all on events.""" 88 def __init__(self): 89 self.indicators = [] 90 91 def Register(self, indicator): 92 self.indicators.append(indicator) 93 94 95 # Forge all generic event-dispatching methods in IndicatorNotifier, which are 96 # part of the ProgressIndicator interface. 97 for func_name in ProgressIndicator.__dict__: 98 func = getattr(ProgressIndicator, func_name) 99 if callable(func) and not func.__name__.startswith('_'): 100 def wrap_functor(f): 101 @wraps(f) 102 def functor(self, *args, **kwargs): 103 """Generic event dispatcher.""" 104 for indicator in self.indicators: 105 getattr(indicator, f.__name__)(*args, **kwargs) 106 return functor 107 setattr(IndicatorNotifier, func_name, wrap_functor(func)) 108 109 110 class SimpleProgressIndicator(ProgressIndicator): 111 """Abstract base class for {Verbose,Dots}ProgressIndicator""" 112 113 def Starting(self): 114 print 'Running %i tests' % self.runner.total 115 116 def Done(self): 117 print 118 for failed in self.runner.failed: 119 self.PrintFailureHeader(failed) 120 if failed.output.stderr: 121 print "--- stderr ---" 122 print failed.output.stderr.strip() 123 if failed.output.stdout: 124 print "--- stdout ---" 125 print failed.output.stdout.strip() 126 print "Command: %s" % self._EscapeCommand(failed) 127 if failed.output.HasCrashed(): 128 print "exit code: %d" % failed.output.exit_code 129 print "--- CRASHED ---" 130 if failed.output.HasTimedOut(): 131 print "--- TIMEOUT ---" 132 if len(self.runner.failed) == 0: 133 print "===" 134 print "=== All tests succeeded" 135 print "===" 136 else: 137 print 138 print "===" 139 print "=== %i tests failed" % len(self.runner.failed) 140 if self.runner.crashed > 0: 141 print "=== %i tests CRASHED" % self.runner.crashed 142 print "===" 143 144 145 class VerboseProgressIndicator(SimpleProgressIndicator): 146 147 def HasRun(self, test, has_unexpected_output): 148 if has_unexpected_output: 149 if test.output.HasCrashed(): 150 outcome = 'CRASH' 151 else: 152 outcome = 'FAIL' 153 else: 154 outcome = 'pass' 155 print 'Done running %s: %s' % (test.GetLabel(), outcome) 156 sys.stdout.flush() 157 158 def Heartbeat(self): 159 print 'Still working...' 160 sys.stdout.flush() 161 162 163 class DotsProgressIndicator(SimpleProgressIndicator): 164 165 def HasRun(self, test, has_unexpected_output): 166 total = self.runner.succeeded + len(self.runner.failed) 167 if (total > 1) and (total % 50 == 1): 168 sys.stdout.write('\n') 169 if has_unexpected_output: 170 if test.output.HasCrashed(): 171 sys.stdout.write('C') 172 sys.stdout.flush() 173 elif test.output.HasTimedOut(): 174 sys.stdout.write('T') 175 sys.stdout.flush() 176 else: 177 sys.stdout.write('F') 178 sys.stdout.flush() 179 else: 180 sys.stdout.write('.') 181 sys.stdout.flush() 182 183 184 class CompactProgressIndicator(ProgressIndicator): 185 """Abstract base class for {Color,Monochrome}ProgressIndicator""" 186 187 def __init__(self, templates): 188 super(CompactProgressIndicator, self).__init__() 189 self.templates = templates 190 self.last_status_length = 0 191 self.start_time = time.time() 192 193 def Done(self): 194 self.PrintProgress('Done') 195 print "" # Line break. 196 197 def HasRun(self, test, has_unexpected_output): 198 self.PrintProgress(test.GetLabel()) 199 if has_unexpected_output: 200 self.ClearLine(self.last_status_length) 201 self.PrintFailureHeader(test) 202 stdout = test.output.stdout.strip() 203 if len(stdout): 204 print self.templates['stdout'] % stdout 205 stderr = test.output.stderr.strip() 206 if len(stderr): 207 print self.templates['stderr'] % stderr 208 print "Command: %s" % self._EscapeCommand(test) 209 if test.output.HasCrashed(): 210 print "exit code: %d" % test.output.exit_code 211 print "--- CRASHED ---" 212 if test.output.HasTimedOut(): 213 print "--- TIMEOUT ---" 214 215 def Truncate(self, string, length): 216 if length and (len(string) > (length - 3)): 217 return string[:(length - 3)] + "..." 218 else: 219 return string 220 221 def PrintProgress(self, name): 222 self.ClearLine(self.last_status_length) 223 elapsed = time.time() - self.start_time 224 progress = 0 if not self.runner.total else ( 225 ((self.runner.total - self.runner.remaining) * 100) // 226 self.runner.total) 227 status = self.templates['status_line'] % { 228 'passed': self.runner.succeeded, 229 'progress': progress, 230 'failed': len(self.runner.failed), 231 'test': name, 232 'mins': int(elapsed) / 60, 233 'secs': int(elapsed) % 60 234 } 235 status = self.Truncate(status, 78) 236 self.last_status_length = len(status) 237 print status, 238 sys.stdout.flush() 239 240 241 class ColorProgressIndicator(CompactProgressIndicator): 242 243 def __init__(self): 244 templates = { 245 'status_line': ("[%(mins)02i:%(secs)02i|" 246 "\033[34m%%%(progress) 4d\033[0m|" 247 "\033[32m+%(passed) 4d\033[0m|" 248 "\033[31m-%(failed) 4d\033[0m]: %(test)s"), 249 'stdout': "\033[1m%s\033[0m", 250 'stderr': "\033[31m%s\033[0m", 251 } 252 super(ColorProgressIndicator, self).__init__(templates) 253 254 def ClearLine(self, last_line_length): 255 print "\033[1K\r", 256 257 258 class MonochromeProgressIndicator(CompactProgressIndicator): 259 260 def __init__(self): 261 templates = { 262 'status_line': ("[%(mins)02i:%(secs)02i|%%%(progress) 4d|" 263 "+%(passed) 4d|-%(failed) 4d]: %(test)s"), 264 'stdout': '%s', 265 'stderr': '%s', 266 } 267 super(MonochromeProgressIndicator, self).__init__(templates) 268 269 def ClearLine(self, last_line_length): 270 print ("\r" + (" " * last_line_length) + "\r"), 271 272 273 class JUnitTestProgressIndicator(ProgressIndicator): 274 275 def __init__(self, junitout, junittestsuite): 276 self.outputter = junit_output.JUnitTestOutput(junittestsuite) 277 if junitout: 278 self.outfile = open(junitout, "w") 279 else: 280 self.outfile = sys.stdout 281 282 def Done(self): 283 self.outputter.FinishAndWrite(self.outfile) 284 if self.outfile != sys.stdout: 285 self.outfile.close() 286 287 def HasRun(self, test, has_unexpected_output): 288 fail_text = "" 289 if has_unexpected_output: 290 stdout = test.output.stdout.strip() 291 if len(stdout): 292 fail_text += "stdout:\n%s\n" % stdout 293 stderr = test.output.stderr.strip() 294 if len(stderr): 295 fail_text += "stderr:\n%s\n" % stderr 296 fail_text += "Command: %s" % self._EscapeCommand(test) 297 if test.output.HasCrashed(): 298 fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code 299 if test.output.HasTimedOut(): 300 fail_text += "--- TIMEOUT ---" 301 self.outputter.HasRunTest( 302 [test.GetLabel()] + self.runner.context.mode_flags + test.flags, 303 test.duration, 304 fail_text) 305 306 307 class JsonTestProgressIndicator(ProgressIndicator): 308 309 def __init__(self, json_test_results, arch, mode, random_seed): 310 self.json_test_results = json_test_results 311 self.arch = arch 312 self.mode = mode 313 self.random_seed = random_seed 314 self.results = [] 315 self.tests = [] 316 317 def Done(self): 318 complete_results = [] 319 if os.path.exists(self.json_test_results): 320 with open(self.json_test_results, "r") as f: 321 # Buildbot might start out with an empty file. 322 complete_results = json.loads(f.read() or "[]") 323 324 duration_mean = None 325 if self.tests: 326 # Get duration mean. 327 duration_mean = ( 328 sum(t.duration for t in self.tests) / float(len(self.tests))) 329 330 # Sort tests by duration. 331 timed_tests = [t for t in self.tests if t.duration is not None] 332 timed_tests.sort(lambda a, b: cmp(b.duration, a.duration)) 333 slowest_tests = [ 334 { 335 "name": test.GetLabel(), 336 "flags": test.flags, 337 "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""), 338 "duration": test.duration, 339 "marked_slow": statusfile.IsSlow(test.outcomes), 340 } for test in timed_tests[:20] 341 ] 342 343 complete_results.append({ 344 "arch": self.arch, 345 "mode": self.mode, 346 "results": self.results, 347 "slowest_tests": slowest_tests, 348 "duration_mean": duration_mean, 349 "test_total": len(self.tests), 350 }) 351 352 with open(self.json_test_results, "w") as f: 353 f.write(json.dumps(complete_results)) 354 355 def HasRun(self, test, has_unexpected_output): 356 # Buffer all tests for sorting the durations in the end. 357 self.tests.append(test) 358 if not has_unexpected_output: 359 # Omit tests that run as expected. Passing tests of reruns after failures 360 # will have unexpected_output to be reported here has well. 361 return 362 363 self.results.append({ 364 "name": test.GetLabel(), 365 "flags": test.flags, 366 "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""), 367 "run": test.run, 368 "stdout": test.output.stdout, 369 "stderr": test.output.stderr, 370 "exit_code": test.output.exit_code, 371 "result": test.suite.GetOutcome(test), 372 "expected": list(test.outcomes or ["PASS"]), 373 "duration": test.duration, 374 375 # TODO(machenbach): This stores only the global random seed from the 376 # context and not possible overrides when using random-seed stress. 377 "random_seed": self.random_seed, 378 "target_name": test.suite.shell(), 379 "variant": test.variant, 380 }) 381 382 383 PROGRESS_INDICATORS = { 384 'verbose': VerboseProgressIndicator, 385 'dots': DotsProgressIndicator, 386 'color': ColorProgressIndicator, 387 'mono': MonochromeProgressIndicator 388 } 389