1 #!/usr/bin/env python 2 # Copyright 2014 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Runs a test repeatedly to measure its flakiness. The return code is non-zero 7 if the failure rate is higher than the specified threshold, but is not 100%.""" 8 9 import argparse 10 import multiprocessing.dummy 11 import subprocess 12 import sys 13 import time 14 15 def load_options(): 16 parser = argparse.ArgumentParser(description=__doc__) 17 parser.add_argument('--retries', default=1000, type=int, 18 help='Number of test retries to measure flakiness.') 19 parser.add_argument('--threshold', default=0.05, type=float, 20 help='Minimum flakiness level at which test is ' 21 'considered flaky.') 22 parser.add_argument('--jobs', '-j', type=int, default=1, 23 help='Number of parallel jobs to run tests.') 24 parser.add_argument('command', nargs='+', help='Command to run test.') 25 return parser.parse_args() 26 27 def run_test(job): 28 print 'Starting retry attempt %d out of %d' % (job['index'] + 1, 29 job['retries']) 30 return subprocess.check_call(job['cmd'], stdout=subprocess.PIPE, 31 stderr=subprocess.STDOUT) 32 33 def main(): 34 options = load_options() 35 num_passed = num_failed = 0 36 running = [] 37 38 pool = multiprocessing.dummy.Pool(processes=options.jobs) 39 args = [{'index': index, 'retries': options.retries, 'cmd': options.command} 40 for index in range(options.retries)] 41 results = pool.map(run_test, args) 42 num_passed = len([retcode for retcode in results if retcode == 0]) 43 num_failed = len(results) - num_passed 44 45 if num_passed == 0: 46 flakiness = 0 47 else: 48 flakiness = num_failed / float(len(results)) 49 50 print 'Flakiness is %.2f' % flakiness 51 if flakiness > options.threshold: 52 return 1 53 else: 54 return 0 55 56 57 if __name__ == '__main__': 58 sys.exit(main()) 59
