1 #!/usr/bin/python2.4 2 3 """Run reliability tests using Android instrumentation. 4 5 A test file consists of list web sites to test is needed as a parameter 6 7 Usage: 8 run_reliability_tests.py path/to/url/list 9 """ 10 11 import logging 12 import optparse 13 import os 14 import subprocess 15 import sys 16 import time 17 from Numeric import * 18 19 TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt" 20 TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt" 21 TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt" 22 TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt" 23 HTTP_URL_FILE = "urllist_http" 24 HTTPS_URL_FILE = "urllist_https" 25 NUM_URLS = 25 26 27 28 def DumpRenderTreeFinished(adb_cmd): 29 """Check if DumpRenderTree finished running. 30 31 Args: 32 adb_cmd: adb command string 33 34 Returns: 35 True if DumpRenderTree has finished, False otherwise 36 """ 37 38 # pull test status file and look for "#DONE" 39 shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 40 adb_output = subprocess.Popen(shell_cmd_str, 41 shell=True, stdout=subprocess.PIPE, 42 stderr=subprocess.PIPE).communicate()[0] 43 return adb_output.strip() == "#DONE" 44 45 46 def RemoveDeviceFile(adb_cmd, file_name): 47 shell_cmd_str = adb_cmd + " shell rm " + file_name 48 subprocess.Popen(shell_cmd_str, 49 shell=True, stdout=subprocess.PIPE, 50 stderr=subprocess.PIPE).communicate() 51 52 53 def Bugreport(url, bugreport_dir, adb_cmd): 54 """Pull a bugreport from the device.""" 55 bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir, 56 int(time.time())) 57 58 # prepend the report with url 59 handle = open(bugreport_filename, "w") 60 handle.writelines("Bugreport for crash in url - %s\n\n" % url) 61 handle.close() 62 63 cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename) 64 os.system(cmd) 65 66 67 def ProcessPageLoadTime(raw_log): 68 """Processes the raw page load time logged by test app.""" 69 log_handle = open(raw_log, "r") 70 load_times = {} 71 72 for line in log_handle: 73 line = line.strip() 74 pair = line.split("|") 75 if len(pair) != 2: 76 logging.info("Line has more than one '|': " + line) 77 continue 78 if pair[0] not in load_times: 79 load_times[pair[0]] = [] 80 try: 81 pair[1] = int(pair[1]) 82 except ValueError: 83 logging.info("Lins has non-numeric load time: " + line) 84 continue 85 load_times[pair[0]].append(pair[1]) 86 87 log_handle.close() 88 89 # rewrite the average time to file 90 log_handle = open(raw_log, "w") 91 for url, times in load_times.iteritems(): 92 # calculate std 93 arr = array(times) 94 avg = average(arr) 95 d = arr - avg 96 std = sqrt(sum(d * d) / len(arr)) 97 output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" % 98 (url, min(arr), max(arr), avg, std, 99 array2string(arr))) 100 log_handle.write(output) 101 log_handle.close() 102 103 104 def main(options, args): 105 """Send the url list to device and start testing, restart if crashed.""" 106 107 # Set up logging format. 108 log_level = logging.INFO 109 if options.verbose: 110 log_level = logging.DEBUG 111 logging.basicConfig(level=log_level, 112 format="%(message)s") 113 114 # Include all tests if none are specified. 115 if not args: 116 print "Missing URL list file" 117 sys.exit(1) 118 else: 119 path = args[0] 120 121 if not options.crash_file: 122 print "Missing crash file name, use --crash-file to specify" 123 sys.exit(1) 124 else: 125 crashed_file = options.crash_file 126 127 if not options.timeout_file: 128 print "Missing timeout file, use --timeout-file to specify" 129 sys.exit(1) 130 else: 131 timedout_file = options.timeout_file 132 133 if not options.delay: 134 manual_delay = 0 135 else: 136 manual_delay = options.delay 137 138 if not options.bugreport: 139 bugreport_dir = "." 140 else: 141 bugreport_dir = options.bugreport 142 if not os.path.exists(bugreport_dir): 143 os.makedirs(bugreport_dir) 144 if not os.path.isdir(bugreport_dir): 145 logging.error("Cannot create results dir: " + bugreport_dir) 146 sys.exit(1) 147 148 adb_cmd = "adb " 149 if options.adb_options: 150 adb_cmd += options.adb_options + " " 151 152 # push url list to device 153 test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\"" 154 proc = subprocess.Popen(test_cmd, shell=True, 155 stdout=subprocess.PIPE, 156 stderr=subprocess.PIPE) 157 (adb_output, adb_error) = proc.communicate() 158 if proc.returncode != 0: 159 logging.error("failed to push url list to device.") 160 logging.error(adb_output) 161 logging.error(adb_error) 162 sys.exit(1) 163 164 # clean up previous results 165 RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE) 166 RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE) 167 RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE) 168 169 logging.info("Running the test ...") 170 171 # Count crashed tests. 172 crashed_tests = [] 173 174 if options.time_out_ms: 175 timeout_ms = options.time_out_ms 176 177 # Run test until it's done 178 test_cmd_prefix = adb_cmd + " shell am instrument" 179 test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner" 180 181 # Call ReliabilityTestsAutoTest#startReliabilityTests 182 test_cmd = (test_cmd_prefix + " -e class " 183 "com.android.dumprendertree.ReliabilityTest#" 184 "runReliabilityTest -e timeout %s -e delay %s" % 185 (str(timeout_ms), str(manual_delay))) 186 187 if options.logtime: 188 test_cmd += " -e logtime true" 189 190 test_cmd += test_cmd_postfix 191 192 adb_output = subprocess.Popen(test_cmd, shell=True, 193 stdout=subprocess.PIPE, 194 stderr=subprocess.PIPE).communicate()[0] 195 while not DumpRenderTreeFinished(adb_cmd): 196 logging.error("DumpRenderTree exited before all URLs are visited.") 197 shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 198 crashed_test = "" 199 while not crashed_test: 200 (crashed_test, err) = subprocess.Popen( 201 shell_cmd_str, shell=True, stdout=subprocess.PIPE, 202 stderr=subprocess.PIPE).communicate() 203 crashed_test = crashed_test.strip() 204 if not crashed_test: 205 logging.error('Cannot get crashed test name, device offline?') 206 logging.error('stderr: ' + err) 207 logging.error('retrying in 10s...') 208 time.sleep(10) 209 210 logging.info(crashed_test + " CRASHED") 211 crashed_tests.append(crashed_test) 212 Bugreport(crashed_test, bugreport_dir, adb_cmd) 213 logging.info("Resuming reliability test runner...") 214 215 adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 216 stderr=subprocess.PIPE).communicate()[0] 217 218 if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or 219 adb_output.find("Process crashed.") != -1): 220 logging.error("Error happened : " + adb_output) 221 sys.exit(1) 222 223 logging.info(adb_output) 224 logging.info("Done\n") 225 226 if crashed_tests: 227 file_handle = open(crashed_file, "w") 228 file_handle.writelines("\n".join(crashed_tests)) 229 logging.info("Crashed URL list stored in: " + crashed_file) 230 file_handle.close() 231 else: 232 logging.info("No crash found.") 233 234 # get timeout file from sdcard 235 test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \"" 236 + timedout_file + "\"") 237 subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 238 stderr=subprocess.PIPE).communicate() 239 240 if options.logtime: 241 # get logged page load times from sdcard 242 test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \"" 243 + options.logtime + "\"") 244 subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 245 stderr=subprocess.PIPE).communicate() 246 ProcessPageLoadTime(options.logtime) 247 248 249 if "__main__" == __name__: 250 option_parser = optparse.OptionParser() 251 option_parser.add_option("-t", "--time-out-ms", 252 default=60000, 253 help="set the timeout for each test") 254 option_parser.add_option("-v", "--verbose", action="store_true", 255 default=False, 256 help="include debug-level logging") 257 option_parser.add_option("-a", "--adb-options", 258 default=None, 259 help="pass options to adb, such as -d -e, etc") 260 option_parser.add_option("-c", "--crash-file", 261 default="reliability_crashed_sites.txt", 262 help="the list of sites that cause browser to crash") 263 option_parser.add_option("-f", "--timeout-file", 264 default="reliability_timedout_sites.txt", 265 help="the list of sites that timedout during test") 266 option_parser.add_option("-d", "--delay", 267 default=0, 268 help="add a manual delay between pages (in ms)") 269 option_parser.add_option("-b", "--bugreport", 270 default=".", 271 help="the directory to store bugreport for crashes") 272 option_parser.add_option("-l", "--logtime", 273 default=None, 274 help="Logs page load time for each url to the file") 275 opts, arguments = option_parser.parse_args() 276 main(opts, arguments) 277
