1 #!/usr/bin/env python 2 3 # Copyright (C) 2015 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the 'License'); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an 'AS IS' BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ 18 Generates storage benchmark from captured strace output. 19 20 Currently assumes that all mmap'ed regions are resource accesses, and emulates as pread(). 21 22 Usage: 23 $ adb shell strace -p `pid zygote` -o /data/local/tmp/trace -f -ff -y -ttt -e trace=file,desc,munmap 24 $ adb pull /data/local/tmp/trace* 25 $ python benchgen.py trace.* 26 27 """ 28 29 import re, sys, collections, traceback, argparse 30 31 from operator import itemgetter 32 from collections import defaultdict 33 34 class Event: 35 def __init__(self, thread, time, call, args, ret): 36 self.thread = thread 37 self.time = time 38 self.call = call 39 self.args = args 40 self.ret = ret 41 42 def __repr__(self): 43 return "%s(%s)=%s" % (self.call, repr(self.args), self.ret) 44 45 46 class File: 47 def __init__(self, name, ident): 48 self.name = name 49 self.ident = ident 50 self.size = 0 51 52 def __repr__(self): 53 return self.name 54 55 56 events = [] 57 files = {} 58 59 def find_file(name): 60 name = name.strip('<>"') 61 if name not in files: 62 files[name] = File(name, len(files)) 63 return files[name] 64 65 def extract_file(e, arg): 66 if "<" in arg: 67 fd, path = arg.split("<") 68 path = path.strip(">") 69 handle = "t%sf%s" % (e.thread, fd) 70 return (fd, find_file(path), handle) 71 else: 72 return (None, None, None) 73 74 def parse_args(s): 75 args = [] 76 arg = "" 77 esc = False 78 quot = False 79 for c in s: 80 if esc: 81 esc = False 82 arg += c 83 continue 84 85 if c == '"': 86 if quot: 87 quot = False 88 continue 89 else: 90 quot = True 91 continue 92 93 if c == '\\': 94 esc = True 95 continue 96 97 if c == ',' and not quot: 98 args.append(arg.strip()) 99 arg = "" 100 else: 101 arg += c 102 103 args.append(arg.strip()) 104 return args 105 106 107 bufsize = 1048576 108 interesting = ["mmap2","read","write","pread64","pwrite64","fsync","fdatasync","openat","close","lseek","_llseek"] 109 110 re_event = re.compile(r"^([\d\.]+) (.+?)\((.+?)\) = (.+?)$") 111 re_arg = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''') 112 for fn in sys.argv[1:]: 113 with open(fn) as f: 114 thread = int(fn.split(".")[-1]) 115 for line in f: 116 line = re_event.match(line) 117 if not line: continue 118 119 time, call, args, ret = line.groups() 120 if call not in interesting: continue 121 if "/data/" not in args: continue 122 123 time = float(time) 124 args = parse_args(args) 125 events.append(Event(thread, time, call, args, ret)) 126 127 128 with open("BenchmarkGen.h", 'w') as bench: 129 print >>bench, """/* 130 * Copyright (C) 2015 The Android Open Source Project 131 * 132 * Licensed under the Apache License, Version 2.0 (the "License"); 133 * you may not use this file except in compliance with the License. 134 * You may obtain a copy of the License at 135 * 136 * http://www.apache.org/licenses/LICENSE-2.0 137 * 138 * Unless required by applicable law or agreed to in writing, software 139 * distributed under the License is distributed on an "AS IS" BASIS, 140 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 141 * See the License for the specific language governing permissions and 142 * limitations under the License. 143 */ 144 145 146 /****************************************************************** 147 * THIS CODE WAS GENERATED BY benchgen.py, DO NOT MODIFY DIRECTLY * 148 ******************************************************************/ 149 150 151 #include <android-base/logging.h> 152 153 #include <stdlib.h> 154 #include <sys/types.h> 155 #include <sys/stat.h> 156 #include <sys/sendfile.h> 157 #include <fcntl.h> 158 159 #include <algorithm> 160 #include <functional> 161 #include <string> 162 163 #include <Utils.h> 164 165 namespace android { 166 namespace vold { 167 168 static status_t BenchmarkRun(std::function<bool(int)> checkpoint) { 169 170 """ 171 172 print >>bench, "char* buf = (char*) malloc(%d);" % (bufsize) 173 174 nread = 0 175 nwrite = 0 176 nsync = 0 177 events = sorted(events, key=lambda e: e.time) 178 active = set() 179 defined = set() 180 i = 0 181 total = len(events) 182 for e in events: 183 i += 1 184 if i % 256 == 0: 185 print >>bench, "if (!checkpoint(%d)) return -1;" % (50 + ((i * 50) / total)) 186 187 if e.call == "openat": 188 fd, f, handle = extract_file(e, e.ret) 189 if f: 190 active.add(handle) 191 if handle not in defined: 192 print >>bench, "int", 193 defined.add(handle) 194 create_mode = '' 195 if 'O_CREAT' in e.args[2]: 196 assert len(e.args) > 3, 'File creation lacks a mode?' 197 create_mode = ', ' + e.args[3] 198 print >>bench, '%s = TEMP_FAILURE_RETRY(open("file%s", %s%s));' \ 199 % (handle, f.ident, e.args[2], create_mode) 200 201 elif e.call == "close": 202 fd, f, handle = extract_file(e, e.args[0]) 203 if handle in active: 204 active.remove(handle) 205 print >>bench, 'close(%s);' % (handle) 206 207 elif e.call == "lseek": 208 fd, f, handle = extract_file(e, e.args[0]) 209 if handle in active: 210 print >>bench, 'TEMP_FAILURE_RETRY(lseek(%s, %s, %s));' % (handle, e.args[1], e.args[2]) 211 212 elif e.call == "_llseek": 213 fd, f, handle = extract_file(e, e.args[0]) 214 if handle in active: 215 print >>bench, 'TEMP_FAILURE_RETRY(lseek(%s, %s, %s));' % (handle, e.args[1], e.args[3]) 216 217 elif e.call == "read": 218 fd, f, handle = extract_file(e, e.args[0]) 219 if handle in active: 220 # TODO: track actual file size instead of guessing 221 count = min(int(e.args[2]), bufsize) 222 f.size += count 223 print >>bench, 'TEMP_FAILURE_RETRY(read(%s, buf, %d));' % (handle, count) 224 nread += 1 225 226 elif e.call == "write": 227 fd, f, handle = extract_file(e, e.args[0]) 228 if handle in active: 229 # TODO: track actual file size instead of guessing 230 count = min(int(e.args[2]), bufsize) 231 f.size += count 232 print >>bench, 'TEMP_FAILURE_RETRY(write(%s, buf, %d));' % (handle, count) 233 nwrite += 1 234 235 elif e.call == "pread64": 236 fd, f, handle = extract_file(e, e.args[0]) 237 if handle in active: 238 f.size = max(f.size, int(e.args[2]) + int(e.args[3])) 239 count = min(int(e.args[2]), bufsize) 240 print >>bench, 'TEMP_FAILURE_RETRY(pread(%s, buf, %d, %s));' % (handle, count, e.args[3]) 241 nread += 1 242 243 elif e.call == "pwrite64": 244 fd, f, handle = extract_file(e, e.args[0]) 245 if handle in active: 246 f.size = max(f.size, int(e.args[2]) + int(e.args[3])) 247 count = min(int(e.args[2]), bufsize) 248 print >>bench, 'TEMP_FAILURE_RETRY(pwrite(%s, buf, %d, %s));' % (handle, count, e.args[3]) 249 nwrite += 1 250 251 elif e.call == "fsync": 252 fd, f, handle = extract_file(e, e.args[0]) 253 if handle in active: 254 print >>bench, 'TEMP_FAILURE_RETRY(fsync(%s));' % (handle) 255 nsync += 1 256 257 elif e.call == "fdatasync": 258 fd, f, handle = extract_file(e, e.args[0]) 259 if handle in active: 260 print >>bench, 'TEMP_FAILURE_RETRY(fdatasync(%s));' % (handle) 261 nsync += 1 262 263 elif e.call == "mmap2": 264 fd, f, handle = extract_file(e, e.args[4]) 265 if handle in active: 266 count = min(int(e.args[1]), bufsize) 267 offset = int(e.args[5], 0) 268 f.size = max(f.size, count + offset) 269 print >>bench, 'TEMP_FAILURE_RETRY(pread(%s, buf, %s, %s)); // mmap2' % (handle, count, offset) 270 nread += 1 271 272 for handle in active: 273 print >>bench, 'close(%s);' % (handle) 274 275 print >>bench, """ 276 free(buf); 277 return 0; 278 } 279 280 static status_t CreateFile(const char* name, int len) { 281 int chunk = std::min(len, 65536); 282 int out = -1; 283 std::string buf; 284 285 if (android::vold::ReadRandomBytes(chunk, buf) != OK) { 286 LOG(ERROR) << "Failed to read random data"; 287 return -EIO; 288 } 289 if ((out = TEMP_FAILURE_RETRY(open(name, O_WRONLY|O_CREAT|O_TRUNC, 0644))) < 0) { 290 PLOG(ERROR) << "Failed to open " << name; 291 return -errno; 292 } 293 294 while (len > 0) { 295 int n = write(out, buf.c_str(), std::min(len, chunk)); 296 if (n < 0) { 297 PLOG(ERROR) << "Failed to write"; 298 close(out); 299 return -errno; 300 } 301 len -= n; 302 } 303 304 close(out); 305 return OK; 306 } 307 308 static status_t BenchmarkCreate(std::function<bool(int)> checkpoint) { 309 status_t res = 0; 310 res |= CreateFile("stub", 0); 311 """ 312 i = 0 313 total = len(files.values()) 314 for f in files.values(): 315 i += 1 316 if i % 12 == 0: 317 print >>bench, "if (!checkpoint(%d)) return -1;" % ((i * 50) / total) 318 319 print >>bench, 'res |= CreateFile("file%s", %d);' % (f.ident, f.size) 320 321 print >>bench, """ 322 return res; 323 } 324 325 static status_t BenchmarkDestroy() { 326 status_t res = 0; 327 res |= unlink("stub"); 328 """ 329 for f in files.values(): 330 print >>bench, 'res |= unlink("file%s");' % (f.ident) 331 332 print >>bench, """ 333 return res; 334 } 335 336 static std::string BenchmarkIdent() {""" 337 print >>bench, """return "r%d:w%d:s%d";""" % (nread, nwrite, nsync) 338 print >>bench, """} 339 340 } // namespace vold 341 } // namespace android 342 """ 343 344 345 size = sum([ f.size for f in files.values() ]) 346 print "Found", len(files), "data files accessed, total size", (size/1024), "kB" 347 348 types = defaultdict(int) 349 for e in events: 350 types[e.call] += 1 351 352 print "Found syscalls:" 353 for t, n in types.iteritems(): 354 print str(n).rjust(8), t 355 356 print 357
