1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2016 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 18 """utils.py: export utility functions. 19 """ 20 21 from __future__ import print_function 22 import logging 23 import os 24 import os.path 25 import shutil 26 import subprocess 27 import sys 28 import time 29 30 def get_script_dir(): 31 return os.path.dirname(os.path.realpath(__file__)) 32 33 def is_windows(): 34 return sys.platform == 'win32' or sys.platform == 'cygwin' 35 36 def is_darwin(): 37 return sys.platform == 'darwin' 38 39 def get_platform(): 40 if is_windows(): 41 return 'windows' 42 if is_darwin(): 43 return 'darwin' 44 return 'linux' 45 46 def is_python3(): 47 return sys.version_info >= (3, 0) 48 49 50 def log_debug(msg): 51 logging.debug(msg) 52 53 54 def log_info(msg): 55 logging.info(msg) 56 57 58 def log_warning(msg): 59 logging.warning(msg) 60 61 62 def log_fatal(msg): 63 raise Exception(msg) 64 65 def log_exit(msg): 66 sys.exit(msg) 67 68 def disable_debug_log(): 69 logging.getLogger().setLevel(logging.WARN) 70 71 def str_to_bytes(str): 72 if not is_python3(): 73 return str 74 # In python 3, str are wide strings whereas the C api expects 8 bit strings, 75 # hence we have to convert. For now using utf-8 as the encoding. 76 return str.encode('utf-8') 77 78 def bytes_to_str(bytes): 79 if not is_python3(): 80 return bytes 81 return bytes.decode('utf-8') 82 83 def get_target_binary_path(arch, binary_name): 84 if arch == 'aarch64': 85 arch = 'arm64' 86 arch_dir = os.path.join(get_script_dir(), "bin", "android", arch) 87 if not os.path.isdir(arch_dir): 88 log_fatal("can't find arch directory: %s" % arch_dir) 89 binary_path = os.path.join(arch_dir, binary_name) 90 if not os.path.isfile(binary_path): 91 log_fatal("can't find binary: %s" % binary_path) 92 return binary_path 93 94 95 def get_host_binary_path(binary_name): 96 dir = os.path.join(get_script_dir(), 'bin') 97 if is_windows(): 98 if binary_name.endswith('.so'): 99 binary_name = binary_name[0:-3] + '.dll' 100 elif '.' not in binary_name: 101 binary_name += '.exe' 102 dir = os.path.join(dir, 'windows') 103 elif sys.platform == 'darwin': # OSX 104 if binary_name.endswith('.so'): 105 binary_name = binary_name[0:-3] + '.dylib' 106 dir = os.path.join(dir, 'darwin') 107 else: 108 dir = os.path.join(dir, 'linux') 109 dir = os.path.join(dir, 'x86_64' if sys.maxsize > 2 ** 32 else 'x86') 110 binary_path = os.path.join(dir, binary_name) 111 if not os.path.isfile(binary_path): 112 log_fatal("can't find binary: %s" % binary_path) 113 return binary_path 114 115 116 def is_executable_available(executable, option='--help'): 117 """ Run an executable to see if it exists. """ 118 try: 119 subproc = subprocess.Popen([executable, option], stdout=subprocess.PIPE, 120 stderr=subprocess.PIPE) 121 subproc.communicate() 122 return subproc.returncode == 0 123 except: 124 return False 125 126 DEFAULT_NDK_PATH = { 127 'darwin': 'Library/Android/sdk/ndk-bundle', 128 'linux': 'Android/Sdk/ndk-bundle', 129 'windows': 'AppData/Local/Android/sdk/ndk-bundle', 130 } 131 132 EXPECTED_TOOLS = { 133 'adb': { 134 'is_binutils': False, 135 'test_option': 'version', 136 'path_in_ndk': '../platform-tools/adb', 137 }, 138 'readelf': { 139 'is_binutils': True, 140 'accept_tool_without_arch': True, 141 }, 142 'addr2line': { 143 'is_binutils': True, 144 'accept_tool_without_arch': True 145 }, 146 'objdump': { 147 'is_binutils': True, 148 }, 149 } 150 151 def _get_binutils_path_in_ndk(toolname, arch, platform): 152 if not arch: 153 arch = 'arm64' 154 if arch == 'arm64': 155 name = 'aarch64-linux-android-' + toolname 156 path = 'toolchains/aarch64-linux-android-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name) 157 elif arch == 'arm': 158 name = 'arm-linux-androideabi-' + toolname 159 path = 'toolchains/arm-linux-androideabi-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name) 160 elif arch == 'x86_64': 161 name = 'x86_64-linux-android-' + toolname 162 path = 'toolchains/x86_64-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name) 163 elif arch == 'x86': 164 name = 'i686-linux-android-' + toolname 165 path = 'toolchains/x86-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name) 166 else: 167 log_fatal('unexpected arch %s' % arch) 168 return (name, path) 169 170 def find_tool_path(toolname, ndk_path=None, arch=None): 171 if toolname not in EXPECTED_TOOLS: 172 return None 173 tool_info = EXPECTED_TOOLS[toolname] 174 is_binutils = tool_info['is_binutils'] 175 test_option = tool_info.get('test_option', '--help') 176 platform = get_platform() 177 if is_binutils: 178 toolname_with_arch, path_in_ndk = _get_binutils_path_in_ndk(toolname, arch, platform) 179 else: 180 toolname_with_arch = toolname 181 path_in_ndk = tool_info['path_in_ndk'] 182 path_in_ndk = path_in_ndk.replace('/', os.sep) 183 184 # 1. Find tool in the given ndk path. 185 if ndk_path: 186 path = os.path.join(ndk_path, path_in_ndk) 187 if is_executable_available(path, test_option): 188 return path 189 190 # 2. Find tool in the ndk directory containing simpleperf scripts. 191 path = os.path.join('..', path_in_ndk) 192 if is_executable_available(path, test_option): 193 return path 194 195 # 3. Find tool in the default ndk installation path. 196 home = os.environ.get('HOMEPATH') if is_windows() else os.environ.get('HOME') 197 if home: 198 default_ndk_path = os.path.join(home, DEFAULT_NDK_PATH[platform].replace('/', os.sep)) 199 path = os.path.join(default_ndk_path, path_in_ndk) 200 if is_executable_available(path, test_option): 201 return path 202 203 # 4. Find tool in $PATH. 204 if is_executable_available(toolname_with_arch, test_option): 205 return toolname_with_arch 206 207 # 5. Find tool without arch in $PATH. 208 if is_binutils and tool_info.get('accept_tool_without_arch'): 209 if is_executable_available(toolname, test_option): 210 return toolname 211 return None 212 213 214 class AdbHelper(object): 215 def __init__(self, enable_switch_to_root=True): 216 adb_path = find_tool_path('adb') 217 if not adb_path: 218 log_exit("Can't find adb in PATH environment.") 219 self.adb_path = adb_path 220 self.enable_switch_to_root = enable_switch_to_root 221 222 223 def run(self, adb_args): 224 return self.run_and_return_output(adb_args)[0] 225 226 227 def run_and_return_output(self, adb_args, stdout_file=None, log_output=True): 228 adb_args = [self.adb_path] + adb_args 229 log_debug('run adb cmd: %s' % adb_args) 230 if stdout_file: 231 with open(stdout_file, 'wb') as stdout_fh: 232 returncode = subprocess.call(adb_args, stdout=stdout_fh) 233 stdoutdata = '' 234 else: 235 subproc = subprocess.Popen(adb_args, stdout=subprocess.PIPE) 236 (stdoutdata, _) = subproc.communicate() 237 returncode = subproc.returncode 238 result = (returncode == 0) 239 if stdoutdata and adb_args[1] != 'push' and adb_args[1] != 'pull': 240 stdoutdata = bytes_to_str(stdoutdata) 241 if log_output: 242 log_debug(stdoutdata) 243 log_debug('run adb cmd: %s [result %s]' % (adb_args, result)) 244 return (result, stdoutdata) 245 246 def check_run(self, adb_args): 247 self.check_run_and_return_output(adb_args) 248 249 250 def check_run_and_return_output(self, adb_args, stdout_file=None, log_output=True): 251 result, stdoutdata = self.run_and_return_output(adb_args, stdout_file, log_output) 252 if not result: 253 log_exit('run "adb %s" failed' % adb_args) 254 return stdoutdata 255 256 257 def _unroot(self): 258 result, stdoutdata = self.run_and_return_output(['shell', 'whoami']) 259 if not result: 260 return 261 if 'root' not in stdoutdata: 262 return 263 log_info('unroot adb') 264 self.run(['unroot']) 265 self.run(['wait-for-device']) 266 time.sleep(1) 267 268 269 def switch_to_root(self): 270 if not self.enable_switch_to_root: 271 self._unroot() 272 return False 273 result, stdoutdata = self.run_and_return_output(['shell', 'whoami']) 274 if not result: 275 return False 276 if 'root' in stdoutdata: 277 return True 278 build_type = self.get_property('ro.build.type') 279 if build_type == 'user': 280 return False 281 self.run(['root']) 282 time.sleep(1) 283 self.run(['wait-for-device']) 284 result, stdoutdata = self.run_and_return_output(['shell', 'whoami']) 285 return result and 'root' in stdoutdata 286 287 def get_property(self, name): 288 result, stdoutdata = self.run_and_return_output(['shell', 'getprop', name]) 289 return stdoutdata if result else None 290 291 def set_property(self, name, value): 292 return self.run(['shell', 'setprop', name, value]) 293 294 295 def get_device_arch(self): 296 output = self.check_run_and_return_output(['shell', 'uname', '-m']) 297 if 'aarch64' in output: 298 return 'arm64' 299 if 'arm' in output: 300 return 'arm' 301 if 'x86_64' in output: 302 return 'x86_64' 303 if '86' in output: 304 return 'x86' 305 log_fatal('unsupported architecture: %s' % output.strip()) 306 307 308 def get_android_version(self): 309 build_version = self.get_property('ro.build.version.release') 310 android_version = 0 311 if build_version: 312 if not build_version[0].isdigit(): 313 c = build_version[0].upper() 314 if c.isupper() and c >= 'L': 315 android_version = ord(c) - ord('L') + 5 316 else: 317 strs = build_version.split('.') 318 if strs: 319 android_version = int(strs[0]) 320 return android_version 321 322 323 def flatten_arg_list(arg_list): 324 res = [] 325 if arg_list: 326 for items in arg_list: 327 res += items 328 return res 329 330 331 def remove(dir_or_file): 332 if os.path.isfile(dir_or_file): 333 os.remove(dir_or_file) 334 elif os.path.isdir(dir_or_file): 335 shutil.rmtree(dir_or_file, ignore_errors=True) 336 337 338 def open_report_in_browser(report_path): 339 if is_darwin(): 340 # On darwin 10.12.6, webbrowser can't open browser, so try `open` cmd first. 341 try: 342 subprocess.check_call(['open', report_path]) 343 return 344 except: 345 pass 346 import webbrowser 347 try: 348 # Try to open the report with Chrome 349 browser_key = '' 350 for key, _ in webbrowser._browsers.items(): 351 if 'chrome' in key: 352 browser_key = key 353 browser = webbrowser.get(browser_key) 354 browser.open(report_path, new=0, autoraise=True) 355 except: 356 # webbrowser.get() doesn't work well on darwin/windows. 357 webbrowser.open_new_tab(report_path) 358 359 360 def find_real_dso_path(dso_path_in_record_file, binary_cache_path): 361 """ Given the path of a shared library in perf.data, find its real path in the file system. """ 362 if dso_path_in_record_file[0] != '/' or dso_path_in_record_file == '//anon': 363 return None 364 if binary_cache_path: 365 tmp_path = os.path.join(binary_cache_path, dso_path_in_record_file[1:]) 366 if os.path.isfile(tmp_path): 367 return tmp_path 368 if os.path.isfile(dso_path_in_record_file): 369 return dso_path_in_record_file 370 return None 371 372 def get_arch_of_dso_path(readelf_path, dso_path): 373 try: 374 output = subprocess.check_output([readelf_path, '-h', dso_path]) 375 if output.find('AArch64') != -1: 376 return 'arm64' 377 if output.find('ARM') != -1: 378 return 'arm' 379 if output.find('X86-64') != -1: 380 return 'x86_64' 381 if output.find('80386') != -1: 382 return 'x86' 383 except subprocess.CalledProcessError: 384 pass 385 return 'unknown' 386 387 388 class Addr2Nearestline(object): 389 """ Use addr2line to convert (dso_path, func_addr, addr) to (source_file, line) pairs. 390 For instructions generated by C++ compilers without a matching statement in source code 391 (like stack corruption check, switch optimization, etc.), addr2line can't generate 392 line information. However, we want to assign the instruction to the nearest line before 393 the instruction (just like objdump -dl). So we use below strategy: 394 Instead of finding the exact line of the instruction in an address, we find the nearest 395 line to the instruction in an address. If an address doesn't have a line info, we find 396 the line info of address - 1. If still no line info, then use address - 2, address - 3, 397 etc. 398 399 The implementation steps are as below: 400 1. Collect all (dso_path, func_addr, addr) requests before converting. This saves the 401 times to call addr2line. 402 2. Convert addrs to (source_file, line) pairs for each dso_path as below: 403 2.1 Check if the dso_path has .debug_line. If not, omit its conversion. 404 2.2 Get arch of the dso_path, and decide the addr_step for it. addr_step is the step we 405 change addr each time. For example, since instructions of arm64 are all 4 bytes long, 406 addr_step for arm64 can be 4. 407 2.3 Use addr2line to find line info for each addr in the dso_path. 408 2.4 For each addr without line info, use addr2line to find line info for 409 range(addr - addr_step, addr - addr_step * 4 - 1, -addr_step). 410 2.5 For each addr without line info, use addr2line to find line info for 411 range(addr - addr_step * 5, addr - addr_step * 128 - 1, -addr_step). 412 (128 is a guess number. A nested switch statement in 413 system/core/demangle/Demangler.cpp has >300 bytes without line info in arm64.) 414 """ 415 class Dso(object): 416 """ Info of a dynamic shared library. 417 addrs: a map from address to Addr object in this dso. 418 """ 419 def __init__(self): 420 self.addrs = {} 421 422 class Addr(object): 423 """ Info of an addr request. 424 func_addr: start_addr of the function containing addr. 425 source_lines: a list of [file_id, line_number] for addr. 426 source_lines[:-1] are all for inlined functions. 427 """ 428 def __init__(self, func_addr): 429 self.func_addr = func_addr 430 self.source_lines = None 431 432 def __init__(self, ndk_path, binary_cache_path): 433 self.addr2line_path = find_tool_path('addr2line', ndk_path) 434 if not self.addr2line_path: 435 log_exit("Can't find addr2line. Please set ndk path by --ndk-path option.") 436 self.readelf_path = find_tool_path('readelf', ndk_path) 437 if not self.readelf_path: 438 log_exit("Can't find readelf. Please set ndk path by --ndk-path option.") 439 self.dso_map = {} # map from dso_path to Dso. 440 self.binary_cache_path = binary_cache_path 441 # Saving file names for each addr takes a lot of memory. So we store file ids in Addr, 442 # and provide data structures connecting file id and file name here. 443 self.file_name_to_id = {} 444 self.file_id_to_name = [] 445 446 def add_addr(self, dso_path, func_addr, addr): 447 dso = self.dso_map.get(dso_path) 448 if dso is None: 449 dso = self.dso_map[dso_path] = self.Dso() 450 if addr not in dso.addrs: 451 dso.addrs[addr] = self.Addr(func_addr) 452 453 def convert_addrs_to_lines(self): 454 for dso_path in self.dso_map: 455 self._convert_addrs_in_one_dso(dso_path, self.dso_map[dso_path]) 456 457 def _convert_addrs_in_one_dso(self, dso_path, dso): 458 real_path = find_real_dso_path(dso_path, self.binary_cache_path) 459 if not real_path: 460 if dso_path not in ['//anon', 'unknown', '[kernel.kallsyms]']: 461 log_debug("Can't find dso %s" % dso_path) 462 return 463 464 if not self._check_debug_line_section(real_path): 465 log_debug("file %s doesn't contain .debug_line section." % real_path) 466 return 467 468 addr_step = self._get_addr_step(real_path) 469 self._collect_line_info(dso, real_path, [0]) 470 self._collect_line_info(dso, real_path, range(-addr_step, -addr_step * 4 - 1, -addr_step)) 471 self._collect_line_info(dso, real_path, 472 range(-addr_step * 5, -addr_step * 128 - 1, -addr_step)) 473 474 def _check_debug_line_section(self, real_path): 475 try: 476 output = subprocess.check_output([self.readelf_path, '-S', real_path]) 477 return output.find('.debug_line') != -1 478 except subprocess.CalledProcessError: 479 return False 480 481 def _get_addr_step(self, real_path): 482 arch = get_arch_of_dso_path(self.readelf_path, real_path) 483 if arch == 'arm64': 484 return 4 485 if arch == 'arm': 486 return 2 487 return 1 488 489 def _collect_line_info(self, dso, real_path, addr_shifts): 490 """ Use addr2line to get line info in a dso, with given addr shifts. """ 491 # 1. Collect addrs to send to addr2line. 492 addr_set = set() 493 for addr in dso.addrs: 494 addr_obj = dso.addrs[addr] 495 if addr_obj.source_lines: # already has source line, no need to search. 496 continue 497 for shift in addr_shifts: 498 # The addr after shift shouldn't change to another function. 499 shifted_addr = max(addr + shift, addr_obj.func_addr) 500 addr_set.add(shifted_addr) 501 if shifted_addr == addr_obj.func_addr: 502 break 503 if not addr_set: 504 return 505 addr_request = '\n'.join(['%x' % addr for addr in sorted(addr_set)]) 506 507 # 2. Use addr2line to collect line info. 508 try: 509 subproc = subprocess.Popen([self.addr2line_path, '-ai', '-e', real_path], 510 stdin=subprocess.PIPE, stdout=subprocess.PIPE) 511 (stdoutdata, _) = subproc.communicate(str_to_bytes(addr_request)) 512 stdoutdata = bytes_to_str(stdoutdata) 513 except: 514 return 515 addr_map = {} 516 cur_line_list = None 517 for line in stdoutdata.strip().split('\n'): 518 if line[:2] == '0x': 519 # a new address 520 cur_line_list = addr_map[int(line, 16)] = [] 521 else: 522 # a file:line. 523 if cur_line_list is None: 524 continue 525 # Handle lines like "C:\Users\...\file:32". 526 items = line.rsplit(':', 1) 527 if len(items) != 2: 528 continue 529 if '?' in line: 530 # if ? in line, it doesn't have a valid line info. 531 # An addr can have a list of (file, line), when the addr belongs to an inlined 532 # function. Sometimes only part of the list has ? mark. In this case, we think 533 # the line info is valid if the first line doesn't have ? mark. 534 if not cur_line_list: 535 cur_line_list = None 536 continue 537 (file_path, line_number) = items 538 line_number = line_number.split()[0] # Remove comments after line number 539 try: 540 line_number = int(line_number) 541 except ValueError: 542 continue 543 file_id = self._get_file_id(file_path) 544 cur_line_list.append((file_id, line_number)) 545 546 # 3. Fill line info in dso.addrs. 547 for addr in dso.addrs: 548 addr_obj = dso.addrs[addr] 549 if addr_obj.source_lines: 550 continue 551 for shift in addr_shifts: 552 shifted_addr = max(addr + shift, addr_obj.func_addr) 553 lines = addr_map.get(shifted_addr) 554 if lines: 555 addr_obj.source_lines = lines 556 break 557 if shifted_addr == addr_obj.func_addr: 558 break 559 560 def _get_file_id(self, file_path): 561 file_id = self.file_name_to_id.get(file_path) 562 if file_id is None: 563 file_id = self.file_name_to_id[file_path] = len(self.file_id_to_name) 564 self.file_id_to_name.append(file_path) 565 return file_id 566 567 def get_dso(self, dso_path): 568 return self.dso_map.get(dso_path) 569 570 def get_addr_source(self, dso, addr): 571 source = dso.addrs[addr].source_lines 572 if source is None: 573 return None 574 return [(self.file_id_to_name[file_id], line) for (file_id, line) in source] 575 576 577 class Objdump(object): 578 """ A wrapper of objdump to disassemble code. """ 579 def __init__(self, ndk_path, binary_cache_path): 580 self.ndk_path = ndk_path 581 self.binary_cache_path = binary_cache_path 582 self.readelf_path = find_tool_path('readelf', ndk_path) 583 if not self.readelf_path: 584 log_exit("Can't find readelf. Please set ndk path by --ndk_path option.") 585 self.objdump_paths = {} 586 587 def disassemble_code(self, dso_path, start_addr, addr_len): 588 """ Disassemble [start_addr, start_addr + addr_len] of dso_path. 589 Return a list of pair (disassemble_code_line, addr). 590 """ 591 # 1. Find real path. 592 real_path = find_real_dso_path(dso_path, self.binary_cache_path) 593 if real_path is None: 594 return None 595 596 # 2. Get path of objdump. 597 arch = get_arch_of_dso_path(self.readelf_path, real_path) 598 objdump_path = self.objdump_paths.get(arch) 599 if not objdump_path: 600 objdump_path = find_tool_path('objdump', self.ndk_path, arch) 601 if not objdump_path: 602 log_exit("Can't find objdump. Please set ndk path by --ndk_path option.") 603 self.objdump_paths[arch] = objdump_path 604 605 # 3. Run objdump. 606 args = [objdump_path, '-dlC', '--no-show-raw-insn', 607 '--start-address=0x%x' % start_addr, 608 '--stop-address=0x%x' % (start_addr + addr_len), 609 real_path] 610 try: 611 subproc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 612 (stdoutdata, _) = subproc.communicate() 613 stdoutdata = bytes_to_str(stdoutdata) 614 except: 615 return None 616 617 if not stdoutdata: 618 return None 619 result = [] 620 for line in stdoutdata.split('\n'): 621 line = line.rstrip() # Remove '\r' on Windows. 622 items = line.split(':', 1) 623 try: 624 addr = int(items[0], 16) 625 except ValueError: 626 addr = 0 627 result.append((line, addr)) 628 return result 629 630 631 logging.getLogger().setLevel(logging.DEBUG) 632
