1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "errno.h" 18 #include <stdio.h> 19 #include <stdlib.h> 20 #include <sys/file.h> 21 #include <sys/stat.h> 22 #include <unistd.h> 23 24 #include <fstream> 25 #include <iostream> 26 #include <set> 27 #include <string> 28 #include <unordered_set> 29 #include <vector> 30 31 #include "android-base/stringprintf.h" 32 #include "android-base/strings.h" 33 34 #include "base/dumpable.h" 35 #include "base/scoped_flock.h" 36 #include "base/stringpiece.h" 37 #include "base/time_utils.h" 38 #include "base/unix_file/fd_file.h" 39 #include "boot_image_profile.h" 40 #include "bytecode_utils.h" 41 #include "dex_file.h" 42 #include "jit/profile_compilation_info.h" 43 #include "profile_assistant.h" 44 #include "runtime.h" 45 #include "type_reference.h" 46 #include "utils.h" 47 #include "type_reference.h" 48 #include "zip_archive.h" 49 50 namespace art { 51 52 static int original_argc; 53 static char** original_argv; 54 55 static std::string CommandLine() { 56 std::vector<std::string> command; 57 for (int i = 0; i < original_argc; ++i) { 58 command.push_back(original_argv[i]); 59 } 60 return android::base::Join(command, ' '); 61 } 62 63 static constexpr int kInvalidFd = -1; 64 65 static bool FdIsValid(int fd) { 66 return fd != kInvalidFd; 67 } 68 69 static void UsageErrorV(const char* fmt, va_list ap) { 70 std::string error; 71 android::base::StringAppendV(&error, fmt, ap); 72 LOG(ERROR) << error; 73 } 74 75 static void UsageError(const char* fmt, ...) { 76 va_list ap; 77 va_start(ap, fmt); 78 UsageErrorV(fmt, ap); 79 va_end(ap); 80 } 81 82 NO_RETURN static void Usage(const char *fmt, ...) { 83 va_list ap; 84 va_start(ap, fmt); 85 UsageErrorV(fmt, ap); 86 va_end(ap); 87 88 UsageError("Command: %s", CommandLine().c_str()); 89 UsageError("Usage: profman [options]..."); 90 UsageError(""); 91 UsageError(" --dump-only: dumps the content of the specified profile files"); 92 UsageError(" to standard output (default) in a human readable form."); 93 UsageError(""); 94 UsageError(" --dump-output-to-fd=<number>: redirects --dump-only output to a file descriptor."); 95 UsageError(""); 96 UsageError(" --dump-classes-and-methods: dumps a sorted list of classes and methods that are"); 97 UsageError(" in the specified profile file to standard output (default) in a human"); 98 UsageError(" readable form. The output is valid input for --create-profile-from"); 99 UsageError(""); 100 UsageError(" --profile-file=<filename>: specify profiler output file to use for compilation."); 101 UsageError(" Can be specified multiple time, in which case the data from the different"); 102 UsageError(" profiles will be aggregated."); 103 UsageError(""); 104 UsageError(" --profile-file-fd=<number>: same as --profile-file but accepts a file descriptor."); 105 UsageError(" Cannot be used together with --profile-file."); 106 UsageError(""); 107 UsageError(" --reference-profile-file=<filename>: specify a reference profile."); 108 UsageError(" The data in this file will be compared with the data obtained by merging"); 109 UsageError(" all the files specified with --profile-file or --profile-file-fd."); 110 UsageError(" If the exit code is EXIT_COMPILE then all --profile-file will be merged into"); 111 UsageError(" --reference-profile-file. "); 112 UsageError(""); 113 UsageError(" --reference-profile-file-fd=<number>: same as --reference-profile-file but"); 114 UsageError(" accepts a file descriptor. Cannot be used together with"); 115 UsageError(" --reference-profile-file."); 116 UsageError(""); 117 UsageError(" --generate-test-profile=<filename>: generates a random profile file for testing."); 118 UsageError(" --generate-test-profile-num-dex=<number>: number of dex files that should be"); 119 UsageError(" included in the generated profile. Defaults to 20."); 120 UsageError(" --generate-test-profile-method-ratio=<number>: the percentage from the maximum"); 121 UsageError(" number of methods that should be generated. Defaults to 5."); 122 UsageError(" --generate-test-profile-class-ratio=<number>: the percentage from the maximum"); 123 UsageError(" number of classes that should be generated. Defaults to 5."); 124 UsageError(" --generate-test-profile-seed=<number>: seed for random number generator used when"); 125 UsageError(" generating random test profiles. Defaults to using NanoTime."); 126 UsageError(""); 127 UsageError(" --create-profile-from=<filename>: creates a profile from a list of classes and"); 128 UsageError(" methods."); 129 UsageError(""); 130 UsageError(" --dex-location=<string>: location string to use with corresponding"); 131 UsageError(" apk-fd to find dex files"); 132 UsageError(""); 133 UsageError(" --apk-fd=<number>: file descriptor containing an open APK to"); 134 UsageError(" search for dex files"); 135 UsageError(" --apk-=<filename>: an APK to search for dex files"); 136 UsageError(""); 137 UsageError(" --generate-boot-image-profile: Generate a boot image profile based on input"); 138 UsageError(" profiles. Requires passing in dex files to inspect properties of classes."); 139 UsageError(" --boot-image-class-threshold=<value>: specify minimum number of class occurrences"); 140 UsageError(" to include a class in the boot image profile. Default is 10."); 141 UsageError(" --boot-image-clean-class-threshold=<value>: specify minimum number of clean class"); 142 UsageError(" occurrences to include a class in the boot image profile. A clean class is a"); 143 UsageError(" class that doesn't have any static fields or native methods and is likely to"); 144 UsageError(" remain clean in the image. Default is 3."); 145 UsageError(" --boot-image-sampled-method-threshold=<value>: minimum number of profiles a"); 146 UsageError(" non-hot method needs to be in order to be hot in the output profile. The"); 147 UsageError(" default is max int."); 148 UsageError(""); 149 150 exit(EXIT_FAILURE); 151 } 152 153 // Note: make sure you update the Usage if you change these values. 154 static constexpr uint16_t kDefaultTestProfileNumDex = 20; 155 static constexpr uint16_t kDefaultTestProfileMethodRatio = 5; 156 static constexpr uint16_t kDefaultTestProfileClassRatio = 5; 157 158 // Separators used when parsing human friendly representation of profiles. 159 static const std::string kMethodSep = "->"; 160 static const std::string kMissingTypesMarker = "missing_types"; 161 static const std::string kInvalidClassDescriptor = "invalid_class"; 162 static const std::string kInvalidMethod = "invalid_method"; 163 static const std::string kClassAllMethods = "*"; 164 static constexpr char kProfileParsingInlineChacheSep = '+'; 165 static constexpr char kProfileParsingTypeSep = ','; 166 static constexpr char kProfileParsingFirstCharInSignature = '('; 167 static constexpr char kMethodFlagStringHot = 'H'; 168 static constexpr char kMethodFlagStringStartup = 'S'; 169 static constexpr char kMethodFlagStringPostStartup = 'P'; 170 171 // TODO(calin): This class has grown too much from its initial design. Split the functionality 172 // into smaller, more contained pieces. 173 class ProfMan FINAL { 174 public: 175 ProfMan() : 176 reference_profile_file_fd_(kInvalidFd), 177 dump_only_(false), 178 dump_classes_and_methods_(false), 179 generate_boot_image_profile_(false), 180 dump_output_to_fd_(kInvalidFd), 181 test_profile_num_dex_(kDefaultTestProfileNumDex), 182 test_profile_method_ratio_(kDefaultTestProfileMethodRatio), 183 test_profile_class_ratio_(kDefaultTestProfileClassRatio), 184 test_profile_seed_(NanoTime()), 185 start_ns_(NanoTime()) {} 186 187 ~ProfMan() { 188 LogCompletionTime(); 189 } 190 191 void ParseArgs(int argc, char **argv) { 192 original_argc = argc; 193 original_argv = argv; 194 195 InitLogging(argv, Runtime::Abort); 196 197 // Skip over the command name. 198 argv++; 199 argc--; 200 201 if (argc == 0) { 202 Usage("No arguments specified"); 203 } 204 205 for (int i = 0; i < argc; ++i) { 206 const StringPiece option(argv[i]); 207 const bool log_options = false; 208 if (log_options) { 209 LOG(INFO) << "profman: option[" << i << "]=" << argv[i]; 210 } 211 if (option == "--dump-only") { 212 dump_only_ = true; 213 } else if (option == "--dump-classes-and-methods") { 214 dump_classes_and_methods_ = true; 215 } else if (option.starts_with("--create-profile-from=")) { 216 create_profile_from_file_ = option.substr(strlen("--create-profile-from=")).ToString(); 217 } else if (option.starts_with("--dump-output-to-fd=")) { 218 ParseUintOption(option, "--dump-output-to-fd", &dump_output_to_fd_, Usage); 219 } else if (option == "--generate-boot-image-profile") { 220 generate_boot_image_profile_ = true; 221 } else if (option.starts_with("--boot-image-class-threshold=")) { 222 ParseUintOption(option, 223 "--boot-image-class-threshold", 224 &boot_image_options_.image_class_theshold, 225 Usage); 226 } else if (option.starts_with("--boot-image-clean-class-threshold=")) { 227 ParseUintOption(option, 228 "--boot-image-clean-class-threshold", 229 &boot_image_options_.image_class_clean_theshold, 230 Usage); 231 } else if (option.starts_with("--boot-image-sampled-method-threshold=")) { 232 ParseUintOption(option, 233 "--boot-image-sampled-method-threshold", 234 &boot_image_options_.compiled_method_threshold, 235 Usage); 236 } else if (option.starts_with("--profile-file=")) { 237 profile_files_.push_back(option.substr(strlen("--profile-file=")).ToString()); 238 } else if (option.starts_with("--profile-file-fd=")) { 239 ParseFdForCollection(option, "--profile-file-fd", &profile_files_fd_); 240 } else if (option.starts_with("--reference-profile-file=")) { 241 reference_profile_file_ = option.substr(strlen("--reference-profile-file=")).ToString(); 242 } else if (option.starts_with("--reference-profile-file-fd=")) { 243 ParseUintOption(option, "--reference-profile-file-fd", &reference_profile_file_fd_, Usage); 244 } else if (option.starts_with("--dex-location=")) { 245 dex_locations_.push_back(option.substr(strlen("--dex-location=")).ToString()); 246 } else if (option.starts_with("--apk-fd=")) { 247 ParseFdForCollection(option, "--apk-fd", &apks_fd_); 248 } else if (option.starts_with("--apk=")) { 249 apk_files_.push_back(option.substr(strlen("--apk=")).ToString()); 250 } else if (option.starts_with("--generate-test-profile=")) { 251 test_profile_ = option.substr(strlen("--generate-test-profile=")).ToString(); 252 } else if (option.starts_with("--generate-test-profile-num-dex=")) { 253 ParseUintOption(option, 254 "--generate-test-profile-num-dex", 255 &test_profile_num_dex_, 256 Usage); 257 } else if (option.starts_with("--generate-test-profile-method-ratio")) { 258 ParseUintOption(option, 259 "--generate-test-profile-method-ratio", 260 &test_profile_method_ratio_, 261 Usage); 262 } else if (option.starts_with("--generate-test-profile-class-ratio")) { 263 ParseUintOption(option, 264 "--generate-test-profile-class-ratio", 265 &test_profile_class_ratio_, 266 Usage); 267 } else if (option.starts_with("--generate-test-profile-seed=")) { 268 ParseUintOption(option, "--generate-test-profile-seed", &test_profile_seed_, Usage); 269 } else { 270 Usage("Unknown argument '%s'", option.data()); 271 } 272 } 273 274 // Validate global consistency between file/fd options. 275 if (!profile_files_.empty() && !profile_files_fd_.empty()) { 276 Usage("Profile files should not be specified with both --profile-file-fd and --profile-file"); 277 } 278 if (!reference_profile_file_.empty() && FdIsValid(reference_profile_file_fd_)) { 279 Usage("Reference profile should not be specified with both " 280 "--reference-profile-file-fd and --reference-profile-file"); 281 } 282 if (!apk_files_.empty() && !apks_fd_.empty()) { 283 Usage("APK files should not be specified with both --apk-fd and --apk"); 284 } 285 } 286 287 ProfileAssistant::ProcessingResult ProcessProfiles() { 288 // Validate that at least one profile file was passed, as well as a reference profile. 289 if (profile_files_.empty() && profile_files_fd_.empty()) { 290 Usage("No profile files specified."); 291 } 292 if (reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) { 293 Usage("No reference profile file specified."); 294 } 295 if ((!profile_files_.empty() && FdIsValid(reference_profile_file_fd_)) || 296 (!profile_files_fd_.empty() && !FdIsValid(reference_profile_file_fd_))) { 297 Usage("Options --profile-file-fd and --reference-profile-file-fd " 298 "should only be used together"); 299 } 300 ProfileAssistant::ProcessingResult result; 301 if (profile_files_.empty()) { 302 // The file doesn't need to be flushed here (ProcessProfiles will do it) 303 // so don't check the usage. 304 File file(reference_profile_file_fd_, false); 305 result = ProfileAssistant::ProcessProfiles(profile_files_fd_, reference_profile_file_fd_); 306 CloseAllFds(profile_files_fd_, "profile_files_fd_"); 307 } else { 308 result = ProfileAssistant::ProcessProfiles(profile_files_, reference_profile_file_); 309 } 310 return result; 311 } 312 313 void OpenApkFilesFromLocations(std::vector<std::unique_ptr<const DexFile>>* dex_files) { 314 bool use_apk_fd_list = !apks_fd_.empty(); 315 if (use_apk_fd_list) { 316 // Get the APKs from the collection of FDs. 317 CHECK_EQ(dex_locations_.size(), apks_fd_.size()); 318 } else if (!apk_files_.empty()) { 319 // Get the APKs from the collection of filenames. 320 CHECK_EQ(dex_locations_.size(), apk_files_.size()); 321 } else { 322 // No APKs were specified. 323 CHECK(dex_locations_.empty()); 324 return; 325 } 326 static constexpr bool kVerifyChecksum = true; 327 for (size_t i = 0; i < dex_locations_.size(); ++i) { 328 std::string error_msg; 329 std::vector<std::unique_ptr<const DexFile>> dex_files_for_location; 330 if (use_apk_fd_list) { 331 if (DexFile::OpenZip(apks_fd_[i], 332 dex_locations_[i], 333 kVerifyChecksum, 334 &error_msg, 335 &dex_files_for_location)) { 336 } else { 337 LOG(WARNING) << "OpenZip failed for '" << dex_locations_[i] << "' " << error_msg; 338 continue; 339 } 340 } else { 341 if (DexFile::Open(apk_files_[i].c_str(), 342 dex_locations_[i], 343 kVerifyChecksum, 344 &error_msg, 345 &dex_files_for_location)) { 346 } else { 347 LOG(WARNING) << "Open failed for '" << dex_locations_[i] << "' " << error_msg; 348 continue; 349 } 350 } 351 for (std::unique_ptr<const DexFile>& dex_file : dex_files_for_location) { 352 dex_files->emplace_back(std::move(dex_file)); 353 } 354 } 355 } 356 357 std::unique_ptr<const ProfileCompilationInfo> LoadProfile(const std::string& filename, int fd) { 358 if (!filename.empty()) { 359 fd = open(filename.c_str(), O_RDWR); 360 if (fd < 0) { 361 LOG(ERROR) << "Cannot open " << filename << strerror(errno); 362 return nullptr; 363 } 364 } 365 std::unique_ptr<ProfileCompilationInfo> info(new ProfileCompilationInfo); 366 if (!info->Load(fd)) { 367 LOG(ERROR) << "Cannot load profile info from fd=" << fd << "\n"; 368 return nullptr; 369 } 370 return info; 371 } 372 373 int DumpOneProfile(const std::string& banner, 374 const std::string& filename, 375 int fd, 376 const std::vector<std::unique_ptr<const DexFile>>* dex_files, 377 std::string* dump) { 378 std::unique_ptr<const ProfileCompilationInfo> info(LoadProfile(filename, fd)); 379 if (info == nullptr) { 380 LOG(ERROR) << "Cannot load profile info from filename=" << filename << " fd=" << fd; 381 return -1; 382 } 383 *dump += banner + "\n" + info->DumpInfo(dex_files) + "\n"; 384 return 0; 385 } 386 387 int DumpProfileInfo() { 388 // Validate that at least one profile file or reference was specified. 389 if (profile_files_.empty() && profile_files_fd_.empty() && 390 reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) { 391 Usage("No profile files or reference profile specified."); 392 } 393 static const char* kEmptyString = ""; 394 static const char* kOrdinaryProfile = "=== profile ==="; 395 static const char* kReferenceProfile = "=== reference profile ==="; 396 397 // Open apk/zip files and and read dex files. 398 MemMap::Init(); // for ZipArchive::OpenFromFd 399 std::vector<std::unique_ptr<const DexFile>> dex_files; 400 OpenApkFilesFromLocations(&dex_files); 401 std::string dump; 402 // Dump individual profile files. 403 if (!profile_files_fd_.empty()) { 404 for (int profile_file_fd : profile_files_fd_) { 405 int ret = DumpOneProfile(kOrdinaryProfile, 406 kEmptyString, 407 profile_file_fd, 408 &dex_files, 409 &dump); 410 if (ret != 0) { 411 return ret; 412 } 413 } 414 } 415 if (!profile_files_.empty()) { 416 for (const std::string& profile_file : profile_files_) { 417 int ret = DumpOneProfile(kOrdinaryProfile, profile_file, kInvalidFd, &dex_files, &dump); 418 if (ret != 0) { 419 return ret; 420 } 421 } 422 } 423 // Dump reference profile file. 424 if (FdIsValid(reference_profile_file_fd_)) { 425 int ret = DumpOneProfile(kReferenceProfile, 426 kEmptyString, 427 reference_profile_file_fd_, 428 &dex_files, 429 &dump); 430 if (ret != 0) { 431 return ret; 432 } 433 } 434 if (!reference_profile_file_.empty()) { 435 int ret = DumpOneProfile(kReferenceProfile, 436 reference_profile_file_, 437 kInvalidFd, 438 &dex_files, 439 &dump); 440 if (ret != 0) { 441 return ret; 442 } 443 } 444 if (!FdIsValid(dump_output_to_fd_)) { 445 std::cout << dump; 446 } else { 447 unix_file::FdFile out_fd(dump_output_to_fd_, false /*check_usage*/); 448 if (!out_fd.WriteFully(dump.c_str(), dump.length())) { 449 return -1; 450 } 451 } 452 return 0; 453 } 454 455 bool ShouldOnlyDumpProfile() { 456 return dump_only_; 457 } 458 459 bool GetClassNamesAndMethods(int fd, 460 std::vector<std::unique_ptr<const DexFile>>* dex_files, 461 std::set<std::string>* out_lines) { 462 ProfileCompilationInfo profile_info; 463 if (!profile_info.Load(fd)) { 464 LOG(ERROR) << "Cannot load profile info"; 465 return false; 466 } 467 for (const std::unique_ptr<const DexFile>& dex_file : *dex_files) { 468 std::set<dex::TypeIndex> class_types; 469 std::set<uint16_t> hot_methods; 470 std::set<uint16_t> startup_methods; 471 std::set<uint16_t> post_startup_methods; 472 std::set<uint16_t> combined_methods; 473 if (profile_info.GetClassesAndMethods(*dex_file.get(), 474 &class_types, 475 &hot_methods, 476 &startup_methods, 477 &post_startup_methods)) { 478 for (const dex::TypeIndex& type_index : class_types) { 479 const DexFile::TypeId& type_id = dex_file->GetTypeId(type_index); 480 out_lines->insert(std::string(dex_file->GetTypeDescriptor(type_id))); 481 } 482 combined_methods = hot_methods; 483 combined_methods.insert(startup_methods.begin(), startup_methods.end()); 484 combined_methods.insert(post_startup_methods.begin(), post_startup_methods.end()); 485 for (uint16_t dex_method_idx : combined_methods) { 486 const DexFile::MethodId& id = dex_file->GetMethodId(dex_method_idx); 487 std::string signature_string(dex_file->GetMethodSignature(id).ToString()); 488 std::string type_string(dex_file->GetTypeDescriptor(dex_file->GetTypeId(id.class_idx_))); 489 std::string method_name(dex_file->GetMethodName(id)); 490 std::string flags_string; 491 if (hot_methods.find(dex_method_idx) != hot_methods.end()) { 492 flags_string += kMethodFlagStringHot; 493 } 494 if (startup_methods.find(dex_method_idx) != startup_methods.end()) { 495 flags_string += kMethodFlagStringStartup; 496 } 497 if (post_startup_methods.find(dex_method_idx) != post_startup_methods.end()) { 498 flags_string += kMethodFlagStringPostStartup; 499 } 500 out_lines->insert(flags_string + 501 type_string + 502 kMethodSep + 503 method_name + 504 signature_string); 505 } 506 } 507 } 508 return true; 509 } 510 511 bool GetClassNamesAndMethods(const std::string& profile_file, 512 std::vector<std::unique_ptr<const DexFile>>* dex_files, 513 std::set<std::string>* out_lines) { 514 int fd = open(profile_file.c_str(), O_RDONLY); 515 if (!FdIsValid(fd)) { 516 LOG(ERROR) << "Cannot open " << profile_file << strerror(errno); 517 return false; 518 } 519 if (!GetClassNamesAndMethods(fd, dex_files, out_lines)) { 520 return false; 521 } 522 if (close(fd) < 0) { 523 PLOG(WARNING) << "Failed to close descriptor"; 524 } 525 return true; 526 } 527 528 int DumpClassesAndMethods() { 529 // Validate that at least one profile file or reference was specified. 530 if (profile_files_.empty() && profile_files_fd_.empty() && 531 reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) { 532 Usage("No profile files or reference profile specified."); 533 } 534 // Open apk/zip files and and read dex files. 535 MemMap::Init(); // for ZipArchive::OpenFromFd 536 // Open the dex files to get the names for classes. 537 std::vector<std::unique_ptr<const DexFile>> dex_files; 538 OpenApkFilesFromLocations(&dex_files); 539 // Build a vector of class names from individual profile files. 540 std::set<std::string> class_names; 541 if (!profile_files_fd_.empty()) { 542 for (int profile_file_fd : profile_files_fd_) { 543 if (!GetClassNamesAndMethods(profile_file_fd, &dex_files, &class_names)) { 544 return -1; 545 } 546 } 547 } 548 if (!profile_files_.empty()) { 549 for (const std::string& profile_file : profile_files_) { 550 if (!GetClassNamesAndMethods(profile_file, &dex_files, &class_names)) { 551 return -1; 552 } 553 } 554 } 555 // Concatenate class names from reference profile file. 556 if (FdIsValid(reference_profile_file_fd_)) { 557 if (!GetClassNamesAndMethods(reference_profile_file_fd_, &dex_files, &class_names)) { 558 return -1; 559 } 560 } 561 if (!reference_profile_file_.empty()) { 562 if (!GetClassNamesAndMethods(reference_profile_file_, &dex_files, &class_names)) { 563 return -1; 564 } 565 } 566 // Dump the class names. 567 std::string dump; 568 for (const std::string& class_name : class_names) { 569 dump += class_name + std::string("\n"); 570 } 571 if (!FdIsValid(dump_output_to_fd_)) { 572 std::cout << dump; 573 } else { 574 unix_file::FdFile out_fd(dump_output_to_fd_, false /*check_usage*/); 575 if (!out_fd.WriteFully(dump.c_str(), dump.length())) { 576 return -1; 577 } 578 } 579 return 0; 580 } 581 582 bool ShouldOnlyDumpClassesAndMethods() { 583 return dump_classes_and_methods_; 584 } 585 586 // Read lines from the given file, dropping comments and empty lines. Post-process each line with 587 // the given function. 588 template <typename T> 589 static T* ReadCommentedInputFromFile( 590 const char* input_filename, std::function<std::string(const char*)>* process) { 591 std::unique_ptr<std::ifstream> input_file(new std::ifstream(input_filename, std::ifstream::in)); 592 if (input_file.get() == nullptr) { 593 LOG(ERROR) << "Failed to open input file " << input_filename; 594 return nullptr; 595 } 596 std::unique_ptr<T> result( 597 ReadCommentedInputStream<T>(*input_file, process)); 598 input_file->close(); 599 return result.release(); 600 } 601 602 // Read lines from the given stream, dropping comments and empty lines. Post-process each line 603 // with the given function. 604 template <typename T> 605 static T* ReadCommentedInputStream( 606 std::istream& in_stream, 607 std::function<std::string(const char*)>* process) { 608 std::unique_ptr<T> output(new T()); 609 while (in_stream.good()) { 610 std::string dot; 611 std::getline(in_stream, dot); 612 if (android::base::StartsWith(dot, "#") || dot.empty()) { 613 continue; 614 } 615 if (process != nullptr) { 616 std::string descriptor((*process)(dot.c_str())); 617 output->insert(output->end(), descriptor); 618 } else { 619 output->insert(output->end(), dot); 620 } 621 } 622 return output.release(); 623 } 624 625 // Find class klass_descriptor in the given dex_files and store its reference 626 // in the out parameter class_ref. 627 // Return true if the definition of the class was found in any of the dex_files. 628 bool FindClass(const std::vector<std::unique_ptr<const DexFile>>& dex_files, 629 const std::string& klass_descriptor, 630 /*out*/TypeReference* class_ref) { 631 constexpr uint16_t kInvalidTypeIndex = std::numeric_limits<uint16_t>::max() - 1; 632 for (const std::unique_ptr<const DexFile>& dex_file_ptr : dex_files) { 633 const DexFile* dex_file = dex_file_ptr.get(); 634 if (klass_descriptor == kInvalidClassDescriptor) { 635 if (kInvalidTypeIndex >= dex_file->NumTypeIds()) { 636 // The dex file does not contain all possible type ids which leaves us room 637 // to add an "invalid" type id. 638 class_ref->dex_file = dex_file; 639 class_ref->type_index = dex::TypeIndex(kInvalidTypeIndex); 640 return true; 641 } else { 642 // The dex file contains all possible type ids. We don't have any free type id 643 // that we can use as invalid. 644 continue; 645 } 646 } 647 648 const DexFile::TypeId* type_id = dex_file->FindTypeId(klass_descriptor.c_str()); 649 if (type_id == nullptr) { 650 continue; 651 } 652 dex::TypeIndex type_index = dex_file->GetIndexForTypeId(*type_id); 653 if (dex_file->FindClassDef(type_index) == nullptr) { 654 // Class is only referenced in the current dex file but not defined in it. 655 continue; 656 } 657 class_ref->dex_file = dex_file; 658 class_ref->type_index = type_index; 659 return true; 660 } 661 return false; 662 } 663 664 // Find the method specified by method_spec in the class class_ref. 665 uint32_t FindMethodIndex(const TypeReference& class_ref, 666 const std::string& method_spec) { 667 const DexFile* dex_file = class_ref.dex_file; 668 if (method_spec == kInvalidMethod) { 669 constexpr uint16_t kInvalidMethodIndex = std::numeric_limits<uint16_t>::max() - 1; 670 return kInvalidMethodIndex >= dex_file->NumMethodIds() 671 ? kInvalidMethodIndex 672 : DexFile::kDexNoIndex; 673 } 674 675 std::vector<std::string> name_and_signature; 676 Split(method_spec, kProfileParsingFirstCharInSignature, &name_and_signature); 677 if (name_and_signature.size() != 2) { 678 LOG(ERROR) << "Invalid method name and signature " << method_spec; 679 return DexFile::kDexNoIndex; 680 } 681 682 const std::string& name = name_and_signature[0]; 683 const std::string& signature = kProfileParsingFirstCharInSignature + name_and_signature[1]; 684 685 const DexFile::StringId* name_id = dex_file->FindStringId(name.c_str()); 686 if (name_id == nullptr) { 687 LOG(WARNING) << "Could not find name: " << name; 688 return DexFile::kDexNoIndex; 689 } 690 dex::TypeIndex return_type_idx; 691 std::vector<dex::TypeIndex> param_type_idxs; 692 if (!dex_file->CreateTypeList(signature, &return_type_idx, ¶m_type_idxs)) { 693 LOG(WARNING) << "Could not create type list" << signature; 694 return DexFile::kDexNoIndex; 695 } 696 const DexFile::ProtoId* proto_id = dex_file->FindProtoId(return_type_idx, param_type_idxs); 697 if (proto_id == nullptr) { 698 LOG(WARNING) << "Could not find proto_id: " << name; 699 return DexFile::kDexNoIndex; 700 } 701 const DexFile::MethodId* method_id = dex_file->FindMethodId( 702 dex_file->GetTypeId(class_ref.type_index), *name_id, *proto_id); 703 if (method_id == nullptr) { 704 LOG(WARNING) << "Could not find method_id: " << name; 705 return DexFile::kDexNoIndex; 706 } 707 708 return dex_file->GetIndexForMethodId(*method_id); 709 } 710 711 // Given a method, return true if the method has a single INVOKE_VIRTUAL in its byte code. 712 // Upon success it returns true and stores the method index and the invoke dex pc 713 // in the output parameters. 714 // The format of the method spec is "inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,LSubC;". 715 // 716 // TODO(calin): support INVOKE_INTERFACE and the range variants. 717 bool HasSingleInvoke(const TypeReference& class_ref, 718 uint16_t method_index, 719 /*out*/uint32_t* dex_pc) { 720 const DexFile* dex_file = class_ref.dex_file; 721 uint32_t offset = dex_file->FindCodeItemOffset( 722 *dex_file->FindClassDef(class_ref.type_index), 723 method_index); 724 const DexFile::CodeItem* code_item = dex_file->GetCodeItem(offset); 725 726 bool found_invoke = false; 727 for (CodeItemIterator it(*code_item); !it.Done(); it.Advance()) { 728 if (it.CurrentInstruction().Opcode() == Instruction::INVOKE_VIRTUAL) { 729 if (found_invoke) { 730 LOG(ERROR) << "Multiple invoke INVOKE_VIRTUAL found: " 731 << dex_file->PrettyMethod(method_index); 732 return false; 733 } 734 found_invoke = true; 735 *dex_pc = it.CurrentDexPc(); 736 } 737 } 738 if (!found_invoke) { 739 LOG(ERROR) << "Could not find any INVOKE_VIRTUAL: " << dex_file->PrettyMethod(method_index); 740 } 741 return found_invoke; 742 } 743 744 // Process a line defining a class or a method and its inline caches. 745 // Upon success return true and add the class or the method info to profile. 746 // The possible line formats are: 747 // "LJustTheCass;". 748 // "LTestInline;->inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,LSubC;". 749 // "LTestInline;->inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,invalid_class". 750 // "LTestInline;->inlineMissingTypes(LSuper;)I+missing_types". 751 // "LTestInline;->inlineNoInlineCaches(LSuper;)I". 752 // "LTestInline;->*". 753 // "invalid_class". 754 // "LTestInline;->invalid_method". 755 // The method and classes are searched only in the given dex files. 756 bool ProcessLine(const std::vector<std::unique_ptr<const DexFile>>& dex_files, 757 const std::string& line, 758 /*out*/ProfileCompilationInfo* profile) { 759 std::string klass; 760 std::string method_str; 761 bool is_hot = false; 762 bool is_startup = false; 763 bool is_post_startup = false; 764 const size_t method_sep_index = line.find(kMethodSep, 0); 765 if (method_sep_index == std::string::npos) { 766 klass = line.substr(0); 767 } else { 768 // The method prefix flags are only valid for method strings. 769 size_t start_index = 0; 770 while (start_index < line.size() && line[start_index] != 'L') { 771 const char c = line[start_index]; 772 if (c == kMethodFlagStringHot) { 773 is_hot = true; 774 } else if (c == kMethodFlagStringStartup) { 775 is_startup = true; 776 } else if (c == kMethodFlagStringPostStartup) { 777 is_post_startup = true; 778 } else { 779 LOG(WARNING) << "Invalid flag " << c; 780 return false; 781 } 782 ++start_index; 783 } 784 klass = line.substr(start_index, method_sep_index - start_index); 785 method_str = line.substr(method_sep_index + kMethodSep.size()); 786 } 787 788 TypeReference class_ref; 789 if (!FindClass(dex_files, klass, &class_ref)) { 790 LOG(WARNING) << "Could not find class: " << klass; 791 return false; 792 } 793 794 if (method_str.empty() || method_str == kClassAllMethods) { 795 // Start by adding the class. 796 std::set<DexCacheResolvedClasses> resolved_class_set; 797 const DexFile* dex_file = class_ref.dex_file; 798 const auto& dex_resolved_classes = resolved_class_set.emplace( 799 dex_file->GetLocation(), 800 dex_file->GetBaseLocation(), 801 dex_file->GetLocationChecksum(), 802 dex_file->NumMethodIds()); 803 dex_resolved_classes.first->AddClass(class_ref.type_index); 804 std::vector<ProfileMethodInfo> methods; 805 if (method_str == kClassAllMethods) { 806 // Add all of the methods. 807 const DexFile::ClassDef* class_def = dex_file->FindClassDef(class_ref.type_index); 808 const uint8_t* class_data = dex_file->GetClassData(*class_def); 809 if (class_data != nullptr) { 810 ClassDataItemIterator it(*dex_file, class_data); 811 it.SkipAllFields(); 812 while (it.HasNextDirectMethod() || it.HasNextVirtualMethod()) { 813 if (it.GetMethodCodeItemOffset() != 0) { 814 // Add all of the methods that have code to the profile. 815 const uint32_t method_idx = it.GetMemberIndex(); 816 methods.push_back(ProfileMethodInfo(MethodReference(dex_file, method_idx))); 817 } 818 it.Next(); 819 } 820 } 821 } 822 // TODO: Check return values? 823 profile->AddMethods(methods); 824 profile->AddClasses(resolved_class_set); 825 return true; 826 } 827 828 // Process the method. 829 std::string method_spec; 830 std::vector<std::string> inline_cache_elems; 831 832 // If none of the flags are set, default to hot. 833 is_hot = is_hot || (!is_hot && !is_startup && !is_post_startup); 834 835 std::vector<std::string> method_elems; 836 bool is_missing_types = false; 837 Split(method_str, kProfileParsingInlineChacheSep, &method_elems); 838 if (method_elems.size() == 2) { 839 method_spec = method_elems[0]; 840 is_missing_types = method_elems[1] == kMissingTypesMarker; 841 if (!is_missing_types) { 842 Split(method_elems[1], kProfileParsingTypeSep, &inline_cache_elems); 843 } 844 } else if (method_elems.size() == 1) { 845 method_spec = method_elems[0]; 846 } else { 847 LOG(ERROR) << "Invalid method line: " << line; 848 return false; 849 } 850 851 const uint32_t method_index = FindMethodIndex(class_ref, method_spec); 852 if (method_index == DexFile::kDexNoIndex) { 853 return false; 854 } 855 856 std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches; 857 if (is_missing_types || !inline_cache_elems.empty()) { 858 uint32_t dex_pc; 859 if (!HasSingleInvoke(class_ref, method_index, &dex_pc)) { 860 return false; 861 } 862 std::vector<TypeReference> classes(inline_cache_elems.size()); 863 size_t class_it = 0; 864 for (const std::string& ic_class : inline_cache_elems) { 865 if (!FindClass(dex_files, ic_class, &(classes[class_it++]))) { 866 LOG(ERROR) << "Could not find class: " << ic_class; 867 return false; 868 } 869 } 870 inline_caches.emplace_back(dex_pc, is_missing_types, classes); 871 } 872 MethodReference ref(class_ref.dex_file, method_index); 873 if (is_hot) { 874 profile->AddMethod(ProfileMethodInfo(ref, inline_caches)); 875 } 876 uint32_t flags = 0; 877 using Hotness = ProfileCompilationInfo::MethodHotness; 878 if (is_startup) { 879 flags |= Hotness::kFlagStartup; 880 } 881 if (is_post_startup) { 882 flags |= Hotness::kFlagPostStartup; 883 } 884 if (flags != 0) { 885 if (!profile->AddMethodIndex(static_cast<Hotness::Flag>(flags), ref)) { 886 return false; 887 } 888 DCHECK(profile->GetMethodHotness(ref).IsInProfile()); 889 } 890 return true; 891 } 892 893 int OpenReferenceProfile() const { 894 int fd = reference_profile_file_fd_; 895 if (!FdIsValid(fd)) { 896 CHECK(!reference_profile_file_.empty()); 897 fd = open(reference_profile_file_.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0644); 898 if (fd < 0) { 899 LOG(ERROR) << "Cannot open " << reference_profile_file_ << strerror(errno); 900 return kInvalidFd; 901 } 902 } 903 return fd; 904 } 905 906 // Creates a profile from a human friendly textual representation. 907 // The expected input format is: 908 // # Classes 909 // Ljava/lang/Comparable; 910 // Ljava/lang/Math; 911 // # Methods with inline caches 912 // LTestInline;->inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,LSubC; 913 // LTestInline;->noInlineCache(LSuper;)I 914 int CreateProfile() { 915 // Validate parameters for this command. 916 if (apk_files_.empty() && apks_fd_.empty()) { 917 Usage("APK files must be specified"); 918 } 919 if (dex_locations_.empty()) { 920 Usage("DEX locations must be specified"); 921 } 922 if (reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) { 923 Usage("Reference profile must be specified with --reference-profile-file or " 924 "--reference-profile-file-fd"); 925 } 926 if (!profile_files_.empty() || !profile_files_fd_.empty()) { 927 Usage("Profile must be specified with --reference-profile-file or " 928 "--reference-profile-file-fd"); 929 } 930 // for ZipArchive::OpenFromFd 931 MemMap::Init(); 932 // Open the profile output file if needed. 933 int fd = OpenReferenceProfile(); 934 if (!FdIsValid(fd)) { 935 return -1; 936 } 937 // Read the user-specified list of classes and methods. 938 std::unique_ptr<std::unordered_set<std::string>> 939 user_lines(ReadCommentedInputFromFile<std::unordered_set<std::string>>( 940 create_profile_from_file_.c_str(), nullptr)); // No post-processing. 941 942 // Open the dex files to look up classes and methods. 943 std::vector<std::unique_ptr<const DexFile>> dex_files; 944 OpenApkFilesFromLocations(&dex_files); 945 946 // Process the lines one by one and add the successful ones to the profile. 947 ProfileCompilationInfo info; 948 949 for (const auto& line : *user_lines) { 950 ProcessLine(dex_files, line, &info); 951 } 952 953 // Write the profile file. 954 CHECK(info.Save(fd)); 955 if (close(fd) < 0) { 956 PLOG(WARNING) << "Failed to close descriptor"; 957 } 958 return 0; 959 } 960 961 bool ShouldCreateBootProfile() const { 962 return generate_boot_image_profile_; 963 } 964 965 int CreateBootProfile() { 966 // Initialize memmap since it's required to open dex files. 967 MemMap::Init(); 968 // Open the profile output file. 969 const int reference_fd = OpenReferenceProfile(); 970 if (!FdIsValid(reference_fd)) { 971 PLOG(ERROR) << "Error opening reference profile"; 972 return -1; 973 } 974 // Open the dex files. 975 std::vector<std::unique_ptr<const DexFile>> dex_files; 976 OpenApkFilesFromLocations(&dex_files); 977 if (dex_files.empty()) { 978 PLOG(ERROR) << "Expected dex files for creating boot profile"; 979 return -2; 980 } 981 // Open the input profiles. 982 std::vector<std::unique_ptr<const ProfileCompilationInfo>> profiles; 983 if (!profile_files_fd_.empty()) { 984 for (int profile_file_fd : profile_files_fd_) { 985 std::unique_ptr<const ProfileCompilationInfo> profile(LoadProfile("", profile_file_fd)); 986 if (profile == nullptr) { 987 return -3; 988 } 989 profiles.emplace_back(std::move(profile)); 990 } 991 } 992 if (!profile_files_.empty()) { 993 for (const std::string& profile_file : profile_files_) { 994 std::unique_ptr<const ProfileCompilationInfo> profile(LoadProfile(profile_file, kInvalidFd)); 995 if (profile == nullptr) { 996 return -4; 997 } 998 profiles.emplace_back(std::move(profile)); 999 } 1000 } 1001 ProfileCompilationInfo out_profile; 1002 GenerateBootImageProfile(dex_files, 1003 profiles, 1004 boot_image_options_, 1005 VLOG_IS_ON(profiler), 1006 &out_profile); 1007 out_profile.Save(reference_fd); 1008 close(reference_fd); 1009 return 0; 1010 } 1011 1012 bool ShouldCreateProfile() { 1013 return !create_profile_from_file_.empty(); 1014 } 1015 1016 int GenerateTestProfile() { 1017 // Validate parameters for this command. 1018 if (test_profile_method_ratio_ > 100) { 1019 Usage("Invalid ratio for --generate-test-profile-method-ratio"); 1020 } 1021 if (test_profile_class_ratio_ > 100) { 1022 Usage("Invalid ratio for --generate-test-profile-class-ratio"); 1023 } 1024 // If given APK files or DEX locations, check that they're ok. 1025 if (!apk_files_.empty() || !apks_fd_.empty() || !dex_locations_.empty()) { 1026 if (apk_files_.empty() && apks_fd_.empty()) { 1027 Usage("APK files must be specified when passing DEX locations to --generate-test-profile"); 1028 } 1029 if (dex_locations_.empty()) { 1030 Usage("DEX locations must be specified when passing APK files to --generate-test-profile"); 1031 } 1032 } 1033 // ShouldGenerateTestProfile confirms !test_profile_.empty(). 1034 int profile_test_fd = open(test_profile_.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0644); 1035 if (profile_test_fd < 0) { 1036 LOG(ERROR) << "Cannot open " << test_profile_ << strerror(errno); 1037 return -1; 1038 } 1039 bool result; 1040 if (apk_files_.empty() && apks_fd_.empty() && dex_locations_.empty()) { 1041 result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd, 1042 test_profile_num_dex_, 1043 test_profile_method_ratio_, 1044 test_profile_class_ratio_, 1045 test_profile_seed_); 1046 } else { 1047 // Initialize MemMap for ZipArchive::OpenFromFd. 1048 MemMap::Init(); 1049 // Open the dex files to look up classes and methods. 1050 std::vector<std::unique_ptr<const DexFile>> dex_files; 1051 OpenApkFilesFromLocations(&dex_files); 1052 // Create a random profile file based on the set of dex files. 1053 result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd, 1054 dex_files, 1055 test_profile_seed_); 1056 } 1057 close(profile_test_fd); // ignore close result. 1058 return result ? 0 : -1; 1059 } 1060 1061 bool ShouldGenerateTestProfile() { 1062 return !test_profile_.empty(); 1063 } 1064 1065 private: 1066 static void ParseFdForCollection(const StringPiece& option, 1067 const char* arg_name, 1068 std::vector<int>* fds) { 1069 int fd; 1070 ParseUintOption(option, arg_name, &fd, Usage); 1071 fds->push_back(fd); 1072 } 1073 1074 static void CloseAllFds(const std::vector<int>& fds, const char* descriptor) { 1075 for (size_t i = 0; i < fds.size(); i++) { 1076 if (close(fds[i]) < 0) { 1077 PLOG(WARNING) << "Failed to close descriptor for " << descriptor << " at index " << i; 1078 } 1079 } 1080 } 1081 1082 void LogCompletionTime() { 1083 static constexpr uint64_t kLogThresholdTime = MsToNs(100); // 100ms 1084 uint64_t time_taken = NanoTime() - start_ns_; 1085 if (time_taken > kLogThresholdTime) { 1086 LOG(WARNING) << "profman took " << PrettyDuration(time_taken); 1087 } 1088 } 1089 1090 std::vector<std::string> profile_files_; 1091 std::vector<int> profile_files_fd_; 1092 std::vector<std::string> dex_locations_; 1093 std::vector<std::string> apk_files_; 1094 std::vector<int> apks_fd_; 1095 std::string reference_profile_file_; 1096 int reference_profile_file_fd_; 1097 bool dump_only_; 1098 bool dump_classes_and_methods_; 1099 bool generate_boot_image_profile_; 1100 int dump_output_to_fd_; 1101 BootImageOptions boot_image_options_; 1102 std::string test_profile_; 1103 std::string create_profile_from_file_; 1104 uint16_t test_profile_num_dex_; 1105 uint16_t test_profile_method_ratio_; 1106 uint16_t test_profile_class_ratio_; 1107 uint32_t test_profile_seed_; 1108 uint64_t start_ns_; 1109 }; 1110 1111 // See ProfileAssistant::ProcessingResult for return codes. 1112 static int profman(int argc, char** argv) { 1113 ProfMan profman; 1114 1115 // Parse arguments. Argument mistakes will lead to exit(EXIT_FAILURE) in UsageError. 1116 profman.ParseArgs(argc, argv); 1117 1118 if (profman.ShouldGenerateTestProfile()) { 1119 return profman.GenerateTestProfile(); 1120 } 1121 if (profman.ShouldOnlyDumpProfile()) { 1122 return profman.DumpProfileInfo(); 1123 } 1124 if (profman.ShouldOnlyDumpClassesAndMethods()) { 1125 return profman.DumpClassesAndMethods(); 1126 } 1127 if (profman.ShouldCreateProfile()) { 1128 return profman.CreateProfile(); 1129 } 1130 1131 if (profman.ShouldCreateBootProfile()) { 1132 return profman.CreateBootProfile(); 1133 } 1134 // Process profile information and assess if we need to do a profile guided compilation. 1135 // This operation involves I/O. 1136 return profman.ProcessProfiles(); 1137 } 1138 1139 } // namespace art 1140 1141 int main(int argc, char **argv) { 1142 return art::profman(argc, argv); 1143 } 1144 1145