1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <dirent.h> 18 #include <err.h> 19 #include <limits.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <sys/stat.h> 23 #include <sys/types.h> 24 #include <unistd.h> 25 26 #if defined(__linux__) 27 #include <sched.h> 28 #endif 29 30 #include <atomic> 31 #include <chrono> 32 #include <functional> 33 #include <iostream> 34 #include <map> 35 #include <memory> 36 #include <set> 37 #include <sstream> 38 #include <string> 39 #include <string_view> 40 #include <thread> 41 #include <unordered_map> 42 #include <vector> 43 44 #include <llvm/ADT/StringRef.h> 45 46 #include <android-base/file.h> 47 #include <android-base/macros.h> 48 #include <android-base/parseint.h> 49 #include <android-base/strings.h> 50 51 #include "Arch.h" 52 #include "DeclarationDatabase.h" 53 #include "Driver.h" 54 #include "Preprocessor.h" 55 #include "SymbolDatabase.h" 56 #include "Utils.h" 57 #include "VFS.h" 58 59 #include "versioner.h" 60 61 using namespace std::chrono_literals; 62 using namespace std::string_literals; 63 64 bool strict; 65 bool verbose; 66 bool add_include; 67 68 static int getCpuCount(); 69 static int max_thread_count = getCpuCount(); 70 71 static int getCpuCount() { 72 #if defined(__linux__) 73 cpu_set_t cpu_set; 74 int rc = sched_getaffinity(getpid(), sizeof(cpu_set), &cpu_set); 75 if (rc != 0) { 76 err(1, "sched_getaffinity failed"); 77 } 78 return CPU_COUNT(&cpu_set); 79 #else 80 return 1; 81 #endif 82 } 83 84 namespace { 85 struct HeaderLocationInformation { 86 std::string header_path; 87 std::string dependency_dir; 88 // Absolute paths to ignore all children -- including subdirectories -- of. 89 std::unordered_set<std::string> ignored_directories; 90 }; 91 } 92 93 static bool is_dir(const std::string& path) { 94 struct stat st; 95 return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); 96 } 97 98 static CompilationRequirements collectRequirements(const Arch& arch, 99 const HeaderLocationInformation& location) { 100 std::vector<std::string> headers = 101 collectHeaders(location.header_path, location.ignored_directories); 102 std::vector<std::string> dependencies; 103 104 if (is_dir(location.header_path)) { 105 dependencies.emplace_back(location.header_path); 106 } 107 108 if (!location.dependency_dir.empty()) { 109 auto collect_children = [&dependencies](const std::string& dir_path) { 110 DIR* dir = opendir(dir_path.c_str()); 111 if (!dir) { 112 err(1, "failed to open dependency directory '%s'", dir_path.c_str()); 113 } 114 115 struct dirent* dent; 116 while ((dent = readdir(dir))) { 117 if (dent->d_name[0] == '.') { 118 continue; 119 } 120 121 // TODO: Resolve symlinks. 122 std::string dependency = dir_path + "/" + dent->d_name; 123 124 struct stat st; 125 if (stat(dependency.c_str(), &st) != 0) { 126 err(1, "failed to stat dependency '%s'", dependency.c_str()); 127 } 128 129 if (!S_ISDIR(st.st_mode)) { 130 errx(1, "'%s' is not a directory", dependency.c_str()); 131 } 132 133 dependencies.push_back(dependency); 134 } 135 136 closedir(dir); 137 }; 138 139 collect_children(location.dependency_dir + "/common"); 140 collect_children(location.dependency_dir + "/" + to_string(arch)); 141 } 142 143 auto new_end = std::remove_if(headers.begin(), headers.end(), [&arch](llvm::StringRef header) { 144 for (const auto& it : header_blacklist) { 145 if (it.second.find(arch) == it.second.end()) { 146 continue; 147 } 148 149 if (header.endswith("/" + it.first)) { 150 return true; 151 } 152 } 153 return false; 154 }); 155 156 headers.erase(new_end, headers.end()); 157 158 CompilationRequirements result = { .headers = headers, .dependencies = dependencies }; 159 return result; 160 } 161 162 static std::set<CompilationType> generateCompilationTypes(const std::set<Arch> selected_architectures, 163 const std::set<int>& selected_levels) { 164 std::set<CompilationType> result; 165 for (const auto& arch : selected_architectures) { 166 int min_api = arch_min_api[arch]; 167 for (int api_level : selected_levels) { 168 if (api_level < min_api) { 169 continue; 170 } 171 172 for (int file_offset_bits : { 32, 64 }) { 173 for (bool cpp : { true, false }) { 174 CompilationType type = { 175 .arch = arch, .cpp = cpp, .api_level = api_level, .file_offset_bits = file_offset_bits 176 }; 177 result.insert(type); 178 } 179 } 180 } 181 } 182 return result; 183 } 184 185 static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<CompilationType>& types, 186 const HeaderLocationInformation& location) { 187 if (types.empty()) { 188 errx(1, "compileHeaders received no CompilationTypes"); 189 } 190 191 auto vfs = createCommonVFS(location.header_path, location.dependency_dir, add_include); 192 193 size_t thread_count = max_thread_count; 194 std::vector<std::thread> threads; 195 196 std::map<CompilationType, HeaderDatabase> header_databases; 197 std::unordered_map<Arch, CompilationRequirements> requirements; 198 199 auto result = std::make_unique<HeaderDatabase>(); 200 for (const auto& type : types) { 201 if (requirements.count(type.arch) == 0) { 202 requirements[type.arch] = collectRequirements(type.arch, location); 203 } 204 } 205 206 initializeTargetCC1FlagCache(vfs, types, requirements); 207 208 std::vector<std::pair<CompilationType, const std::string&>> jobs; 209 std::atomic<size_t> job_index(0); 210 for (CompilationType type : types) { 211 CompilationRequirements& req = requirements[type.arch]; 212 for (const std::string& header : req.headers) { 213 jobs.emplace_back(type, header); 214 } 215 } 216 217 // Dup an empty file to stdin, so that we can use `clang -include a.h -` instead of `clang a.h`, 218 // since some warnings don't get generated in files that are compiled directly. 219 FILE* empty_file = tmpfile(); 220 if (!empty_file) { 221 err(1, "failed to create temporary file"); 222 } 223 224 int empty_file_fd = fileno(empty_file); 225 if (empty_file_fd == -1) { 226 errx(1, "fileno failed on tmpfile"); 227 } 228 229 dup2(empty_file_fd, STDIN_FILENO); 230 fclose(empty_file); 231 232 thread_count = std::min(thread_count, jobs.size()); 233 234 if (thread_count == 1) { 235 for (const auto& job : jobs) { 236 compileHeader(vfs, result.get(), job.first, job.second); 237 } 238 } else { 239 // Spawn threads. 240 for (size_t i = 0; i < thread_count; ++i) { 241 threads.emplace_back([&jobs, &job_index, &result, vfs]() { 242 while (true) { 243 size_t idx = job_index++; 244 if (idx >= jobs.size()) { 245 return; 246 } 247 248 const auto& job = jobs[idx]; 249 compileHeader(vfs, result.get(), job.first, job.second); 250 } 251 }); 252 } 253 254 // Reap them. 255 for (auto& thread : threads) { 256 thread.join(); 257 } 258 threads.clear(); 259 } 260 261 return result; 262 } 263 264 static std::set<CompilationType> getCompilationTypes(const Declaration* decl) { 265 std::set<CompilationType> result; 266 for (const auto& it : decl->availability) { 267 result.insert(it.first); 268 } 269 return result; 270 } 271 272 template<typename T> 273 static std::vector<T> Intersection(const std::set<T>& a, const std::set<T>& b) { 274 std::vector<T> intersection; 275 std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(intersection)); 276 return intersection; 277 } 278 279 // Perform a sanity check on a symbol's declarations, enforcing the following invariants: 280 // 1. At most one inline definition of the function exists. 281 // 2. All of the availability declarations for a symbol are compatible. 282 // If a function is declared as an inline before a certain version, the inline definition 283 // should have no version tag. 284 // 3. Each availability type must only be present globally or on a per-arch basis. 285 // (e.g. __INTRODUCED_IN_ARM(9) __INTRODUCED_IN_X86(10) __DEPRECATED_IN(11) is fine, 286 // but not __INTRODUCED_IN(9) __INTRODUCED_IN_X86(10)) 287 static bool checkSymbol(const Symbol& symbol) { 288 std::string cwd = getWorkingDir() + "/"; 289 290 std::unordered_map<const Declaration*, std::set<CompilationType>> inline_definitions; 291 for (const auto& decl_it : symbol.declarations) { 292 const Declaration* decl = &decl_it.second; 293 if (decl->is_definition) { 294 std::set<CompilationType> compilation_types = getCompilationTypes(decl); 295 for (const auto& inline_def_it : inline_definitions) { 296 auto intersection = Intersection(compilation_types, inline_def_it.second); 297 if (!intersection.empty()) { 298 fprintf(stderr, "versioner: conflicting inline definitions for symbol %s:\n", 299 symbol.name.c_str()); 300 fprintf(stderr, " declarations visible in: %s\n", Join(intersection, ", ").c_str()); 301 decl->dump(cwd, stderr, 4); 302 inline_def_it.first->dump(cwd, stderr, 4); 303 return false; 304 } 305 } 306 307 inline_definitions[decl] = std::move(compilation_types); 308 } 309 310 DeclarationAvailability availability; 311 if (!decl->calculateAvailability(&availability)) { 312 fprintf(stderr, "versioner: failed to calculate availability for declaration:\n"); 313 decl->dump(cwd, stderr, 2); 314 return false; 315 } 316 317 if (decl->is_definition && !availability.empty()) { 318 fprintf(stderr, "versioner: inline definition has non-empty versioning information:\n"); 319 decl->dump(cwd, stderr, 2); 320 return false; 321 } 322 } 323 324 DeclarationAvailability availability; 325 if (!symbol.calculateAvailability(&availability)) { 326 fprintf(stderr, "versioner: inconsistent availability for symbol '%s'\n", symbol.name.c_str()); 327 symbol.dump(cwd); 328 return false; 329 } 330 331 // TODO: Check invariant #3. 332 return true; 333 } 334 335 static bool sanityCheck(const HeaderDatabase* database) { 336 bool error = false; 337 std::string cwd = getWorkingDir() + "/"; 338 339 for (const auto& symbol_it : database->symbols) { 340 if (!checkSymbol(symbol_it.second)) { 341 error = true; 342 } 343 } 344 return !error; 345 } 346 347 // Check that our symbol availability declarations match the actual NDK 348 // platform symbol availability. 349 static bool checkVersions(const std::set<CompilationType>& types, 350 const HeaderDatabase* header_database, 351 const NdkSymbolDatabase& symbol_database) { 352 std::string cwd = getWorkingDir() + "/"; 353 bool failed = false; 354 355 std::map<Arch, std::set<CompilationType>> arch_types; 356 for (const CompilationType& type : types) { 357 arch_types[type.arch].insert(type); 358 } 359 360 std::set<std::string> completely_unavailable; 361 std::map<std::string, std::set<CompilationType>> missing_availability; 362 std::map<std::string, std::set<CompilationType>> extra_availability; 363 364 for (const auto& symbol_it : header_database->symbols) { 365 const auto& symbol_name = symbol_it.first; 366 DeclarationAvailability symbol_availability; 367 368 if (!symbol_it.second.calculateAvailability(&symbol_availability)) { 369 errx(1, "failed to calculate symbol availability"); 370 } 371 372 const auto platform_availability_it = symbol_database.find(symbol_name); 373 if (platform_availability_it == symbol_database.end()) { 374 completely_unavailable.insert(symbol_name); 375 continue; 376 } 377 378 const auto& platform_availability = platform_availability_it->second; 379 380 for (const CompilationType& type : types) { 381 bool should_be_available = true; 382 const auto& global_availability = symbol_availability.global_availability; 383 const auto& arch_availability = symbol_availability.arch_availability[type.arch]; 384 if (global_availability.introduced != 0 && global_availability.introduced > type.api_level) { 385 should_be_available = false; 386 } 387 388 if (arch_availability.introduced != 0 && arch_availability.introduced > type.api_level) { 389 should_be_available = false; 390 } 391 392 if (global_availability.obsoleted != 0 && global_availability.obsoleted <= type.api_level) { 393 should_be_available = false; 394 } 395 396 if (arch_availability.obsoleted != 0 && arch_availability.obsoleted <= type.api_level) { 397 should_be_available = false; 398 } 399 400 // The function declaration might be (validly) missing for the given CompilationType. 401 if (!symbol_it.second.hasDeclaration(type)) { 402 should_be_available = false; 403 } 404 405 bool is_available = platform_availability.count(type); 406 407 if (should_be_available != is_available) { 408 if (is_available) { 409 extra_availability[symbol_name].insert(type); 410 } else { 411 missing_availability[symbol_name].insert(type); 412 } 413 } 414 } 415 } 416 417 for (const auto& it : symbol_database) { 418 const std::string& symbol_name = it.first; 419 420 bool symbol_error = false; 421 if (auto missing_it = missing_availability.find(symbol_name); 422 missing_it != missing_availability.end()) { 423 printf("%s: declaration marked available but symbol missing in [%s]\n", symbol_name.c_str(), 424 Join(missing_it->second, ", ").c_str()); 425 symbol_error = true; 426 failed = true; 427 } 428 429 if (strict) { 430 if (auto extra_it = extra_availability.find(symbol_name); 431 extra_it != extra_availability.end()) { 432 printf("%s: declaration marked unavailable but symbol available in [%s]\n", 433 symbol_name.c_str(), Join(extra_it->second, ", ").c_str()); 434 symbol_error = true; 435 failed = true; 436 } 437 } 438 439 if (symbol_error) { 440 if (auto symbol_it = header_database->symbols.find(symbol_name); 441 symbol_it != header_database->symbols.end()) { 442 symbol_it->second.dump(cwd); 443 } else { 444 errx(1, "failed to find symbol in header database"); 445 } 446 } 447 } 448 449 // TODO: Verify that function/variable declarations are actually function/variable symbols. 450 return !failed; 451 } 452 453 static void usage(bool help = false) { 454 fprintf(stderr, "Usage: versioner [OPTION]... [HEADER_PATH] [DEPS_PATH]\n"); 455 if (!help) { 456 printf("Try 'versioner -h' for more information.\n"); 457 exit(1); 458 } else { 459 fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n"); 460 fprintf(stderr, "Autodetects paths if HEADER_PATH and DEPS_PATH are not specified\n"); 461 fprintf(stderr, "\n"); 462 fprintf(stderr, "Target specification (defaults to all):\n"); 463 fprintf(stderr, " -a API_LEVEL\tbuild with specified API level (can be repeated)\n"); 464 fprintf(stderr, " \t\tdefaults to %s\n", Join(default_levels).c_str()); 465 fprintf(stderr, " -r ARCH\tbuild with specified architecture (can be repeated)\n"); 466 fprintf(stderr, " \t\tvalid architectures are %s\n", Join(supported_archs).c_str()); 467 fprintf(stderr, "\n"); 468 fprintf(stderr, "Validation:\n"); 469 fprintf(stderr, " -p PATH\tcompare against NDK platform at PATH\n"); 470 fprintf(stderr, " -s\t\tenable strict warnings\n"); 471 fprintf(stderr, "\n"); 472 fprintf(stderr, "Preprocessing:\n"); 473 fprintf(stderr, " -o PATH\tpreprocess header files and emit them at PATH\n"); 474 fprintf(stderr, " -f\t\tpreprocess header files even if validation fails\n"); 475 fprintf(stderr, "\n"); 476 fprintf(stderr, "Miscellaneous:\n"); 477 fprintf(stderr, " -F\t\tdo not ignore FORTIFY headers by default\n"); 478 fprintf(stderr, " -d\t\tdump function availability\n"); 479 fprintf(stderr, " -j THREADS\tmaximum number of threads to use\n"); 480 fprintf(stderr, " -v\t\tenable verbose logging\n"); 481 fprintf(stderr, " -h\t\tdisplay this message\n"); 482 exit(0); 483 } 484 } 485 486 // versioner uses a prebuilt version of clang, which is not up-to-date wrt/ 487 // container annotations. So disable container overflow checking. b/37775238 488 extern "C" const char* __asan_default_options() { 489 return "detect_container_overflow=0"; 490 } 491 492 int main(int argc, char** argv) { 493 std::string cwd = getWorkingDir() + "/"; 494 std::string platform_dir; 495 std::set<Arch> selected_architectures; 496 std::set<int> selected_levels; 497 std::string preprocessor_output_path; 498 bool force = false; 499 bool dump = false; 500 bool ignore_fortify_headers = true; 501 502 int c; 503 while ((c = getopt(argc, argv, "a:r:p:so:fdj:vhFi")) != -1) { 504 switch (c) { 505 case 'a': { 506 char* end; 507 int api_level = strtol(optarg, &end, 10); 508 if (end == optarg || strlen(end) > 0) { 509 usage(); 510 } 511 512 selected_levels.insert(api_level); 513 break; 514 } 515 516 case 'r': { 517 std::optional<Arch> arch = arch_from_string(optarg); 518 if (!arch) { 519 errx(1, "unknown architecture '%s'", optarg); 520 } 521 selected_architectures.insert(*arch); 522 break; 523 } 524 525 case 'p': { 526 if (!platform_dir.empty()) { 527 usage(); 528 } 529 530 platform_dir = optarg; 531 532 if (platform_dir.empty()) { 533 usage(); 534 } 535 536 struct stat st; 537 if (stat(platform_dir.c_str(), &st) != 0) { 538 err(1, "failed to stat platform directory '%s'", platform_dir.c_str()); 539 } 540 if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) { 541 errx(1, "'%s' is not a file or directory", optarg); 542 } 543 break; 544 } 545 546 case 's': 547 strict = true; 548 break; 549 550 case 'o': 551 if (!preprocessor_output_path.empty()) { 552 usage(); 553 } 554 preprocessor_output_path = optarg; 555 if (preprocessor_output_path.empty()) { 556 usage(); 557 } 558 break; 559 560 case 'f': 561 force = true; 562 break; 563 564 case 'd': 565 dump = true; 566 break; 567 568 case 'j': 569 if (!android::base::ParseInt<int>(optarg, &max_thread_count, 1)) { 570 usage(); 571 } 572 break; 573 574 case 'v': 575 verbose = true; 576 break; 577 578 case 'h': 579 usage(true); 580 break; 581 582 case 'i': 583 // Secret option for tests to -include <android/versioning.h>. 584 add_include = true; 585 break; 586 587 case 'F': 588 ignore_fortify_headers = false; 589 break; 590 591 default: 592 usage(); 593 break; 594 } 595 } 596 597 if (argc - optind > 2 || optind > argc) { 598 usage(); 599 } 600 601 HeaderLocationInformation location; 602 603 const char* top = getenv("ANDROID_BUILD_TOP"); 604 if (!top && (optind == argc || add_include)) { 605 fprintf(stderr, "versioner: failed to autodetect bionic paths. Is ANDROID_BUILD_TOP set?\n"); 606 usage(); 607 } 608 609 if (optind == argc) { 610 // Neither HEADER_PATH nor DEPS_PATH were specified, so try to figure them out. 611 std::string versioner_dir = to_string(top) + "/bionic/tools/versioner"; 612 location.header_path = versioner_dir + "/current"; 613 location.dependency_dir = versioner_dir + "/dependencies"; 614 if (platform_dir.empty()) { 615 platform_dir = versioner_dir + "/platforms"; 616 } 617 } else { 618 if (!android::base::Realpath(argv[optind], &location.header_path)) { 619 err(1, "failed to get realpath for path '%s'", argv[optind]); 620 } 621 622 if (argc - optind == 2) { 623 location.dependency_dir = argv[optind + 1]; 624 } 625 } 626 627 // Every file that lives in bits/fortify is logically a part of a header outside of bits/fortify. 628 // This makes the files there impossible to build on their own. 629 if (ignore_fortify_headers) { 630 std::string fortify_path = location.header_path; 631 if (!android::base::EndsWith(location.header_path, "/")) { 632 fortify_path += '/'; 633 } 634 fortify_path += "bits/fortify"; 635 location.ignored_directories.insert(std::move(fortify_path)); 636 } 637 638 if (selected_levels.empty()) { 639 selected_levels = default_levels; 640 } 641 642 if (selected_architectures.empty()) { 643 selected_architectures = supported_archs; 644 } 645 646 647 struct stat st; 648 if (const char *path = location.header_path.c_str(); stat(path, &st) != 0) { 649 err(1, "failed to stat '%s'", path); 650 } 651 652 std::set<CompilationType> compilation_types; 653 std::optional<NdkSymbolDatabase> symbol_database; 654 655 compilation_types = generateCompilationTypes(selected_architectures, selected_levels); 656 657 // Do this before compiling so that we can early exit if the platforms don't match what we 658 // expect. 659 if (!platform_dir.empty()) { 660 symbol_database = parsePlatforms(compilation_types, platform_dir); 661 } 662 663 auto start = std::chrono::high_resolution_clock::now(); 664 std::unique_ptr<HeaderDatabase> declaration_database = 665 compileHeaders(compilation_types, location); 666 auto end = std::chrono::high_resolution_clock::now(); 667 668 if (verbose) { 669 auto diff = (end - start) / 1.0ms; 670 printf("Compiled headers for %zu targets in %0.2LFms\n", compilation_types.size(), diff); 671 } 672 673 bool failed = false; 674 if (dump) { 675 declaration_database->dump(location.header_path + "/"); 676 } else { 677 if (!sanityCheck(declaration_database.get())) { 678 printf("versioner: sanity check failed\n"); 679 failed = true; 680 } 681 682 if (symbol_database) { 683 if (!checkVersions(compilation_types, declaration_database.get(), *symbol_database)) { 684 printf("versioner: version check failed\n"); 685 failed = true; 686 } 687 } 688 } 689 690 if (!preprocessor_output_path.empty() && (force || !failed)) { 691 failed = !preprocessHeaders(preprocessor_output_path, location.header_path, 692 declaration_database.get()); 693 } 694 return failed; 695 } 696