1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <dirent.h> 18 #include <err.h> 19 #include <limits.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <sys/stat.h> 23 #include <sys/types.h> 24 #include <unistd.h> 25 26 #if defined(__linux__) 27 #include <sched.h> 28 #endif 29 30 #include <atomic> 31 #include <chrono> 32 #include <functional> 33 #include <iostream> 34 #include <map> 35 #include <memory> 36 #include <optional> 37 #include <set> 38 #include <sstream> 39 #include <string> 40 #include <string_view> 41 #include <thread> 42 #include <unordered_map> 43 #include <vector> 44 45 #include <llvm/ADT/StringRef.h> 46 47 #include <android-base/file.h> 48 #include <android-base/macros.h> 49 #include <android-base/parseint.h> 50 #include <android-base/strings.h> 51 52 #include "Arch.h" 53 #include "DeclarationDatabase.h" 54 #include "Driver.h" 55 #include "Preprocessor.h" 56 #include "SymbolDatabase.h" 57 #include "Utils.h" 58 #include "VFS.h" 59 60 #include "versioner.h" 61 62 using namespace std::chrono_literals; 63 using namespace std::string_literals; 64 65 bool strict; 66 bool verbose; 67 bool add_include; 68 69 static int getCpuCount(); 70 static int max_thread_count = getCpuCount(); 71 72 static int getCpuCount() { 73 #if defined(__linux__) 74 cpu_set_t cpu_set; 75 int rc = sched_getaffinity(getpid(), sizeof(cpu_set), &cpu_set); 76 if (rc != 0) { 77 err(1, "sched_getaffinity failed"); 78 } 79 return CPU_COUNT(&cpu_set); 80 #else 81 return 1; 82 #endif 83 } 84 85 namespace { 86 struct HeaderLocationInformation { 87 std::string header_path; 88 std::string dependency_dir; 89 // Absolute paths to ignore all children -- including subdirectories -- of. 90 std::unordered_set<std::string> ignored_directories; 91 }; 92 } 93 94 static bool is_dir(const std::string& path) { 95 struct stat st; 96 return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); 97 } 98 99 static CompilationRequirements collectRequirements(const Arch& arch, 100 const HeaderLocationInformation& location) { 101 std::vector<std::string> headers = 102 collectHeaders(location.header_path, location.ignored_directories); 103 std::vector<std::string> dependencies; 104 105 if (is_dir(location.header_path)) { 106 dependencies.emplace_back(location.header_path); 107 } 108 109 if (!location.dependency_dir.empty()) { 110 auto collect_children = [&dependencies](const std::string& dir_path) { 111 DIR* dir = opendir(dir_path.c_str()); 112 if (!dir) { 113 err(1, "failed to open dependency directory '%s'", dir_path.c_str()); 114 } 115 116 struct dirent* dent; 117 while ((dent = readdir(dir))) { 118 if (dent->d_name[0] == '.') { 119 continue; 120 } 121 122 // TODO: Resolve symlinks. 123 std::string dependency = dir_path + "/" + dent->d_name; 124 125 struct stat st; 126 if (stat(dependency.c_str(), &st) != 0) { 127 err(1, "failed to stat dependency '%s'", dependency.c_str()); 128 } 129 130 if (!S_ISDIR(st.st_mode)) { 131 errx(1, "'%s' is not a directory", dependency.c_str()); 132 } 133 134 dependencies.push_back(dependency); 135 } 136 137 closedir(dir); 138 }; 139 140 collect_children(location.dependency_dir + "/common"); 141 collect_children(location.dependency_dir + "/" + to_string(arch)); 142 } 143 144 auto new_end = std::remove_if(headers.begin(), headers.end(), [&arch](llvm::StringRef header) { 145 for (const auto& it : header_blacklist) { 146 if (it.second.find(arch) == it.second.end()) { 147 continue; 148 } 149 150 if (header.endswith("/" + it.first)) { 151 return true; 152 } 153 } 154 return false; 155 }); 156 157 headers.erase(new_end, headers.end()); 158 159 CompilationRequirements result = { .headers = headers, .dependencies = dependencies }; 160 return result; 161 } 162 163 static std::set<CompilationType> generateCompilationTypes(const std::set<Arch> selected_architectures, 164 const std::set<int>& selected_levels) { 165 std::set<CompilationType> result; 166 for (const auto& arch : selected_architectures) { 167 int min_api = arch_min_api[arch]; 168 for (int api_level : selected_levels) { 169 if (api_level < min_api) { 170 continue; 171 } 172 173 for (int file_offset_bits : { 32, 64 }) { 174 for (bool cpp : { true, false }) { 175 CompilationType type = { 176 .arch = arch, .cpp = cpp, .api_level = api_level, .file_offset_bits = file_offset_bits 177 }; 178 result.insert(type); 179 } 180 } 181 } 182 } 183 return result; 184 } 185 186 static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<CompilationType>& types, 187 const HeaderLocationInformation& location) { 188 if (types.empty()) { 189 errx(1, "compileHeaders received no CompilationTypes"); 190 } 191 192 auto vfs = createCommonVFS(location.header_path, location.dependency_dir, add_include); 193 194 size_t thread_count = max_thread_count; 195 std::vector<std::thread> threads; 196 197 std::map<CompilationType, HeaderDatabase> header_databases; 198 std::unordered_map<Arch, CompilationRequirements> requirements; 199 200 auto result = std::make_unique<HeaderDatabase>(); 201 for (const auto& type : types) { 202 if (requirements.count(type.arch) == 0) { 203 requirements[type.arch] = collectRequirements(type.arch, location); 204 } 205 } 206 207 initializeTargetCC1FlagCache(vfs, types, requirements); 208 209 std::vector<std::pair<CompilationType, const std::string&>> jobs; 210 std::atomic<size_t> job_index(0); 211 for (CompilationType type : types) { 212 CompilationRequirements& req = requirements[type.arch]; 213 for (const std::string& header : req.headers) { 214 jobs.emplace_back(type, header); 215 } 216 } 217 218 // Dup an empty file to stdin, so that we can use `clang -include a.h -` instead of `clang a.h`, 219 // since some warnings don't get generated in files that are compiled directly. 220 FILE* empty_file = tmpfile(); 221 if (!empty_file) { 222 err(1, "failed to create temporary file"); 223 } 224 225 int empty_file_fd = fileno(empty_file); 226 if (empty_file_fd == -1) { 227 errx(1, "fileno failed on tmpfile"); 228 } 229 230 dup2(empty_file_fd, STDIN_FILENO); 231 fclose(empty_file); 232 233 thread_count = std::min(thread_count, jobs.size()); 234 235 if (thread_count == 1) { 236 for (const auto& job : jobs) { 237 compileHeader(vfs, result.get(), job.first, job.second); 238 } 239 } else { 240 // Spawn threads. 241 for (size_t i = 0; i < thread_count; ++i) { 242 threads.emplace_back([&jobs, &job_index, &result, vfs]() { 243 while (true) { 244 size_t idx = job_index++; 245 if (idx >= jobs.size()) { 246 return; 247 } 248 249 const auto& job = jobs[idx]; 250 compileHeader(vfs, result.get(), job.first, job.second); 251 } 252 }); 253 } 254 255 // Reap them. 256 for (auto& thread : threads) { 257 thread.join(); 258 } 259 threads.clear(); 260 } 261 262 return result; 263 } 264 265 static std::set<CompilationType> getCompilationTypes(const Declaration* decl) { 266 std::set<CompilationType> result; 267 for (const auto& it : decl->availability) { 268 result.insert(it.first); 269 } 270 return result; 271 } 272 273 template<typename T> 274 static std::vector<T> Intersection(const std::set<T>& a, const std::set<T>& b) { 275 std::vector<T> intersection; 276 std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(intersection)); 277 return intersection; 278 } 279 280 // Perform a sanity check on a symbol's declarations, enforcing the following invariants: 281 // 1. At most one inline definition of the function exists. 282 // 2. All of the availability declarations for a symbol are compatible. 283 // If a function is declared as an inline before a certain version, the inline definition 284 // should have no version tag. 285 // 3. Each availability type must only be present globally or on a per-arch basis. 286 // (e.g. __INTRODUCED_IN_ARM(9) __INTRODUCED_IN_X86(10) __DEPRECATED_IN(11) is fine, 287 // but not __INTRODUCED_IN(9) __INTRODUCED_IN_X86(10)) 288 static bool checkSymbol(const Symbol& symbol) { 289 std::string cwd = getWorkingDir() + "/"; 290 291 std::unordered_map<const Declaration*, std::set<CompilationType>> inline_definitions; 292 for (const auto& decl_it : symbol.declarations) { 293 const Declaration* decl = &decl_it.second; 294 if (decl->is_definition) { 295 std::set<CompilationType> compilation_types = getCompilationTypes(decl); 296 for (const auto& inline_def_it : inline_definitions) { 297 auto intersection = Intersection(compilation_types, inline_def_it.second); 298 if (!intersection.empty()) { 299 fprintf(stderr, "versioner: conflicting inline definitions for symbol %s:\n", 300 symbol.name.c_str()); 301 fprintf(stderr, " declarations visible in: %s\n", Join(intersection, ", ").c_str()); 302 decl->dump(cwd, stderr, 4); 303 inline_def_it.first->dump(cwd, stderr, 4); 304 return false; 305 } 306 } 307 308 inline_definitions[decl] = std::move(compilation_types); 309 } 310 311 DeclarationAvailability availability; 312 if (!decl->calculateAvailability(&availability)) { 313 fprintf(stderr, "versioner: failed to calculate availability for declaration:\n"); 314 decl->dump(cwd, stderr, 2); 315 return false; 316 } 317 318 if (decl->is_definition && !availability.empty()) { 319 fprintf(stderr, "versioner: inline definition has non-empty versioning information:\n"); 320 decl->dump(cwd, stderr, 2); 321 return false; 322 } 323 } 324 325 DeclarationAvailability availability; 326 if (!symbol.calculateAvailability(&availability)) { 327 fprintf(stderr, "versioner: inconsistent availability for symbol '%s'\n", symbol.name.c_str()); 328 symbol.dump(cwd); 329 return false; 330 } 331 332 // TODO: Check invariant #3. 333 return true; 334 } 335 336 static bool sanityCheck(const HeaderDatabase* database) { 337 bool error = false; 338 std::string cwd = getWorkingDir() + "/"; 339 340 for (const auto& symbol_it : database->symbols) { 341 if (!checkSymbol(symbol_it.second)) { 342 error = true; 343 } 344 } 345 return !error; 346 } 347 348 // Check that our symbol availability declarations match the actual NDK 349 // platform symbol availability. 350 static bool checkVersions(const std::set<CompilationType>& types, 351 const HeaderDatabase* header_database, 352 const NdkSymbolDatabase& symbol_database) { 353 std::string cwd = getWorkingDir() + "/"; 354 bool failed = false; 355 356 std::map<Arch, std::set<CompilationType>> arch_types; 357 for (const CompilationType& type : types) { 358 arch_types[type.arch].insert(type); 359 } 360 361 std::set<std::string> completely_unavailable; 362 std::map<std::string, std::set<CompilationType>> missing_availability; 363 std::map<std::string, std::set<CompilationType>> extra_availability; 364 365 for (const auto& symbol_it : header_database->symbols) { 366 const auto& symbol_name = symbol_it.first; 367 DeclarationAvailability symbol_availability; 368 369 if (!symbol_it.second.calculateAvailability(&symbol_availability)) { 370 errx(1, "failed to calculate symbol availability"); 371 } 372 373 const auto platform_availability_it = symbol_database.find(symbol_name); 374 if (platform_availability_it == symbol_database.end()) { 375 completely_unavailable.insert(symbol_name); 376 continue; 377 } 378 379 const auto& platform_availability = platform_availability_it->second; 380 381 for (const CompilationType& type : types) { 382 bool should_be_available = true; 383 const auto& global_availability = symbol_availability.global_availability; 384 const auto& arch_availability = symbol_availability.arch_availability[type.arch]; 385 if (global_availability.introduced != 0 && global_availability.introduced > type.api_level) { 386 should_be_available = false; 387 } 388 389 if (arch_availability.introduced != 0 && arch_availability.introduced > type.api_level) { 390 should_be_available = false; 391 } 392 393 if (global_availability.obsoleted != 0 && global_availability.obsoleted <= type.api_level) { 394 should_be_available = false; 395 } 396 397 if (arch_availability.obsoleted != 0 && arch_availability.obsoleted <= type.api_level) { 398 should_be_available = false; 399 } 400 401 if (arch_availability.future) { 402 continue; 403 } 404 405 // The function declaration might be (validly) missing for the given CompilationType. 406 if (!symbol_it.second.hasDeclaration(type)) { 407 should_be_available = false; 408 } 409 410 bool is_available = platform_availability.count(type); 411 412 if (should_be_available != is_available) { 413 if (is_available) { 414 extra_availability[symbol_name].insert(type); 415 } else { 416 missing_availability[symbol_name].insert(type); 417 } 418 } 419 } 420 } 421 422 for (const auto& it : symbol_database) { 423 const std::string& symbol_name = it.first; 424 425 bool symbol_error = false; 426 if (auto missing_it = missing_availability.find(symbol_name); 427 missing_it != missing_availability.end()) { 428 printf("%s: declaration marked available but symbol missing in [%s]\n", symbol_name.c_str(), 429 Join(missing_it->second, ", ").c_str()); 430 symbol_error = true; 431 failed = true; 432 } 433 434 if (strict) { 435 if (auto extra_it = extra_availability.find(symbol_name); 436 extra_it != extra_availability.end()) { 437 printf("%s: declaration marked unavailable but symbol available in [%s]\n", 438 symbol_name.c_str(), Join(extra_it->second, ", ").c_str()); 439 symbol_error = true; 440 failed = true; 441 } 442 } 443 444 if (symbol_error) { 445 if (auto symbol_it = header_database->symbols.find(symbol_name); 446 symbol_it != header_database->symbols.end()) { 447 symbol_it->second.dump(cwd); 448 } else { 449 errx(1, "failed to find symbol in header database"); 450 } 451 } 452 } 453 454 // TODO: Verify that function/variable declarations are actually function/variable symbols. 455 return !failed; 456 } 457 458 static void usage(bool help = false) { 459 fprintf(stderr, "Usage: versioner [OPTION]... [HEADER_PATH] [DEPS_PATH]\n"); 460 if (!help) { 461 printf("Try 'versioner -h' for more information.\n"); 462 exit(1); 463 } else { 464 fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n"); 465 fprintf(stderr, "Autodetects paths if HEADER_PATH and DEPS_PATH are not specified\n"); 466 fprintf(stderr, "\n"); 467 fprintf(stderr, "Target specification (defaults to all):\n"); 468 fprintf(stderr, " -a API_LEVEL\tbuild with specified API level (can be repeated)\n"); 469 fprintf(stderr, " \t\tdefaults to %s\n", Join(default_levels).c_str()); 470 fprintf(stderr, " -r ARCH\tbuild with specified architecture (can be repeated)\n"); 471 fprintf(stderr, " \t\tvalid architectures are %s\n", Join(supported_archs).c_str()); 472 fprintf(stderr, "\n"); 473 fprintf(stderr, "Validation:\n"); 474 fprintf(stderr, " -p PATH\tcompare against NDK platform at PATH\n"); 475 fprintf(stderr, " -s\t\tenable strict warnings\n"); 476 fprintf(stderr, "\n"); 477 fprintf(stderr, "Preprocessing:\n"); 478 fprintf(stderr, " -o PATH\tpreprocess header files and emit them at PATH\n"); 479 fprintf(stderr, " -f\t\tpreprocess header files even if validation fails\n"); 480 fprintf(stderr, "\n"); 481 fprintf(stderr, "Miscellaneous:\n"); 482 fprintf(stderr, " -F\t\tdo not ignore FORTIFY headers by default\n"); 483 fprintf(stderr, " -d\t\tdump function availability\n"); 484 fprintf(stderr, " -j THREADS\tmaximum number of threads to use\n"); 485 fprintf(stderr, " -v\t\tenable verbose logging\n"); 486 fprintf(stderr, " -h\t\tdisplay this message\n"); 487 exit(0); 488 } 489 } 490 491 // versioner uses a prebuilt version of clang, which is not up-to-date wrt/ 492 // container annotations. So disable container overflow checking. b/37775238 493 extern "C" const char* __asan_default_options() { 494 return "detect_container_overflow=0"; 495 } 496 497 int main(int argc, char** argv) { 498 std::string cwd = getWorkingDir() + "/"; 499 std::string platform_dir; 500 std::set<Arch> selected_architectures; 501 std::set<int> selected_levels; 502 std::string preprocessor_output_path; 503 bool force = false; 504 bool dump = false; 505 bool ignore_fortify_headers = true; 506 507 int c; 508 while ((c = getopt(argc, argv, "a:r:p:so:fdj:vhFi")) != -1) { 509 switch (c) { 510 case 'a': { 511 char* end; 512 int api_level = strtol(optarg, &end, 10); 513 if (end == optarg || strlen(end) > 0) { 514 usage(); 515 } 516 517 selected_levels.insert(api_level); 518 break; 519 } 520 521 case 'r': { 522 Arch arch = arch_from_string(optarg); 523 selected_architectures.insert(arch); 524 break; 525 } 526 527 case 'p': { 528 if (!platform_dir.empty()) { 529 usage(); 530 } 531 532 platform_dir = optarg; 533 534 if (platform_dir.empty()) { 535 usage(); 536 } 537 538 struct stat st; 539 if (stat(platform_dir.c_str(), &st) != 0) { 540 err(1, "failed to stat platform directory '%s'", platform_dir.c_str()); 541 } 542 if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) { 543 errx(1, "'%s' is not a file or directory", optarg); 544 } 545 break; 546 } 547 548 case 's': 549 strict = true; 550 break; 551 552 case 'o': 553 if (!preprocessor_output_path.empty()) { 554 usage(); 555 } 556 preprocessor_output_path = optarg; 557 if (preprocessor_output_path.empty()) { 558 usage(); 559 } 560 break; 561 562 case 'f': 563 force = true; 564 break; 565 566 case 'd': 567 dump = true; 568 break; 569 570 case 'j': 571 if (!android::base::ParseInt<int>(optarg, &max_thread_count, 1)) { 572 usage(); 573 } 574 break; 575 576 case 'v': 577 verbose = true; 578 break; 579 580 case 'h': 581 usage(true); 582 break; 583 584 case 'i': 585 // Secret option for tests to -include <android/versioning.h>. 586 add_include = true; 587 break; 588 589 case 'F': 590 ignore_fortify_headers = false; 591 break; 592 593 default: 594 usage(); 595 break; 596 } 597 } 598 599 if (argc - optind > 2 || optind > argc) { 600 usage(); 601 } 602 603 HeaderLocationInformation location; 604 605 const char* top = getenv("ANDROID_BUILD_TOP"); 606 if (!top && (optind == argc || add_include)) { 607 fprintf(stderr, "versioner: failed to autodetect bionic paths. Is ANDROID_BUILD_TOP set?\n"); 608 usage(); 609 } 610 611 if (optind == argc) { 612 // Neither HEADER_PATH nor DEPS_PATH were specified, so try to figure them out. 613 std::string versioner_dir = to_string(top) + "/bionic/tools/versioner"; 614 location.header_path = versioner_dir + "/current"; 615 location.dependency_dir = versioner_dir + "/dependencies"; 616 } else { 617 if (!android::base::Realpath(argv[optind], &location.header_path)) { 618 err(1, "failed to get realpath for path '%s'", argv[optind]); 619 } 620 621 if (argc - optind == 2) { 622 location.dependency_dir = argv[optind + 1]; 623 } 624 } 625 626 // Every file that lives in bits/fortify is logically a part of a header outside of bits/fortify. 627 // This makes the files there impossible to build on their own. 628 if (ignore_fortify_headers) { 629 std::string fortify_path = location.header_path; 630 if (!android::base::EndsWith(location.header_path, "/")) { 631 fortify_path += '/'; 632 } 633 fortify_path += "bits/fortify"; 634 location.ignored_directories.insert(std::move(fortify_path)); 635 } 636 637 if (selected_levels.empty()) { 638 selected_levels = default_levels; 639 } 640 641 if (selected_architectures.empty()) { 642 selected_architectures = supported_archs; 643 } 644 645 646 struct stat st; 647 if (const char *path = location.header_path.c_str(); stat(path, &st) != 0) { 648 err(1, "failed to stat '%s'", path); 649 } 650 651 std::set<CompilationType> compilation_types; 652 std::optional<NdkSymbolDatabase> symbol_database; 653 654 compilation_types = generateCompilationTypes(selected_architectures, selected_levels); 655 656 auto start = std::chrono::high_resolution_clock::now(); 657 std::unique_ptr<HeaderDatabase> declaration_database = 658 compileHeaders(compilation_types, location); 659 auto end = std::chrono::high_resolution_clock::now(); 660 661 if (verbose) { 662 auto diff = (end - start) / 1.0ms; 663 printf("Compiled headers for %zu targets in %0.2LFms\n", compilation_types.size(), diff); 664 } 665 666 bool failed = false; 667 if (dump) { 668 declaration_database->dump(location.header_path + "/"); 669 } else { 670 if (!sanityCheck(declaration_database.get())) { 671 printf("versioner: sanity check failed\n"); 672 failed = true; 673 } 674 675 if (symbol_database) { 676 if (!checkVersions(compilation_types, declaration_database.get(), *symbol_database)) { 677 printf("versioner: version check failed\n"); 678 failed = true; 679 } 680 } 681 } 682 683 if (!preprocessor_output_path.empty() && (force || !failed)) { 684 failed = !preprocessHeaders(preprocessor_output_path, location.header_path, 685 declaration_database.get()); 686 } 687 return failed; 688 } 689