1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "Preprocessor.h" 18 19 #include <err.h> 20 #include <fcntl.h> 21 #include <fts.h> 22 #include <libgen.h> 23 #include <string.h> 24 #include <sys/stat.h> 25 #include <sys/types.h> 26 #include <unistd.h> 27 28 #include <deque> 29 #include <fstream> 30 #include <string> 31 #include <unordered_map> 32 33 #include <llvm/ADT/StringRef.h> 34 #include <llvm/ADT/Twine.h> 35 #include <llvm/Support/FileSystem.h> 36 #include <llvm/Support/Path.h> 37 38 #include "Arch.h" 39 #include "DeclarationDatabase.h" 40 #include "versioner.h" 41 42 using namespace std::string_literals; 43 44 static DeclarationAvailability calculateRequiredGuard(const Declaration& declaration) { 45 // To avoid redundant macro guards, the availability calculated by this function is the set 46 // difference of 'targets marked-available' from 'targets the declaration is visible in'. 47 // For example, a declaration that is visible always and introduced in 9 would return introduced 48 // in 9, but the same declaration, except only visible in 9+ would return an empty 49 // DeclarationAvailability. 50 51 // This currently only handles __INTRODUCED_IN. 52 // TODO: Do the same for __REMOVED_IN. 53 int global_min_api_visible = 0; 54 ArchMap<int> arch_visibility; 55 56 for (const auto& it : declaration.availability) { 57 const CompilationType& type = it.first; 58 59 if (global_min_api_visible == 0 || global_min_api_visible > type.api_level) { 60 global_min_api_visible = type.api_level; 61 } 62 63 if (arch_visibility[type.arch] == 0 || arch_visibility[type.arch] > type.api_level) { 64 arch_visibility[type.arch] = type.api_level; 65 } 66 } 67 68 DeclarationAvailability decl_av; 69 if (!declaration.calculateAvailability(&decl_av)) { 70 fprintf(stderr, "versioner: failed to calculate availability while preprocessing:\n"); 71 declaration.dump("", stderr, 2); 72 exit(1); 73 } 74 75 D("Calculating required guard for %s:\n", declaration.name.c_str()); 76 D(" Declaration availability: %s\n", to_string(decl_av).c_str()); 77 78 if (verbose) { 79 std::string arch_visibility_str; 80 for (Arch arch : supported_archs) { 81 if (arch_visibility[arch] != 0) { 82 arch_visibility_str += to_string(arch); 83 arch_visibility_str += ": "; 84 arch_visibility_str += std::to_string(arch_visibility[arch]); 85 arch_visibility_str += ", "; 86 } 87 } 88 if (!arch_visibility_str.empty()) { 89 arch_visibility_str.resize(arch_visibility_str.size() - 2); 90 } 91 D(" Declaration visibility: global = %d, arch = %s\n", global_min_api_visible, 92 arch_visibility_str.c_str()); 93 } 94 95 DeclarationAvailability result = decl_av; 96 if (result.global_availability.introduced <= global_min_api_visible) { 97 result.global_availability.introduced = 0; 98 } 99 100 for (Arch arch : supported_archs) { 101 if (result.arch_availability[arch].introduced <= arch_visibility[arch]) { 102 result.arch_availability[arch].introduced = 0; 103 } 104 } 105 106 D(" Calculated result: %s\n", to_string(result).c_str()); 107 D("\n"); 108 109 return result; 110 } 111 112 static std::deque<std::string> readFileLines(const std::string& path) { 113 std::ifstream is(path.c_str()); 114 std::deque<std::string> result; 115 std::string line; 116 117 while (std::getline(is, line)) { 118 result.push_back(std::move(line)); 119 } 120 121 return result; 122 } 123 124 static void writeFileLines(const std::string& path, const std::deque<std::string>& lines) { 125 if (!mkdirs(dirname(path))) { 126 err(1, "failed to create directory '%s'", dirname(path).c_str()); 127 } 128 129 std::ofstream os(path.c_str(), std::ios_base::out | std::ios_base::trunc); 130 131 for (const std::string& line : lines) { 132 os << line << "\n"; 133 } 134 } 135 136 using GuardMap = std::map<Location, DeclarationAvailability>; 137 138 static std::string generateGuardCondition(const DeclarationAvailability& avail) { 139 // Logically orred expressions that constitute the macro guard. 140 std::vector<std::string> expressions; 141 static const std::vector<std::pair<std::string, std::set<Arch>>> arch_sets = { 142 { "", supported_archs }, 143 { "!defined(__LP64__)", { Arch::arm, Arch::mips, Arch::x86 } }, 144 { "defined(__LP64__)", { Arch::arm64, Arch::mips64, Arch::x86_64 } }, 145 { "defined(__mips__)", { Arch::mips, Arch::mips64 } }, 146 }; 147 std::map<Arch, std::string> individual_archs = { 148 { Arch::arm, "defined(__arm__)" }, 149 { Arch::arm64, "defined(__aarch64__)" }, 150 { Arch::mips, "defined(__mips__) && !defined(__LP64__)" }, 151 { Arch::mips64, "defined(__mips__) && defined(__LP64__)" }, 152 { Arch::x86, "defined(__i386__)" }, 153 { Arch::x86_64, "defined(__x86_64__)" }, 154 }; 155 156 auto generate_guard = [](const std::string& arch_expr, int min_version) { 157 if (min_version == 0) { 158 return arch_expr; 159 } 160 return arch_expr + " && __ANDROID_API__ >= " + std::to_string(min_version); 161 }; 162 163 D("Generating guard for availability: %s\n", to_string(avail).c_str()); 164 if (!avail.global_availability.empty()) { 165 for (Arch arch : supported_archs) { 166 if (!avail.arch_availability[arch].empty()) { 167 errx(1, "attempted to generate guard with global and per-arch values: %s", 168 to_string(avail).c_str()); 169 } 170 } 171 172 if (avail.global_availability.introduced == 0) { 173 fprintf(stderr, "warning: attempted to generate guard with empty availability: %s\n", 174 to_string(avail).c_str()); 175 return ""; 176 } 177 178 if (avail.global_availability.introduced <= 9) { 179 return ""; 180 } 181 182 return "__ANDROID_API__ >= "s + std::to_string(avail.global_availability.introduced); 183 } 184 185 for (const auto& it : arch_sets) { 186 const std::string& arch_expr = it.first; 187 const std::set<Arch>& archs = it.second; 188 189 D(" Checking arch set '%s'\n", arch_expr.c_str()); 190 191 int version = avail.arch_availability[*it.second.begin()].introduced; 192 193 // The maximum min_version of the set. 194 int max_min_version = 0; 195 for (Arch arch : archs) { 196 if (arch_min_api[arch] > max_min_version) { 197 max_min_version = arch_min_api[arch]; 198 } 199 200 if (avail.arch_availability[arch].introduced != version) { 201 D(" Skipping arch set, availability for %s doesn't match %s\n", 202 to_string(*it.second.begin()).c_str(), to_string(arch).c_str()); 203 goto skip; 204 } 205 } 206 207 // If all of the archs in the set have a min_api that satifies version, elide the check. 208 if (max_min_version >= version) { 209 version = 0; 210 } 211 212 expressions.emplace_back(generate_guard(arch_expr, version)); 213 214 D(" Generated expression '%s'\n", expressions.rbegin()->c_str()); 215 216 for (Arch arch : archs) { 217 individual_archs.erase(arch); 218 } 219 220 skip: 221 continue; 222 } 223 224 for (const auto& it : individual_archs) { 225 const std::string& arch_expr = it.second; 226 int introduced = avail.arch_availability[it.first].introduced; 227 if (introduced == 0) { 228 expressions.emplace_back(arch_expr); 229 } else { 230 expressions.emplace_back(generate_guard(arch_expr, introduced)); 231 } 232 } 233 234 if (expressions.size() == 0) { 235 errx(1, "generated empty guard for availability %s", to_string(avail).c_str()); 236 } else if (expressions.size() == 1) { 237 return expressions[0]; 238 } 239 240 return "("s + Join(expressions, ") || (") + ")"; 241 } 242 243 // Assumes that nothing crazy is happening (e.g. having the semicolon be in a macro) 244 static FileLocation findNextSemicolon(const std::deque<std::string>& lines, FileLocation start) { 245 unsigned current_line = start.line; 246 unsigned current_column = start.column; 247 while (current_line <= lines.size()) { 248 size_t result = lines[current_line - 1].find_first_of(';', current_column - 1); 249 250 if (result != std::string::npos) { 251 FileLocation loc = { 252 .line = current_line, 253 .column = unsigned(result) + 1, 254 }; 255 256 return loc; 257 } 258 259 ++current_line; 260 current_column = 0; 261 } 262 263 errx(1, "failed to find semicolon starting from %u:%u", start.line, start.column); 264 } 265 266 // Merge adjacent blocks with identical guards. 267 static void mergeGuards(std::deque<std::string>& file_lines, GuardMap& guard_map) { 268 if (guard_map.size() < 2) { 269 return; 270 } 271 272 auto current = guard_map.begin(); 273 auto next = current; 274 ++next; 275 276 while (next != guard_map.end()) { 277 if (current->second != next->second) { 278 ++current; 279 ++next; 280 continue; 281 } 282 283 // Scan from the end of current to the beginning of next. 284 bool in_block_comment = false; 285 bool valid = true; 286 287 FileLocation current_location = current->first.end; 288 FileLocation end_location = next->first.start; 289 290 auto nextLine = [¤t_location]() { 291 ++current_location.line; 292 current_location.column = 1; 293 }; 294 295 auto nextCol = [&file_lines, ¤t_location, &nextLine]() { 296 if (current_location.column == file_lines[current_location.line - 1].length()) { 297 nextLine(); 298 } else { 299 ++current_location.column; 300 } 301 }; 302 303 // The end location will point to the semicolon, which we don't want to read, so skip it. 304 nextCol(); 305 306 while (current_location < end_location) { 307 const std::string& line = file_lines[current_location.line - 1]; 308 size_t line_index = current_location.column - 1; 309 310 if (in_block_comment) { 311 size_t pos = line.find("*/", line_index); 312 if (pos == std::string::npos) { 313 D("Didn't find block comment terminator, skipping line\n"); 314 nextLine(); 315 continue; 316 } else { 317 D("Found block comment terminator\n"); 318 in_block_comment = false; 319 current_location.column = pos + 2; 320 nextCol(); 321 continue; 322 } 323 } else { 324 size_t pos = line.find_first_not_of(" \t", line_index); 325 if (pos == std::string::npos) { 326 nextLine(); 327 continue; 328 } 329 330 current_location.column = pos + 1; 331 if (line[pos] != '/') { 332 valid = false; 333 break; 334 } 335 336 nextCol(); 337 if (line.length() <= pos + 1) { 338 // Trailing slash at the end of a line? 339 D("Trailing slash at end of line\n"); 340 valid = false; 341 break; 342 } 343 344 if (line[pos + 1] == '/') { 345 // C++ style comment 346 nextLine(); 347 } else if (line[pos + 1] == '*') { 348 // Block comment 349 nextCol(); 350 in_block_comment = true; 351 D("In a block comment\n"); 352 } else { 353 // Garbage? 354 D("Unexpected output after /: %s\n", line.substr(pos).c_str()); 355 valid = false; 356 break; 357 } 358 } 359 } 360 361 if (!valid) { 362 D("Not merging blocks %s and %s\n", to_string(current->first).c_str(), 363 to_string(next->first).c_str()); 364 ++current; 365 ++next; 366 continue; 367 } 368 369 D("Merging blocks %s and %s\n", to_string(current->first).c_str(), 370 to_string(next->first).c_str()); 371 372 Location merged = current->first; 373 merged.end = next->first.end; 374 375 DeclarationAvailability avail = current->second; 376 377 guard_map.erase(current); 378 guard_map.erase(next); 379 bool dummy; 380 std::tie(current, dummy) = guard_map.insert(std::make_pair(merged, avail)); 381 next = current; 382 ++next; 383 } 384 } 385 386 static void rewriteFile(const std::string& output_path, std::deque<std::string>& file_lines, 387 const GuardMap& guard_map) { 388 for (auto it = guard_map.rbegin(); it != guard_map.rend(); ++it) { 389 const Location& loc = it->first; 390 const DeclarationAvailability& avail = it->second; 391 392 std::string condition = generateGuardCondition(avail); 393 if (condition.empty()) { 394 continue; 395 } 396 397 std::string prologue = "\n#if "s + condition + "\n"; 398 std::string epilogue = "\n#endif /* " + condition + " */\n"; 399 400 file_lines[loc.end.line - 1].insert(loc.end.column, epilogue); 401 file_lines[loc.start.line - 1].insert(loc.start.column - 1, prologue); 402 } 403 404 if (verbose) { 405 printf("Preprocessing %s...\n", output_path.c_str()); 406 } 407 writeFileLines(output_path, file_lines); 408 } 409 410 bool preprocessHeaders(const std::string& dst_dir, const std::string& src_dir, 411 HeaderDatabase* database) { 412 std::unordered_map<std::string, GuardMap> guards; 413 std::unordered_map<std::string, std::deque<std::string>> file_lines; 414 415 for (const auto& symbol_it : database->symbols) { 416 const Symbol& symbol = symbol_it.second; 417 418 for (const auto& decl_it : symbol.declarations) { 419 const Location& location = decl_it.first; 420 const Declaration& decl = decl_it.second; 421 422 if (decl.no_guard) { 423 // No guard required. 424 continue; 425 } 426 427 DeclarationAvailability macro_guard = calculateRequiredGuard(decl); 428 if (!macro_guard.empty()) { 429 guards[location.filename][location] = macro_guard; 430 } 431 } 432 } 433 434 // Copy over the original headers before preprocessing. 435 char* fts_paths[2] = { const_cast<char*>(src_dir.c_str()), nullptr }; 436 std::unique_ptr<FTS, decltype(&fts_close)> fts(fts_open(fts_paths, FTS_LOGICAL, nullptr), 437 fts_close); 438 if (!fts) { 439 err(1, "failed to open directory %s", src_dir.c_str()); 440 } 441 442 while (FTSENT* ent = fts_read(fts.get())) { 443 llvm::StringRef path = ent->fts_path; 444 if (!path.startswith(src_dir)) { 445 err(1, "path '%s' doesn't start with source dir '%s'", ent->fts_path, src_dir.c_str()); 446 } 447 448 if (ent->fts_info != FTS_F) { 449 continue; 450 } 451 452 std::string rel_path = path.substr(src_dir.length() + 1); 453 std::string dst_path = dst_dir + "/" + rel_path; 454 llvm::StringRef parent_path = llvm::sys::path::parent_path(dst_path); 455 if (llvm::sys::fs::create_directories(parent_path)) { 456 errx(1, "failed to ensure existence of directory '%s'", parent_path.str().c_str()); 457 } 458 if (llvm::sys::fs::copy_file(path, dst_path)) { 459 errx(1, "failed to copy '%s/%s' to '%s'", src_dir.c_str(), path.str().c_str(), 460 dst_path.c_str()); 461 } 462 } 463 464 for (const auto& file_it : guards) { 465 file_lines[file_it.first] = readFileLines(file_it.first); 466 } 467 468 for (auto& file_it : guards) { 469 llvm::StringRef file_path = file_it.first; 470 GuardMap& orig_guard_map = file_it.second; 471 472 // The end positions given to us are the end of the declaration, which is some point before the 473 // semicolon. Fix up the end positions by scanning for the next semicolon. 474 GuardMap guard_map; 475 for (const auto& it : orig_guard_map) { 476 Location loc = it.first; 477 loc.end = findNextSemicolon(file_lines[file_path], loc.end); 478 guard_map[loc] = it.second; 479 } 480 481 // TODO: Make sure that the Locations don't overlap. 482 // TODO: Merge adjacent non-identical guards. 483 mergeGuards(file_lines[file_path], guard_map); 484 485 if (!file_path.startswith(src_dir)) { 486 errx(1, "input file %s is not in %s\n", file_path.str().c_str(), src_dir.c_str()); 487 } 488 489 // rel_path has a leading slash. 490 llvm::StringRef rel_path = file_path.substr(src_dir.size(), file_path.size() - src_dir.size()); 491 std::string output_path = (llvm::Twine(dst_dir) + rel_path).str(); 492 493 rewriteFile(output_path, file_lines[file_path], guard_map); 494 } 495 496 return true; 497 } 498