1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_DEX2OAT_LINKER_IMAGE_TEST_H_ 18 #define ART_DEX2OAT_LINKER_IMAGE_TEST_H_ 19 20 #include "image.h" 21 22 #include <memory> 23 #include <string> 24 #include <string_view> 25 #include <vector> 26 27 #include "android-base/stringprintf.h" 28 #include "android-base/strings.h" 29 30 #include "art_method-inl.h" 31 #include "base/file_utils.h" 32 #include "base/hash_set.h" 33 #include "base/stl_util.h" 34 #include "base/unix_file/fd_file.h" 35 #include "base/utils.h" 36 #include "class_linker-inl.h" 37 #include "common_compiler_driver_test.h" 38 #include "compiler_callbacks.h" 39 #include "debug/method_debug_info.h" 40 #include "dex/quick_compiler_callbacks.h" 41 #include "dex/signature-inl.h" 42 #include "driver/compiler_driver.h" 43 #include "driver/compiler_options.h" 44 #include "gc/space/image_space.h" 45 #include "image_writer.h" 46 #include "linker/elf_writer.h" 47 #include "linker/elf_writer_quick.h" 48 #include "linker/multi_oat_relative_patcher.h" 49 #include "lock_word.h" 50 #include "mirror/object-inl.h" 51 #include "oat_writer.h" 52 #include "scoped_thread_state_change-inl.h" 53 #include "signal_catcher.h" 54 #include "stream/buffered_output_stream.h" 55 #include "stream/file_output_stream.h" 56 57 namespace art { 58 namespace linker { 59 60 static const uintptr_t kRequestedImageBase = ART_BASE_ADDRESS; 61 62 struct CompilationHelper { 63 std::vector<std::string> dex_file_locations; 64 std::vector<ScratchFile> image_locations; 65 std::vector<std::unique_ptr<const DexFile>> extra_dex_files; 66 std::vector<ScratchFile> image_files; 67 std::vector<ScratchFile> oat_files; 68 std::vector<ScratchFile> vdex_files; 69 std::string image_dir; 70 71 std::vector<size_t> GetImageObjectSectionSizes(); 72 73 ~CompilationHelper(); 74 }; 75 76 class ImageTest : public CommonCompilerDriverTest { 77 protected: 78 void SetUp() override { 79 ReserveImageSpace(); 80 CommonCompilerTest::SetUp(); 81 } 82 83 void TestWriteRead(ImageHeader::StorageMode storage_mode, uint32_t max_image_block_size); 84 85 void Compile(ImageHeader::StorageMode storage_mode, 86 uint32_t max_image_block_size, 87 /*out*/ CompilationHelper& out_helper, 88 const std::string& extra_dex = "", 89 const std::initializer_list<std::string>& image_classes = {}, 90 const std::initializer_list<std::string>& image_classes_failing_aot_clinit = {}); 91 92 void SetUpRuntimeOptions(RuntimeOptions* options) override { 93 CommonCompilerTest::SetUpRuntimeOptions(options); 94 QuickCompilerCallbacks* new_callbacks = 95 new QuickCompilerCallbacks(CompilerCallbacks::CallbackMode::kCompileBootImage); 96 new_callbacks->SetVerificationResults(verification_results_.get()); 97 callbacks_.reset(new_callbacks); 98 options->push_back(std::make_pair("compilercallbacks", callbacks_.get())); 99 } 100 101 std::unique_ptr<HashSet<std::string>> GetImageClasses() override { 102 return std::make_unique<HashSet<std::string>>(image_classes_); 103 } 104 105 ArtMethod* FindCopiedMethod(ArtMethod* origin, ObjPtr<mirror::Class> klass) 106 REQUIRES_SHARED(Locks::mutator_lock_) { 107 PointerSize pointer_size = class_linker_->GetImagePointerSize(); 108 for (ArtMethod& m : klass->GetCopiedMethods(pointer_size)) { 109 if (strcmp(origin->GetName(), m.GetName()) == 0 && 110 origin->GetSignature() == m.GetSignature()) { 111 return &m; 112 } 113 } 114 return nullptr; 115 } 116 117 private: 118 void DoCompile(ImageHeader::StorageMode storage_mode, /*out*/ CompilationHelper& out_helper); 119 120 HashSet<std::string> image_classes_; 121 }; 122 123 inline CompilationHelper::~CompilationHelper() { 124 for (ScratchFile& image_file : image_files) { 125 image_file.Unlink(); 126 } 127 for (ScratchFile& oat_file : oat_files) { 128 oat_file.Unlink(); 129 } 130 for (ScratchFile& vdex_file : vdex_files) { 131 vdex_file.Unlink(); 132 } 133 const int rmdir_result = rmdir(image_dir.c_str()); 134 CHECK_EQ(0, rmdir_result); 135 } 136 137 inline std::vector<size_t> CompilationHelper::GetImageObjectSectionSizes() { 138 std::vector<size_t> ret; 139 for (ScratchFile& image_file : image_files) { 140 std::unique_ptr<File> file(OS::OpenFileForReading(image_file.GetFilename().c_str())); 141 CHECK(file.get() != nullptr); 142 ImageHeader image_header; 143 CHECK_EQ(file->ReadFully(&image_header, sizeof(image_header)), true); 144 CHECK(image_header.IsValid()); 145 ret.push_back(image_header.GetObjectsSection().Size()); 146 } 147 return ret; 148 } 149 150 inline void ImageTest::DoCompile(ImageHeader::StorageMode storage_mode, 151 /*out*/ CompilationHelper& out_helper) { 152 CompilerDriver* driver = compiler_driver_.get(); 153 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 154 std::vector<const DexFile*> class_path = class_linker->GetBootClassPath(); 155 156 for (const std::unique_ptr<const DexFile>& dex_file : out_helper.extra_dex_files) { 157 { 158 ScopedObjectAccess soa(Thread::Current()); 159 // Inject in boot class path so that the compiler driver can see it. 160 class_linker->AppendToBootClassPath(soa.Self(), *dex_file.get()); 161 } 162 class_path.push_back(dex_file.get()); 163 } 164 165 // Enable write for dex2dex. 166 for (const DexFile* dex_file : class_path) { 167 out_helper.dex_file_locations.push_back(dex_file->GetLocation()); 168 if (dex_file->IsReadOnly()) { 169 dex_file->EnableWrite(); 170 } 171 } 172 { 173 // Create a generic tmp file, to be the base of the .art and .oat temporary files. 174 ScratchFile location; 175 std::vector<std::string> image_locations = 176 gc::space::ImageSpace::ExpandMultiImageLocations(out_helper.dex_file_locations, 177 location.GetFilename() + ".art"); 178 for (size_t i = 0u; i != class_path.size(); ++i) { 179 out_helper.image_locations.push_back(ScratchFile(image_locations[i])); 180 } 181 } 182 std::vector<std::string> image_filenames; 183 for (ScratchFile& file : out_helper.image_locations) { 184 std::string image_filename(GetSystemImageFilename(file.GetFilename().c_str(), kRuntimeISA)); 185 image_filenames.push_back(image_filename); 186 size_t pos = image_filename.rfind('/'); 187 CHECK_NE(pos, std::string::npos) << image_filename; 188 if (out_helper.image_dir.empty()) { 189 out_helper.image_dir = image_filename.substr(0, pos); 190 int mkdir_result = mkdir(out_helper.image_dir.c_str(), 0700); 191 CHECK_EQ(0, mkdir_result) << out_helper.image_dir; 192 } 193 out_helper.image_files.push_back(ScratchFile(OS::CreateEmptyFile(image_filename.c_str()))); 194 } 195 196 std::vector<std::string> oat_filenames; 197 std::vector<std::string> vdex_filenames; 198 for (const std::string& image_filename : image_filenames) { 199 std::string oat_filename = ReplaceFileExtension(image_filename, "oat"); 200 out_helper.oat_files.push_back(ScratchFile(OS::CreateEmptyFile(oat_filename.c_str()))); 201 oat_filenames.push_back(oat_filename); 202 std::string vdex_filename = ReplaceFileExtension(image_filename, "vdex"); 203 out_helper.vdex_files.push_back(ScratchFile(OS::CreateEmptyFile(vdex_filename.c_str()))); 204 vdex_filenames.push_back(vdex_filename); 205 } 206 207 std::unordered_map<const DexFile*, size_t> dex_file_to_oat_index_map; 208 size_t image_idx = 0; 209 for (const DexFile* dex_file : class_path) { 210 dex_file_to_oat_index_map.emplace(dex_file, image_idx); 211 ++image_idx; 212 } 213 // TODO: compile_pic should be a test argument. 214 std::unique_ptr<ImageWriter> writer(new ImageWriter(*compiler_options_, 215 kRequestedImageBase, 216 storage_mode, 217 oat_filenames, 218 dex_file_to_oat_index_map, 219 /*class_loader=*/ nullptr, 220 /*dirty_image_objects=*/ nullptr)); 221 { 222 { 223 jobject class_loader = nullptr; 224 TimingLogger timings("ImageTest::WriteRead", false, false); 225 CompileAll(class_loader, class_path, &timings); 226 227 TimingLogger::ScopedTiming t("WriteElf", &timings); 228 SafeMap<std::string, std::string> key_value_store; 229 key_value_store.Put(OatHeader::kBootClassPathKey, 230 android::base::Join(out_helper.dex_file_locations, ':')); 231 232 std::vector<std::unique_ptr<ElfWriter>> elf_writers; 233 std::vector<std::unique_ptr<OatWriter>> oat_writers; 234 for (ScratchFile& oat_file : out_helper.oat_files) { 235 elf_writers.emplace_back(CreateElfWriterQuick(*compiler_options_, oat_file.GetFile())); 236 elf_writers.back()->Start(); 237 oat_writers.emplace_back(new OatWriter(*compiler_options_, 238 &timings, 239 /*profile_compilation_info*/nullptr, 240 CompactDexLevel::kCompactDexLevelNone)); 241 } 242 243 std::vector<OutputStream*> rodata; 244 std::vector<MemMap> opened_dex_files_maps; 245 std::vector<std::unique_ptr<const DexFile>> opened_dex_files; 246 // Now that we have finalized key_value_store_, start writing the oat file. 247 for (size_t i = 0, size = oat_writers.size(); i != size; ++i) { 248 const DexFile* dex_file = class_path[i]; 249 rodata.push_back(elf_writers[i]->StartRoData()); 250 ArrayRef<const uint8_t> raw_dex_file( 251 reinterpret_cast<const uint8_t*>(&dex_file->GetHeader()), 252 dex_file->GetHeader().file_size_); 253 oat_writers[i]->AddRawDexFileSource(raw_dex_file, 254 dex_file->GetLocation().c_str(), 255 dex_file->GetLocationChecksum()); 256 257 std::vector<MemMap> cur_opened_dex_files_maps; 258 std::vector<std::unique_ptr<const DexFile>> cur_opened_dex_files; 259 bool dex_files_ok = oat_writers[i]->WriteAndOpenDexFiles( 260 out_helper.vdex_files[i].GetFile(), 261 rodata.back(), 262 (i == 0u) ? &key_value_store : nullptr, 263 /* verify */ false, // Dex files may be dex-to-dex-ed, don't verify. 264 /* update_input_vdex */ false, 265 /* copy_dex_files */ CopyOption::kOnlyIfCompressed, 266 &cur_opened_dex_files_maps, 267 &cur_opened_dex_files); 268 ASSERT_TRUE(dex_files_ok); 269 270 if (!cur_opened_dex_files_maps.empty()) { 271 for (MemMap& cur_map : cur_opened_dex_files_maps) { 272 opened_dex_files_maps.push_back(std::move(cur_map)); 273 } 274 for (std::unique_ptr<const DexFile>& cur_dex_file : cur_opened_dex_files) { 275 // dex_file_oat_index_map_.emplace(dex_file.get(), i); 276 opened_dex_files.push_back(std::move(cur_dex_file)); 277 } 278 } else { 279 ASSERT_TRUE(cur_opened_dex_files.empty()); 280 } 281 } 282 bool image_space_ok = writer->PrepareImageAddressSpace(&timings); 283 ASSERT_TRUE(image_space_ok); 284 285 DCHECK_EQ(out_helper.vdex_files.size(), out_helper.oat_files.size()); 286 for (size_t i = 0, size = out_helper.oat_files.size(); i != size; ++i) { 287 MultiOatRelativePatcher patcher(compiler_options_->GetInstructionSet(), 288 compiler_options_->GetInstructionSetFeatures(), 289 driver->GetCompiledMethodStorage()); 290 OatWriter* const oat_writer = oat_writers[i].get(); 291 ElfWriter* const elf_writer = elf_writers[i].get(); 292 std::vector<const DexFile*> cur_dex_files(1u, class_path[i]); 293 oat_writer->Initialize(driver, writer.get(), cur_dex_files); 294 295 std::unique_ptr<BufferedOutputStream> vdex_out = 296 std::make_unique<BufferedOutputStream>( 297 std::make_unique<FileOutputStream>(out_helper.vdex_files[i].GetFile())); 298 oat_writer->WriteVerifierDeps(vdex_out.get(), nullptr); 299 oat_writer->WriteQuickeningInfo(vdex_out.get()); 300 oat_writer->WriteChecksumsAndVdexHeader(vdex_out.get()); 301 302 oat_writer->PrepareLayout(&patcher); 303 elf_writer->PrepareDynamicSection(oat_writer->GetOatHeader().GetExecutableOffset(), 304 oat_writer->GetCodeSize(), 305 oat_writer->GetDataBimgRelRoSize(), 306 oat_writer->GetBssSize(), 307 oat_writer->GetBssMethodsOffset(), 308 oat_writer->GetBssRootsOffset(), 309 oat_writer->GetVdexSize()); 310 311 writer->UpdateOatFileLayout(i, 312 elf_writer->GetLoadedSize(), 313 oat_writer->GetOatDataOffset(), 314 oat_writer->GetOatSize()); 315 316 bool rodata_ok = oat_writer->WriteRodata(rodata[i]); 317 ASSERT_TRUE(rodata_ok); 318 elf_writer->EndRoData(rodata[i]); 319 320 OutputStream* text = elf_writer->StartText(); 321 bool text_ok = oat_writer->WriteCode(text); 322 ASSERT_TRUE(text_ok); 323 elf_writer->EndText(text); 324 325 if (oat_writer->GetDataBimgRelRoSize() != 0u) { 326 OutputStream* data_bimg_rel_ro = elf_writer->StartDataBimgRelRo(); 327 bool data_bimg_rel_ro_ok = oat_writer->WriteDataBimgRelRo(data_bimg_rel_ro); 328 ASSERT_TRUE(data_bimg_rel_ro_ok); 329 elf_writer->EndDataBimgRelRo(data_bimg_rel_ro); 330 } 331 332 bool header_ok = oat_writer->WriteHeader(elf_writer->GetStream()); 333 ASSERT_TRUE(header_ok); 334 335 writer->UpdateOatFileHeader(i, oat_writer->GetOatHeader()); 336 337 elf_writer->WriteDynamicSection(); 338 elf_writer->WriteDebugInfo(oat_writer->GetDebugInfo()); 339 340 bool success = elf_writer->End(); 341 ASSERT_TRUE(success); 342 } 343 } 344 345 bool success_image = writer->Write(kInvalidFd, 346 image_filenames, 347 oat_filenames); 348 ASSERT_TRUE(success_image); 349 350 for (size_t i = 0, size = oat_filenames.size(); i != size; ++i) { 351 const char* oat_filename = oat_filenames[i].c_str(); 352 std::unique_ptr<File> oat_file(OS::OpenFileReadWrite(oat_filename)); 353 ASSERT_TRUE(oat_file != nullptr); 354 bool success_fixup = ElfWriter::Fixup(oat_file.get(), writer->GetOatDataBegin(i)); 355 ASSERT_TRUE(success_fixup); 356 ASSERT_EQ(oat_file->FlushCloseOrErase(), 0) << "Could not flush and close oat file " 357 << oat_filename; 358 } 359 } 360 } 361 362 inline void ImageTest::Compile( 363 ImageHeader::StorageMode storage_mode, 364 uint32_t max_image_block_size, 365 CompilationHelper& helper, 366 const std::string& extra_dex, 367 const std::initializer_list<std::string>& image_classes, 368 const std::initializer_list<std::string>& image_classes_failing_aot_clinit) { 369 for (const std::string& image_class : image_classes_failing_aot_clinit) { 370 ASSERT_TRUE(ContainsElement(image_classes, image_class)); 371 } 372 for (const std::string& image_class : image_classes) { 373 image_classes_.insert(image_class); 374 } 375 number_of_threads_ = kIsTargetBuild ? 2U : 16U; 376 CreateCompilerDriver(); 377 // Set inline filter values. 378 compiler_options_->SetInlineMaxCodeUnits(CompilerOptions::kDefaultInlineMaxCodeUnits); 379 compiler_options_->SetMaxImageBlockSize(max_image_block_size); 380 image_classes_.clear(); 381 if (!extra_dex.empty()) { 382 helper.extra_dex_files = OpenTestDexFiles(extra_dex.c_str()); 383 } 384 DoCompile(storage_mode, helper); 385 if (image_classes.begin() != image_classes.end()) { 386 // Make sure the class got initialized. 387 ScopedObjectAccess soa(Thread::Current()); 388 ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); 389 for (const std::string& image_class : image_classes) { 390 ObjPtr<mirror::Class> klass = 391 class_linker->FindSystemClass(Thread::Current(), image_class.c_str()); 392 EXPECT_TRUE(klass != nullptr); 393 EXPECT_TRUE(klass->IsResolved()); 394 if (ContainsElement(image_classes_failing_aot_clinit, image_class)) { 395 EXPECT_FALSE(klass->IsInitialized()); 396 } else { 397 EXPECT_TRUE(klass->IsInitialized()); 398 } 399 } 400 } 401 } 402 403 inline void ImageTest::TestWriteRead(ImageHeader::StorageMode storage_mode, 404 uint32_t max_image_block_size) { 405 CompilationHelper helper; 406 Compile(storage_mode, max_image_block_size, /*out*/ helper); 407 std::vector<uint64_t> image_file_sizes; 408 for (ScratchFile& image_file : helper.image_files) { 409 std::unique_ptr<File> file(OS::OpenFileForReading(image_file.GetFilename().c_str())); 410 ASSERT_TRUE(file.get() != nullptr); 411 ImageHeader image_header; 412 ASSERT_EQ(file->ReadFully(&image_header, sizeof(image_header)), true); 413 ASSERT_TRUE(image_header.IsValid()); 414 const auto& bitmap_section = image_header.GetImageBitmapSection(); 415 ASSERT_GE(bitmap_section.Offset(), sizeof(image_header)); 416 ASSERT_NE(0U, bitmap_section.Size()); 417 418 gc::Heap* heap = Runtime::Current()->GetHeap(); 419 ASSERT_TRUE(heap->HaveContinuousSpaces()); 420 gc::space::ContinuousSpace* space = heap->GetNonMovingSpace(); 421 ASSERT_FALSE(space->IsImageSpace()); 422 ASSERT_TRUE(space != nullptr); 423 ASSERT_TRUE(space->IsMallocSpace()); 424 image_file_sizes.push_back(file->GetLength()); 425 } 426 427 // Need to delete the compiler since it has worker threads which are attached to runtime. 428 compiler_driver_.reset(); 429 430 // Tear down old runtime before making a new one, clearing out misc state. 431 432 // Remove the reservation of the memory for use to load the image. 433 // Need to do this before we reset the runtime. 434 UnreserveImageSpace(); 435 436 helper.extra_dex_files.clear(); 437 runtime_.reset(); 438 java_lang_dex_file_ = nullptr; 439 440 MemMap::Init(); 441 442 RuntimeOptions options; 443 options.emplace_back(GetClassPathOption("-Xbootclasspath:", GetLibCoreDexFileNames()), nullptr); 444 options.emplace_back( 445 GetClassPathOption("-Xbootclasspath-locations:", GetLibCoreDexLocations()), nullptr); 446 std::string image("-Ximage:"); 447 image.append(helper.image_locations[0].GetFilename()); 448 options.push_back(std::make_pair(image.c_str(), static_cast<void*>(nullptr))); 449 // By default the compiler this creates will not include patch information. 450 options.push_back(std::make_pair("-Xnorelocate", nullptr)); 451 452 if (!Runtime::Create(options, false)) { 453 LOG(FATAL) << "Failed to create runtime"; 454 return; 455 } 456 runtime_.reset(Runtime::Current()); 457 // Runtime::Create acquired the mutator_lock_ that is normally given away when we Runtime::Start, 458 // give it away now and then switch to a more managable ScopedObjectAccess. 459 Thread::Current()->TransitionFromRunnableToSuspended(kNative); 460 ScopedObjectAccess soa(Thread::Current()); 461 ASSERT_TRUE(runtime_.get() != nullptr); 462 class_linker_ = runtime_->GetClassLinker(); 463 464 gc::Heap* heap = Runtime::Current()->GetHeap(); 465 ASSERT_TRUE(heap->HasBootImageSpace()); 466 ASSERT_TRUE(heap->GetNonMovingSpace()->IsMallocSpace()); 467 468 // We loaded the runtime with an explicit image, so it must exist. 469 ASSERT_EQ(heap->GetBootImageSpaces().size(), image_file_sizes.size()); 470 const HashSet<std::string>& image_classes = compiler_options_->GetImageClasses(); 471 for (size_t i = 0; i < helper.dex_file_locations.size(); ++i) { 472 std::unique_ptr<const DexFile> dex( 473 LoadExpectSingleDexFile(helper.dex_file_locations[i].c_str())); 474 ASSERT_TRUE(dex != nullptr); 475 uint64_t image_file_size = image_file_sizes[i]; 476 gc::space::ImageSpace* image_space = heap->GetBootImageSpaces()[i]; 477 ASSERT_TRUE(image_space != nullptr); 478 if (storage_mode == ImageHeader::kStorageModeUncompressed) { 479 // Uncompressed, image should be smaller than file. 480 ASSERT_LE(image_space->GetImageHeader().GetImageSize(), image_file_size); 481 } else if (image_file_size > 16 * KB) { 482 // Compressed, file should be smaller than image. Not really valid for small images. 483 ASSERT_LE(image_file_size, image_space->GetImageHeader().GetImageSize()); 484 // TODO: Actually validate the blocks, this is hard since the blocks are not copied over for 485 // compressed images. Add kPageSize since image_size is rounded up to this. 486 ASSERT_GT(image_space->GetImageHeader().GetBlockCount() * max_image_block_size, 487 image_space->GetImageHeader().GetImageSize() - kPageSize); 488 } 489 490 image_space->VerifyImageAllocations(); 491 uint8_t* image_begin = image_space->Begin(); 492 uint8_t* image_end = image_space->End(); 493 if (i == 0) { 494 // This check is only valid for image 0. 495 CHECK_EQ(kRequestedImageBase, reinterpret_cast<uintptr_t>(image_begin)); 496 } 497 for (size_t j = 0; j < dex->NumClassDefs(); ++j) { 498 const dex::ClassDef& class_def = dex->GetClassDef(j); 499 const char* descriptor = dex->GetClassDescriptor(class_def); 500 ObjPtr<mirror::Class> klass = class_linker_->FindSystemClass(soa.Self(), descriptor); 501 EXPECT_TRUE(klass != nullptr) << descriptor; 502 uint8_t* raw_klass = reinterpret_cast<uint8_t*>(klass.Ptr()); 503 if (image_classes.find(std::string_view(descriptor)) == image_classes.end()) { 504 EXPECT_TRUE(raw_klass >= image_end || raw_klass < image_begin) << descriptor; 505 } else { 506 // Image classes should be located inside the image. 507 EXPECT_LT(image_begin, raw_klass) << descriptor; 508 EXPECT_LT(raw_klass, image_end) << descriptor; 509 } 510 EXPECT_TRUE(Monitor::IsValidLockWord(klass->GetLockWord(false))); 511 } 512 } 513 } 514 515 } // namespace linker 516 } // namespace art 517 518 #endif // ART_DEX2OAT_LINKER_IMAGE_TEST_H_ 519
