1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "jit_code_cache.h" 18 19 #include <sstream> 20 21 #include "arch/context.h" 22 #include "art_method-inl.h" 23 #include "base/enums.h" 24 #include "base/stl_util.h" 25 #include "base/systrace.h" 26 #include "base/time_utils.h" 27 #include "cha.h" 28 #include "debugger_interface.h" 29 #include "entrypoints/runtime_asm_entrypoints.h" 30 #include "gc/accounting/bitmap-inl.h" 31 #include "gc/scoped_gc_critical_section.h" 32 #include "intern_table.h" 33 #include "jit/jit.h" 34 #include "jit/profiling_info.h" 35 #include "linear_alloc.h" 36 #include "mem_map.h" 37 #include "oat_file-inl.h" 38 #include "oat_quick_method_header.h" 39 #include "object_callbacks.h" 40 #include "scoped_thread_state_change-inl.h" 41 #include "stack.h" 42 #include "thread_list.h" 43 44 namespace art { 45 namespace jit { 46 47 static constexpr int kProtAll = PROT_READ | PROT_WRITE | PROT_EXEC; 48 static constexpr int kProtData = PROT_READ | PROT_WRITE; 49 static constexpr int kProtCode = PROT_READ | PROT_EXEC; 50 51 static constexpr size_t kCodeSizeLogThreshold = 50 * KB; 52 static constexpr size_t kStackMapSizeLogThreshold = 50 * KB; 53 54 #define CHECKED_MPROTECT(memory, size, prot) \ 55 do { \ 56 int rc = mprotect(memory, size, prot); \ 57 if (UNLIKELY(rc != 0)) { \ 58 errno = rc; \ 59 PLOG(FATAL) << "Failed to mprotect jit code cache"; \ 60 } \ 61 } while (false) \ 62 63 JitCodeCache* JitCodeCache::Create(size_t initial_capacity, 64 size_t max_capacity, 65 bool generate_debug_info, 66 std::string* error_msg) { 67 ScopedTrace trace(__PRETTY_FUNCTION__); 68 CHECK_GE(max_capacity, initial_capacity); 69 70 // Generating debug information is for using the Linux perf tool on 71 // host which does not work with ashmem. 72 // Also, target linux does not support ashmem. 73 bool use_ashmem = !generate_debug_info && !kIsTargetLinux; 74 75 // With 'perf', we want a 1-1 mapping between an address and a method. 76 bool garbage_collect_code = !generate_debug_info; 77 78 // We need to have 32 bit offsets from method headers in code cache which point to things 79 // in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work. 80 // Ensure we're below 1 GB to be safe. 81 if (max_capacity > 1 * GB) { 82 std::ostringstream oss; 83 oss << "Maxium code cache capacity is limited to 1 GB, " 84 << PrettySize(max_capacity) << " is too big"; 85 *error_msg = oss.str(); 86 return nullptr; 87 } 88 89 std::string error_str; 90 // Map name specific for android_os_Debug.cpp accounting. 91 // Map in low 4gb to simplify accessing root tables for x86_64. 92 // We could do PC-relative addressing to avoid this problem, but that 93 // would require reserving code and data area before submitting, which 94 // means more windows for the code memory to be RWX. 95 std::unique_ptr<MemMap> data_map(MemMap::MapAnonymous( 96 "data-code-cache", nullptr, 97 max_capacity, 98 kProtData, 99 /* low_4gb */ true, 100 /* reuse */ false, 101 &error_str, 102 use_ashmem)); 103 if (data_map == nullptr) { 104 std::ostringstream oss; 105 oss << "Failed to create read write cache: " << error_str << " size=" << max_capacity; 106 *error_msg = oss.str(); 107 return nullptr; 108 } 109 110 // Align both capacities to page size, as that's the unit mspaces use. 111 initial_capacity = RoundDown(initial_capacity, 2 * kPageSize); 112 max_capacity = RoundDown(max_capacity, 2 * kPageSize); 113 114 // Data cache is 1 / 2 of the map. 115 // TODO: Make this variable? 116 size_t data_size = max_capacity / 2; 117 size_t code_size = max_capacity - data_size; 118 DCHECK_EQ(code_size + data_size, max_capacity); 119 uint8_t* divider = data_map->Begin() + data_size; 120 121 MemMap* code_map = 122 data_map->RemapAtEnd(divider, "jit-code-cache", kProtAll, &error_str, use_ashmem); 123 if (code_map == nullptr) { 124 std::ostringstream oss; 125 oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity; 126 *error_msg = oss.str(); 127 return nullptr; 128 } 129 DCHECK_EQ(code_map->Begin(), divider); 130 data_size = initial_capacity / 2; 131 code_size = initial_capacity - data_size; 132 DCHECK_EQ(code_size + data_size, initial_capacity); 133 return new JitCodeCache( 134 code_map, data_map.release(), code_size, data_size, max_capacity, garbage_collect_code); 135 } 136 137 JitCodeCache::JitCodeCache(MemMap* code_map, 138 MemMap* data_map, 139 size_t initial_code_capacity, 140 size_t initial_data_capacity, 141 size_t max_capacity, 142 bool garbage_collect_code) 143 : lock_("Jit code cache", kJitCodeCacheLock), 144 lock_cond_("Jit code cache condition variable", lock_), 145 collection_in_progress_(false), 146 code_map_(code_map), 147 data_map_(data_map), 148 max_capacity_(max_capacity), 149 current_capacity_(initial_code_capacity + initial_data_capacity), 150 code_end_(initial_code_capacity), 151 data_end_(initial_data_capacity), 152 last_collection_increased_code_cache_(false), 153 last_update_time_ns_(0), 154 garbage_collect_code_(garbage_collect_code), 155 used_memory_for_data_(0), 156 used_memory_for_code_(0), 157 number_of_compilations_(0), 158 number_of_osr_compilations_(0), 159 number_of_collections_(0), 160 histogram_stack_map_memory_use_("Memory used for stack maps", 16), 161 histogram_code_memory_use_("Memory used for compiled code", 16), 162 histogram_profiling_info_memory_use_("Memory used for profiling info", 16), 163 is_weak_access_enabled_(true), 164 inline_cache_cond_("Jit inline cache condition variable", lock_) { 165 166 DCHECK_GE(max_capacity, initial_code_capacity + initial_data_capacity); 167 code_mspace_ = create_mspace_with_base(code_map_->Begin(), code_end_, false /*locked*/); 168 data_mspace_ = create_mspace_with_base(data_map_->Begin(), data_end_, false /*locked*/); 169 170 if (code_mspace_ == nullptr || data_mspace_ == nullptr) { 171 PLOG(FATAL) << "create_mspace_with_base failed"; 172 } 173 174 SetFootprintLimit(current_capacity_); 175 176 CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtCode); 177 CHECKED_MPROTECT(data_map_->Begin(), data_map_->Size(), kProtData); 178 179 VLOG(jit) << "Created jit code cache: initial data size=" 180 << PrettySize(initial_data_capacity) 181 << ", initial code size=" 182 << PrettySize(initial_code_capacity); 183 } 184 185 bool JitCodeCache::ContainsPc(const void* ptr) const { 186 return code_map_->Begin() <= ptr && ptr < code_map_->End(); 187 } 188 189 bool JitCodeCache::ContainsMethod(ArtMethod* method) { 190 MutexLock mu(Thread::Current(), lock_); 191 for (auto& it : method_code_map_) { 192 if (it.second == method) { 193 return true; 194 } 195 } 196 return false; 197 } 198 199 class ScopedCodeCacheWrite : ScopedTrace { 200 public: 201 explicit ScopedCodeCacheWrite(MemMap* code_map, bool only_for_tlb_shootdown = false) 202 : ScopedTrace("ScopedCodeCacheWrite"), 203 code_map_(code_map), 204 only_for_tlb_shootdown_(only_for_tlb_shootdown) { 205 ScopedTrace trace("mprotect all"); 206 CHECKED_MPROTECT( 207 code_map_->Begin(), only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), kProtAll); 208 } 209 ~ScopedCodeCacheWrite() { 210 ScopedTrace trace("mprotect code"); 211 CHECKED_MPROTECT( 212 code_map_->Begin(), only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), kProtCode); 213 } 214 private: 215 MemMap* const code_map_; 216 217 // If we're using ScopedCacheWrite only for TLB shootdown, we limit the scope of mprotect to 218 // one page. 219 const bool only_for_tlb_shootdown_; 220 221 DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite); 222 }; 223 224 uint8_t* JitCodeCache::CommitCode(Thread* self, 225 ArtMethod* method, 226 uint8_t* stack_map, 227 uint8_t* method_info, 228 uint8_t* roots_data, 229 size_t frame_size_in_bytes, 230 size_t core_spill_mask, 231 size_t fp_spill_mask, 232 const uint8_t* code, 233 size_t code_size, 234 size_t data_size, 235 bool osr, 236 Handle<mirror::ObjectArray<mirror::Object>> roots, 237 bool has_should_deoptimize_flag, 238 const ArenaSet<ArtMethod*>& cha_single_implementation_list) { 239 uint8_t* result = CommitCodeInternal(self, 240 method, 241 stack_map, 242 method_info, 243 roots_data, 244 frame_size_in_bytes, 245 core_spill_mask, 246 fp_spill_mask, 247 code, 248 code_size, 249 data_size, 250 osr, 251 roots, 252 has_should_deoptimize_flag, 253 cha_single_implementation_list); 254 if (result == nullptr) { 255 // Retry. 256 GarbageCollectCache(self); 257 result = CommitCodeInternal(self, 258 method, 259 stack_map, 260 method_info, 261 roots_data, 262 frame_size_in_bytes, 263 core_spill_mask, 264 fp_spill_mask, 265 code, 266 code_size, 267 data_size, 268 osr, 269 roots, 270 has_should_deoptimize_flag, 271 cha_single_implementation_list); 272 } 273 return result; 274 } 275 276 bool JitCodeCache::WaitForPotentialCollectionToComplete(Thread* self) { 277 bool in_collection = false; 278 while (collection_in_progress_) { 279 in_collection = true; 280 lock_cond_.Wait(self); 281 } 282 return in_collection; 283 } 284 285 static uintptr_t FromCodeToAllocation(const void* code) { 286 size_t alignment = GetInstructionSetAlignment(kRuntimeISA); 287 return reinterpret_cast<uintptr_t>(code) - RoundUp(sizeof(OatQuickMethodHeader), alignment); 288 } 289 290 static uint32_t ComputeRootTableSize(uint32_t number_of_roots) { 291 return sizeof(uint32_t) + number_of_roots * sizeof(GcRoot<mirror::Object>); 292 } 293 294 static uint32_t GetNumberOfRoots(const uint8_t* stack_map) { 295 // The length of the table is stored just before the stack map (and therefore at the end of 296 // the table itself), in order to be able to fetch it from a `stack_map` pointer. 297 return reinterpret_cast<const uint32_t*>(stack_map)[-1]; 298 } 299 300 static void FillRootTableLength(uint8_t* roots_data, uint32_t length) { 301 // Store the length of the table at the end. This will allow fetching it from a `stack_map` 302 // pointer. 303 reinterpret_cast<uint32_t*>(roots_data)[length] = length; 304 } 305 306 static const uint8_t* FromStackMapToRoots(const uint8_t* stack_map_data) { 307 return stack_map_data - ComputeRootTableSize(GetNumberOfRoots(stack_map_data)); 308 } 309 310 static void FillRootTable(uint8_t* roots_data, Handle<mirror::ObjectArray<mirror::Object>> roots) 311 REQUIRES_SHARED(Locks::mutator_lock_) { 312 GcRoot<mirror::Object>* gc_roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); 313 const uint32_t length = roots->GetLength(); 314 // Put all roots in `roots_data`. 315 for (uint32_t i = 0; i < length; ++i) { 316 ObjPtr<mirror::Object> object = roots->Get(i); 317 if (kIsDebugBuild) { 318 // Ensure the string is strongly interned. b/32995596 319 if (object->IsString()) { 320 ObjPtr<mirror::String> str = reinterpret_cast<mirror::String*>(object.Ptr()); 321 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 322 CHECK(class_linker->GetInternTable()->LookupStrong(Thread::Current(), str) != nullptr); 323 } 324 } 325 gc_roots[i] = GcRoot<mirror::Object>(object); 326 } 327 } 328 329 static uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr) { 330 OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 331 uint8_t* data = method_header->GetOptimizedCodeInfoPtr(); 332 uint32_t roots = GetNumberOfRoots(data); 333 if (number_of_roots != nullptr) { 334 *number_of_roots = roots; 335 } 336 return data - ComputeRootTableSize(roots); 337 } 338 339 // Use a sentinel for marking entries in the JIT table that have been cleared. 340 // This helps diagnosing in case the compiled code tries to wrongly access such 341 // entries. 342 static mirror::Class* const weak_sentinel = 343 reinterpret_cast<mirror::Class*>(Context::kBadGprBase + 0xff); 344 345 // Helper for the GC to process a weak class in a JIT root table. 346 static inline void ProcessWeakClass(GcRoot<mirror::Class>* root_ptr, 347 IsMarkedVisitor* visitor, 348 mirror::Class* update) 349 REQUIRES_SHARED(Locks::mutator_lock_) { 350 // This does not need a read barrier because this is called by GC. 351 mirror::Class* cls = root_ptr->Read<kWithoutReadBarrier>(); 352 if (cls != nullptr && cls != weak_sentinel) { 353 DCHECK((cls->IsClass<kDefaultVerifyFlags, kWithoutReadBarrier>())); 354 // Look at the classloader of the class to know if it has been unloaded. 355 // This does not need a read barrier because this is called by GC. 356 mirror::Object* class_loader = 357 cls->GetClassLoader<kDefaultVerifyFlags, kWithoutReadBarrier>(); 358 if (class_loader == nullptr || visitor->IsMarked(class_loader) != nullptr) { 359 // The class loader is live, update the entry if the class has moved. 360 mirror::Class* new_cls = down_cast<mirror::Class*>(visitor->IsMarked(cls)); 361 // Note that new_object can be null for CMS and newly allocated objects. 362 if (new_cls != nullptr && new_cls != cls) { 363 *root_ptr = GcRoot<mirror::Class>(new_cls); 364 } 365 } else { 366 // The class loader is not live, clear the entry. 367 *root_ptr = GcRoot<mirror::Class>(update); 368 } 369 } 370 } 371 372 void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) { 373 MutexLock mu(Thread::Current(), lock_); 374 for (const auto& entry : method_code_map_) { 375 uint32_t number_of_roots = 0; 376 uint8_t* roots_data = GetRootTable(entry.first, &number_of_roots); 377 GcRoot<mirror::Object>* roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); 378 for (uint32_t i = 0; i < number_of_roots; ++i) { 379 // This does not need a read barrier because this is called by GC. 380 mirror::Object* object = roots[i].Read<kWithoutReadBarrier>(); 381 if (object == nullptr || object == weak_sentinel) { 382 // entry got deleted in a previous sweep. 383 } else if (object->IsString<kDefaultVerifyFlags, kWithoutReadBarrier>()) { 384 mirror::Object* new_object = visitor->IsMarked(object); 385 // We know the string is marked because it's a strongly-interned string that 386 // is always alive. The IsMarked implementation of the CMS collector returns 387 // null for newly allocated objects, but we know those haven't moved. Therefore, 388 // only update the entry if we get a different non-null string. 389 // TODO: Do not use IsMarked for j.l.Class, and adjust once we move this method 390 // out of the weak access/creation pause. b/32167580 391 if (new_object != nullptr && new_object != object) { 392 DCHECK(new_object->IsString()); 393 roots[i] = GcRoot<mirror::Object>(new_object); 394 } 395 } else { 396 ProcessWeakClass( 397 reinterpret_cast<GcRoot<mirror::Class>*>(&roots[i]), visitor, weak_sentinel); 398 } 399 } 400 } 401 // Walk over inline caches to clear entries containing unloaded classes. 402 for (ProfilingInfo* info : profiling_infos_) { 403 for (size_t i = 0; i < info->number_of_inline_caches_; ++i) { 404 InlineCache* cache = &info->cache_[i]; 405 for (size_t j = 0; j < InlineCache::kIndividualCacheSize; ++j) { 406 ProcessWeakClass(&cache->classes_[j], visitor, nullptr); 407 } 408 } 409 } 410 } 411 412 void JitCodeCache::FreeCode(const void* code_ptr) { 413 uintptr_t allocation = FromCodeToAllocation(code_ptr); 414 // Notify native debugger that we are about to remove the code. 415 // It does nothing if we are not using native debugger. 416 DeleteJITCodeEntryForAddress(reinterpret_cast<uintptr_t>(code_ptr)); 417 FreeData(GetRootTable(code_ptr)); 418 FreeCode(reinterpret_cast<uint8_t*>(allocation)); 419 } 420 421 void JitCodeCache::FreeAllMethodHeaders( 422 const std::unordered_set<OatQuickMethodHeader*>& method_headers) { 423 { 424 MutexLock mu(Thread::Current(), *Locks::cha_lock_); 425 Runtime::Current()->GetClassLinker()->GetClassHierarchyAnalysis() 426 ->RemoveDependentsWithMethodHeaders(method_headers); 427 } 428 429 // We need to remove entries in method_headers from CHA dependencies 430 // first since once we do FreeCode() below, the memory can be reused 431 // so it's possible for the same method_header to start representing 432 // different compile code. 433 MutexLock mu(Thread::Current(), lock_); 434 ScopedCodeCacheWrite scc(code_map_.get()); 435 for (const OatQuickMethodHeader* method_header : method_headers) { 436 FreeCode(method_header->GetCode()); 437 } 438 } 439 440 void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) { 441 ScopedTrace trace(__PRETTY_FUNCTION__); 442 // We use a set to first collect all method_headers whose code need to be 443 // removed. We need to free the underlying code after we remove CHA dependencies 444 // for entries in this set. And it's more efficient to iterate through 445 // the CHA dependency map just once with an unordered_set. 446 std::unordered_set<OatQuickMethodHeader*> method_headers; 447 { 448 MutexLock mu(self, lock_); 449 // We do not check if a code cache GC is in progress, as this method comes 450 // with the classlinker_classes_lock_ held, and suspending ourselves could 451 // lead to a deadlock. 452 { 453 ScopedCodeCacheWrite scc(code_map_.get()); 454 for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { 455 if (alloc.ContainsUnsafe(it->second)) { 456 method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first)); 457 it = method_code_map_.erase(it); 458 } else { 459 ++it; 460 } 461 } 462 } 463 for (auto it = osr_code_map_.begin(); it != osr_code_map_.end();) { 464 if (alloc.ContainsUnsafe(it->first)) { 465 // Note that the code has already been pushed to method_headers in the loop 466 // above and is going to be removed in FreeCode() below. 467 it = osr_code_map_.erase(it); 468 } else { 469 ++it; 470 } 471 } 472 for (auto it = profiling_infos_.begin(); it != profiling_infos_.end();) { 473 ProfilingInfo* info = *it; 474 if (alloc.ContainsUnsafe(info->GetMethod())) { 475 info->GetMethod()->SetProfilingInfo(nullptr); 476 FreeData(reinterpret_cast<uint8_t*>(info)); 477 it = profiling_infos_.erase(it); 478 } else { 479 ++it; 480 } 481 } 482 } 483 FreeAllMethodHeaders(method_headers); 484 } 485 486 bool JitCodeCache::IsWeakAccessEnabled(Thread* self) const { 487 return kUseReadBarrier 488 ? self->GetWeakRefAccessEnabled() 489 : is_weak_access_enabled_.LoadSequentiallyConsistent(); 490 } 491 492 void JitCodeCache::WaitUntilInlineCacheAccessible(Thread* self) { 493 if (IsWeakAccessEnabled(self)) { 494 return; 495 } 496 ScopedThreadSuspension sts(self, kWaitingWeakGcRootRead); 497 MutexLock mu(self, lock_); 498 while (!IsWeakAccessEnabled(self)) { 499 inline_cache_cond_.Wait(self); 500 } 501 } 502 503 void JitCodeCache::BroadcastForInlineCacheAccess() { 504 Thread* self = Thread::Current(); 505 MutexLock mu(self, lock_); 506 inline_cache_cond_.Broadcast(self); 507 } 508 509 void JitCodeCache::AllowInlineCacheAccess() { 510 DCHECK(!kUseReadBarrier); 511 is_weak_access_enabled_.StoreSequentiallyConsistent(true); 512 BroadcastForInlineCacheAccess(); 513 } 514 515 void JitCodeCache::DisallowInlineCacheAccess() { 516 DCHECK(!kUseReadBarrier); 517 is_weak_access_enabled_.StoreSequentiallyConsistent(false); 518 } 519 520 void JitCodeCache::CopyInlineCacheInto(const InlineCache& ic, 521 Handle<mirror::ObjectArray<mirror::Class>> array) { 522 WaitUntilInlineCacheAccessible(Thread::Current()); 523 // Note that we don't need to lock `lock_` here, the compiler calling 524 // this method has already ensured the inline cache will not be deleted. 525 for (size_t in_cache = 0, in_array = 0; 526 in_cache < InlineCache::kIndividualCacheSize; 527 ++in_cache) { 528 mirror::Class* object = ic.classes_[in_cache].Read(); 529 if (object != nullptr) { 530 array->Set(in_array++, object); 531 } 532 } 533 } 534 535 static void ClearMethodCounter(ArtMethod* method, bool was_warm) { 536 if (was_warm) { 537 method->AddAccessFlags(kAccPreviouslyWarm); 538 } 539 // We reset the counter to 1 so that the profile knows that the method was executed at least once. 540 // This is required for layout purposes. 541 // We also need to make sure we'll pass the warmup threshold again, so we set to 0 if 542 // the warmup threshold is 1. 543 uint16_t jit_warmup_threshold = Runtime::Current()->GetJITOptions()->GetWarmupThreshold(); 544 method->SetCounter(std::min(jit_warmup_threshold - 1, 1)); 545 } 546 547 uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, 548 ArtMethod* method, 549 uint8_t* stack_map, 550 uint8_t* method_info, 551 uint8_t* roots_data, 552 size_t frame_size_in_bytes, 553 size_t core_spill_mask, 554 size_t fp_spill_mask, 555 const uint8_t* code, 556 size_t code_size, 557 size_t data_size, 558 bool osr, 559 Handle<mirror::ObjectArray<mirror::Object>> roots, 560 bool has_should_deoptimize_flag, 561 const ArenaSet<ArtMethod*>& 562 cha_single_implementation_list) { 563 DCHECK(stack_map != nullptr); 564 size_t alignment = GetInstructionSetAlignment(kRuntimeISA); 565 // Ensure the header ends up at expected instruction alignment. 566 size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); 567 size_t total_size = header_size + code_size; 568 569 OatQuickMethodHeader* method_header = nullptr; 570 uint8_t* code_ptr = nullptr; 571 uint8_t* memory = nullptr; 572 { 573 ScopedThreadSuspension sts(self, kSuspended); 574 MutexLock mu(self, lock_); 575 WaitForPotentialCollectionToComplete(self); 576 { 577 ScopedCodeCacheWrite scc(code_map_.get()); 578 memory = AllocateCode(total_size); 579 if (memory == nullptr) { 580 return nullptr; 581 } 582 code_ptr = memory + header_size; 583 584 std::copy(code, code + code_size, code_ptr); 585 method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 586 new (method_header) OatQuickMethodHeader( 587 code_ptr - stack_map, 588 code_ptr - method_info, 589 frame_size_in_bytes, 590 core_spill_mask, 591 fp_spill_mask, 592 code_size); 593 // Flush caches before we remove write permission because some ARMv8 Qualcomm kernels may 594 // trigger a segfault if a page fault occurs when requesting a cache maintenance operation. 595 // This is a kernel bug that we need to work around until affected devices (e.g. Nexus 5X and 596 // 6P) stop being supported or their kernels are fixed. 597 // 598 // For reference, this behavior is caused by this commit: 599 // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c 600 FlushInstructionCache(reinterpret_cast<char*>(code_ptr), 601 reinterpret_cast<char*>(code_ptr + code_size)); 602 DCHECK(!Runtime::Current()->IsAotCompiler()); 603 if (has_should_deoptimize_flag) { 604 method_header->SetHasShouldDeoptimizeFlag(); 605 } 606 } 607 608 number_of_compilations_++; 609 } 610 // We need to update the entry point in the runnable state for the instrumentation. 611 { 612 // Need cha_lock_ for checking all single-implementation flags and register 613 // dependencies. 614 MutexLock cha_mu(self, *Locks::cha_lock_); 615 bool single_impl_still_valid = true; 616 for (ArtMethod* single_impl : cha_single_implementation_list) { 617 if (!single_impl->HasSingleImplementation()) { 618 // Simply discard the compiled code. Clear the counter so that it may be recompiled later. 619 // Hopefully the class hierarchy will be more stable when compilation is retried. 620 single_impl_still_valid = false; 621 ClearMethodCounter(method, /*was_warm*/ false); 622 break; 623 } 624 } 625 626 // Discard the code if any single-implementation assumptions are now invalid. 627 if (!single_impl_still_valid) { 628 VLOG(jit) << "JIT discarded jitted code due to invalid single-implementation assumptions."; 629 return nullptr; 630 } 631 DCHECK(cha_single_implementation_list.empty() || !Runtime::Current()->IsJavaDebuggable()) 632 << "Should not be using cha on debuggable apps/runs!"; 633 634 for (ArtMethod* single_impl : cha_single_implementation_list) { 635 Runtime::Current()->GetClassLinker()->GetClassHierarchyAnalysis()->AddDependency( 636 single_impl, method, method_header); 637 } 638 639 // The following needs to be guarded by cha_lock_ also. Otherwise it's 640 // possible that the compiled code is considered invalidated by some class linking, 641 // but below we still make the compiled code valid for the method. 642 MutexLock mu(self, lock_); 643 // Fill the root table before updating the entry point. 644 DCHECK_EQ(FromStackMapToRoots(stack_map), roots_data); 645 DCHECK_LE(roots_data, stack_map); 646 FillRootTable(roots_data, roots); 647 { 648 // Flush data cache, as compiled code references literals in it. 649 // We also need a TLB shootdown to act as memory barrier across cores. 650 ScopedCodeCacheWrite ccw(code_map_.get(), /* only_for_tlb_shootdown */ true); 651 FlushDataCache(reinterpret_cast<char*>(roots_data), 652 reinterpret_cast<char*>(roots_data + data_size)); 653 } 654 method_code_map_.Put(code_ptr, method); 655 if (osr) { 656 number_of_osr_compilations_++; 657 osr_code_map_.Put(method, code_ptr); 658 } else { 659 Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( 660 method, method_header->GetEntryPoint()); 661 } 662 if (collection_in_progress_) { 663 // We need to update the live bitmap if there is a GC to ensure it sees this new 664 // code. 665 GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr)); 666 } 667 last_update_time_ns_.StoreRelease(NanoTime()); 668 VLOG(jit) 669 << "JIT added (osr=" << std::boolalpha << osr << std::noboolalpha << ") " 670 << ArtMethod::PrettyMethod(method) << "@" << method 671 << " ccache_size=" << PrettySize(CodeCacheSizeLocked()) << ": " 672 << " dcache_size=" << PrettySize(DataCacheSizeLocked()) << ": " 673 << reinterpret_cast<const void*>(method_header->GetEntryPoint()) << "," 674 << reinterpret_cast<const void*>(method_header->GetEntryPoint() + 675 method_header->GetCodeSize()); 676 histogram_code_memory_use_.AddValue(code_size); 677 if (code_size > kCodeSizeLogThreshold) { 678 LOG(INFO) << "JIT allocated " 679 << PrettySize(code_size) 680 << " for compiled code of " 681 << ArtMethod::PrettyMethod(method); 682 } 683 } 684 685 return reinterpret_cast<uint8_t*>(method_header); 686 } 687 688 size_t JitCodeCache::CodeCacheSize() { 689 MutexLock mu(Thread::Current(), lock_); 690 return CodeCacheSizeLocked(); 691 } 692 693 bool JitCodeCache::RemoveMethod(ArtMethod* method, bool release_memory) { 694 MutexLock mu(Thread::Current(), lock_); 695 if (method->IsNative()) { 696 return false; 697 } 698 699 bool in_cache = false; 700 { 701 ScopedCodeCacheWrite ccw(code_map_.get()); 702 for (auto code_iter = method_code_map_.begin(); code_iter != method_code_map_.end();) { 703 if (code_iter->second == method) { 704 if (release_memory) { 705 FreeCode(code_iter->first); 706 } 707 code_iter = method_code_map_.erase(code_iter); 708 in_cache = true; 709 continue; 710 } 711 ++code_iter; 712 } 713 } 714 715 bool osr = false; 716 auto code_map = osr_code_map_.find(method); 717 if (code_map != osr_code_map_.end()) { 718 osr_code_map_.erase(code_map); 719 osr = true; 720 } 721 722 if (!in_cache) { 723 return false; 724 } 725 726 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 727 if (info != nullptr) { 728 auto profile = std::find(profiling_infos_.begin(), profiling_infos_.end(), info); 729 DCHECK(profile != profiling_infos_.end()); 730 profiling_infos_.erase(profile); 731 } 732 method->SetProfilingInfo(nullptr); 733 method->ClearCounter(); 734 Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( 735 method, GetQuickToInterpreterBridge()); 736 VLOG(jit) 737 << "JIT removed (osr=" << std::boolalpha << osr << std::noboolalpha << ") " 738 << ArtMethod::PrettyMethod(method) << "@" << method 739 << " ccache_size=" << PrettySize(CodeCacheSizeLocked()) << ": " 740 << " dcache_size=" << PrettySize(DataCacheSizeLocked()); 741 return true; 742 } 743 744 // This notifies the code cache that the given method has been redefined and that it should remove 745 // any cached information it has on the method. All threads must be suspended before calling this 746 // method. The compiled code for the method (if there is any) must not be in any threads call stack. 747 void JitCodeCache::NotifyMethodRedefined(ArtMethod* method) { 748 MutexLock mu(Thread::Current(), lock_); 749 if (method->IsNative()) { 750 return; 751 } 752 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 753 if (info != nullptr) { 754 auto profile = std::find(profiling_infos_.begin(), profiling_infos_.end(), info); 755 DCHECK(profile != profiling_infos_.end()); 756 profiling_infos_.erase(profile); 757 } 758 method->SetProfilingInfo(nullptr); 759 ScopedCodeCacheWrite ccw(code_map_.get()); 760 for (auto code_iter = method_code_map_.begin(); code_iter != method_code_map_.end();) { 761 if (code_iter->second == method) { 762 FreeCode(code_iter->first); 763 code_iter = method_code_map_.erase(code_iter); 764 continue; 765 } 766 ++code_iter; 767 } 768 auto code_map = osr_code_map_.find(method); 769 if (code_map != osr_code_map_.end()) { 770 osr_code_map_.erase(code_map); 771 } 772 } 773 774 // This invalidates old_method. Once this function returns one can no longer use old_method to 775 // execute code unless it is fixed up. This fixup will happen later in the process of installing a 776 // class redefinition. 777 // TODO We should add some info to ArtMethod to note that 'old_method' has been invalidated and 778 // shouldn't be used since it is no longer logically in the jit code cache. 779 // TODO We should add DCHECKS that validate that the JIT is paused when this method is entered. 780 void JitCodeCache::MoveObsoleteMethod(ArtMethod* old_method, ArtMethod* new_method) { 781 // Native methods have no profiling info and need no special handling from the JIT code cache. 782 if (old_method->IsNative()) { 783 return; 784 } 785 MutexLock mu(Thread::Current(), lock_); 786 // Update ProfilingInfo to the new one and remove it from the old_method. 787 if (old_method->GetProfilingInfo(kRuntimePointerSize) != nullptr) { 788 DCHECK_EQ(old_method->GetProfilingInfo(kRuntimePointerSize)->GetMethod(), old_method); 789 ProfilingInfo* info = old_method->GetProfilingInfo(kRuntimePointerSize); 790 old_method->SetProfilingInfo(nullptr); 791 // Since the JIT should be paused and all threads suspended by the time this is called these 792 // checks should always pass. 793 DCHECK(!info->IsInUseByCompiler()); 794 new_method->SetProfilingInfo(info); 795 info->method_ = new_method; 796 } 797 // Update method_code_map_ to point to the new method. 798 for (auto& it : method_code_map_) { 799 if (it.second == old_method) { 800 it.second = new_method; 801 } 802 } 803 // Update osr_code_map_ to point to the new method. 804 auto code_map = osr_code_map_.find(old_method); 805 if (code_map != osr_code_map_.end()) { 806 osr_code_map_.Put(new_method, code_map->second); 807 osr_code_map_.erase(old_method); 808 } 809 } 810 811 size_t JitCodeCache::CodeCacheSizeLocked() { 812 return used_memory_for_code_; 813 } 814 815 size_t JitCodeCache::DataCacheSize() { 816 MutexLock mu(Thread::Current(), lock_); 817 return DataCacheSizeLocked(); 818 } 819 820 size_t JitCodeCache::DataCacheSizeLocked() { 821 return used_memory_for_data_; 822 } 823 824 void JitCodeCache::ClearData(Thread* self, 825 uint8_t* stack_map_data, 826 uint8_t* roots_data) { 827 DCHECK_EQ(FromStackMapToRoots(stack_map_data), roots_data); 828 MutexLock mu(self, lock_); 829 FreeData(reinterpret_cast<uint8_t*>(roots_data)); 830 } 831 832 size_t JitCodeCache::ReserveData(Thread* self, 833 size_t stack_map_size, 834 size_t method_info_size, 835 size_t number_of_roots, 836 ArtMethod* method, 837 uint8_t** stack_map_data, 838 uint8_t** method_info_data, 839 uint8_t** roots_data) { 840 size_t table_size = ComputeRootTableSize(number_of_roots); 841 size_t size = RoundUp(stack_map_size + method_info_size + table_size, sizeof(void*)); 842 uint8_t* result = nullptr; 843 844 { 845 ScopedThreadSuspension sts(self, kSuspended); 846 MutexLock mu(self, lock_); 847 WaitForPotentialCollectionToComplete(self); 848 result = AllocateData(size); 849 } 850 851 if (result == nullptr) { 852 // Retry. 853 GarbageCollectCache(self); 854 ScopedThreadSuspension sts(self, kSuspended); 855 MutexLock mu(self, lock_); 856 WaitForPotentialCollectionToComplete(self); 857 result = AllocateData(size); 858 } 859 860 MutexLock mu(self, lock_); 861 histogram_stack_map_memory_use_.AddValue(size); 862 if (size > kStackMapSizeLogThreshold) { 863 LOG(INFO) << "JIT allocated " 864 << PrettySize(size) 865 << " for stack maps of " 866 << ArtMethod::PrettyMethod(method); 867 } 868 if (result != nullptr) { 869 *roots_data = result; 870 *stack_map_data = result + table_size; 871 *method_info_data = *stack_map_data + stack_map_size; 872 FillRootTableLength(*roots_data, number_of_roots); 873 return size; 874 } else { 875 *roots_data = nullptr; 876 *stack_map_data = nullptr; 877 *method_info_data = nullptr; 878 return 0; 879 } 880 } 881 882 class MarkCodeVisitor FINAL : public StackVisitor { 883 public: 884 MarkCodeVisitor(Thread* thread_in, JitCodeCache* code_cache_in) 885 : StackVisitor(thread_in, nullptr, StackVisitor::StackWalkKind::kSkipInlinedFrames), 886 code_cache_(code_cache_in), 887 bitmap_(code_cache_->GetLiveBitmap()) {} 888 889 bool VisitFrame() OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { 890 const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader(); 891 if (method_header == nullptr) { 892 return true; 893 } 894 const void* code = method_header->GetCode(); 895 if (code_cache_->ContainsPc(code)) { 896 // Use the atomic set version, as multiple threads are executing this code. 897 bitmap_->AtomicTestAndSet(FromCodeToAllocation(code)); 898 } 899 return true; 900 } 901 902 private: 903 JitCodeCache* const code_cache_; 904 CodeCacheBitmap* const bitmap_; 905 }; 906 907 class MarkCodeClosure FINAL : public Closure { 908 public: 909 MarkCodeClosure(JitCodeCache* code_cache, Barrier* barrier) 910 : code_cache_(code_cache), barrier_(barrier) {} 911 912 void Run(Thread* thread) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { 913 ScopedTrace trace(__PRETTY_FUNCTION__); 914 DCHECK(thread == Thread::Current() || thread->IsSuspended()); 915 MarkCodeVisitor visitor(thread, code_cache_); 916 visitor.WalkStack(); 917 if (kIsDebugBuild) { 918 // The stack walking code queries the side instrumentation stack if it 919 // sees an instrumentation exit pc, so the JIT code of methods in that stack 920 // must have been seen. We sanity check this below. 921 for (const instrumentation::InstrumentationStackFrame& frame 922 : *thread->GetInstrumentationStack()) { 923 // The 'method_' in InstrumentationStackFrame is the one that has return_pc_ in 924 // its stack frame, it is not the method owning return_pc_. We just pass null to 925 // LookupMethodHeader: the method is only checked against in debug builds. 926 OatQuickMethodHeader* method_header = 927 code_cache_->LookupMethodHeader(frame.return_pc_, nullptr); 928 if (method_header != nullptr) { 929 const void* code = method_header->GetCode(); 930 CHECK(code_cache_->GetLiveBitmap()->Test(FromCodeToAllocation(code))); 931 } 932 } 933 } 934 barrier_->Pass(Thread::Current()); 935 } 936 937 private: 938 JitCodeCache* const code_cache_; 939 Barrier* const barrier_; 940 }; 941 942 void JitCodeCache::NotifyCollectionDone(Thread* self) { 943 collection_in_progress_ = false; 944 lock_cond_.Broadcast(self); 945 } 946 947 void JitCodeCache::SetFootprintLimit(size_t new_footprint) { 948 size_t per_space_footprint = new_footprint / 2; 949 DCHECK(IsAlignedParam(per_space_footprint, kPageSize)); 950 DCHECK_EQ(per_space_footprint * 2, new_footprint); 951 mspace_set_footprint_limit(data_mspace_, per_space_footprint); 952 { 953 ScopedCodeCacheWrite scc(code_map_.get()); 954 mspace_set_footprint_limit(code_mspace_, per_space_footprint); 955 } 956 } 957 958 bool JitCodeCache::IncreaseCodeCacheCapacity() { 959 if (current_capacity_ == max_capacity_) { 960 return false; 961 } 962 963 // Double the capacity if we're below 1MB, or increase it by 1MB if 964 // we're above. 965 if (current_capacity_ < 1 * MB) { 966 current_capacity_ *= 2; 967 } else { 968 current_capacity_ += 1 * MB; 969 } 970 if (current_capacity_ > max_capacity_) { 971 current_capacity_ = max_capacity_; 972 } 973 974 if (!kIsDebugBuild || VLOG_IS_ON(jit)) { 975 LOG(INFO) << "Increasing code cache capacity to " << PrettySize(current_capacity_); 976 } 977 978 SetFootprintLimit(current_capacity_); 979 980 return true; 981 } 982 983 void JitCodeCache::MarkCompiledCodeOnThreadStacks(Thread* self) { 984 Barrier barrier(0); 985 size_t threads_running_checkpoint = 0; 986 MarkCodeClosure closure(this, &barrier); 987 threads_running_checkpoint = Runtime::Current()->GetThreadList()->RunCheckpoint(&closure); 988 // Now that we have run our checkpoint, move to a suspended state and wait 989 // for other threads to run the checkpoint. 990 ScopedThreadSuspension sts(self, kSuspended); 991 if (threads_running_checkpoint != 0) { 992 barrier.Increment(self, threads_running_checkpoint); 993 } 994 } 995 996 bool JitCodeCache::ShouldDoFullCollection() { 997 if (current_capacity_ == max_capacity_) { 998 // Always do a full collection when the code cache is full. 999 return true; 1000 } else if (current_capacity_ < kReservedCapacity) { 1001 // Always do partial collection when the code cache size is below the reserved 1002 // capacity. 1003 return false; 1004 } else if (last_collection_increased_code_cache_) { 1005 // This time do a full collection. 1006 return true; 1007 } else { 1008 // This time do a partial collection. 1009 return false; 1010 } 1011 } 1012 1013 void JitCodeCache::GarbageCollectCache(Thread* self) { 1014 ScopedTrace trace(__FUNCTION__); 1015 if (!garbage_collect_code_) { 1016 MutexLock mu(self, lock_); 1017 IncreaseCodeCacheCapacity(); 1018 return; 1019 } 1020 1021 // Wait for an existing collection, or let everyone know we are starting one. 1022 { 1023 ScopedThreadSuspension sts(self, kSuspended); 1024 MutexLock mu(self, lock_); 1025 if (WaitForPotentialCollectionToComplete(self)) { 1026 return; 1027 } else { 1028 number_of_collections_++; 1029 live_bitmap_.reset(CodeCacheBitmap::Create( 1030 "code-cache-bitmap", 1031 reinterpret_cast<uintptr_t>(code_map_->Begin()), 1032 reinterpret_cast<uintptr_t>(code_map_->Begin() + current_capacity_ / 2))); 1033 collection_in_progress_ = true; 1034 } 1035 } 1036 1037 TimingLogger logger("JIT code cache timing logger", true, VLOG_IS_ON(jit)); 1038 { 1039 TimingLogger::ScopedTiming st("Code cache collection", &logger); 1040 1041 bool do_full_collection = false; 1042 { 1043 MutexLock mu(self, lock_); 1044 do_full_collection = ShouldDoFullCollection(); 1045 } 1046 1047 if (!kIsDebugBuild || VLOG_IS_ON(jit)) { 1048 LOG(INFO) << "Do " 1049 << (do_full_collection ? "full" : "partial") 1050 << " code cache collection, code=" 1051 << PrettySize(CodeCacheSize()) 1052 << ", data=" << PrettySize(DataCacheSize()); 1053 } 1054 1055 DoCollection(self, /* collect_profiling_info */ do_full_collection); 1056 1057 if (!kIsDebugBuild || VLOG_IS_ON(jit)) { 1058 LOG(INFO) << "After code cache collection, code=" 1059 << PrettySize(CodeCacheSize()) 1060 << ", data=" << PrettySize(DataCacheSize()); 1061 } 1062 1063 { 1064 MutexLock mu(self, lock_); 1065 1066 // Increase the code cache only when we do partial collections. 1067 // TODO: base this strategy on how full the code cache is? 1068 if (do_full_collection) { 1069 last_collection_increased_code_cache_ = false; 1070 } else { 1071 last_collection_increased_code_cache_ = true; 1072 IncreaseCodeCacheCapacity(); 1073 } 1074 1075 bool next_collection_will_be_full = ShouldDoFullCollection(); 1076 1077 // Start polling the liveness of compiled code to prepare for the next full collection. 1078 if (next_collection_will_be_full) { 1079 // Save the entry point of methods we have compiled, and update the entry 1080 // point of those methods to the interpreter. If the method is invoked, the 1081 // interpreter will update its entry point to the compiled code and call it. 1082 for (ProfilingInfo* info : profiling_infos_) { 1083 const void* entry_point = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); 1084 if (ContainsPc(entry_point)) { 1085 info->SetSavedEntryPoint(entry_point); 1086 // Don't call Instrumentation::UpdateMethods, as it can check the declaring 1087 // class of the method. We may be concurrently running a GC which makes accessing 1088 // the class unsafe. We know it is OK to bypass the instrumentation as we've just 1089 // checked that the current entry point is JIT compiled code. 1090 info->GetMethod()->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge()); 1091 } 1092 } 1093 1094 DCHECK(CheckLiveCompiledCodeHasProfilingInfo()); 1095 } 1096 live_bitmap_.reset(nullptr); 1097 NotifyCollectionDone(self); 1098 } 1099 } 1100 Runtime::Current()->GetJit()->AddTimingLogger(logger); 1101 } 1102 1103 void JitCodeCache::RemoveUnmarkedCode(Thread* self) { 1104 ScopedTrace trace(__FUNCTION__); 1105 std::unordered_set<OatQuickMethodHeader*> method_headers; 1106 { 1107 MutexLock mu(self, lock_); 1108 ScopedCodeCacheWrite scc(code_map_.get()); 1109 // Iterate over all compiled code and remove entries that are not marked. 1110 for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { 1111 const void* code_ptr = it->first; 1112 uintptr_t allocation = FromCodeToAllocation(code_ptr); 1113 if (GetLiveBitmap()->Test(allocation)) { 1114 ++it; 1115 } else { 1116 method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first)); 1117 it = method_code_map_.erase(it); 1118 } 1119 } 1120 } 1121 FreeAllMethodHeaders(method_headers); 1122 } 1123 1124 void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { 1125 ScopedTrace trace(__FUNCTION__); 1126 { 1127 MutexLock mu(self, lock_); 1128 if (collect_profiling_info) { 1129 // Clear the profiling info of methods that do not have compiled code as entrypoint. 1130 // Also remove the saved entry point from the ProfilingInfo objects. 1131 for (ProfilingInfo* info : profiling_infos_) { 1132 const void* ptr = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); 1133 if (!ContainsPc(ptr) && !info->IsInUseByCompiler()) { 1134 info->GetMethod()->SetProfilingInfo(nullptr); 1135 } 1136 1137 if (info->GetSavedEntryPoint() != nullptr) { 1138 info->SetSavedEntryPoint(nullptr); 1139 // We are going to move this method back to interpreter. Clear the counter now to 1140 // give it a chance to be hot again. 1141 ClearMethodCounter(info->GetMethod(), /*was_warm*/ true); 1142 } 1143 } 1144 } else if (kIsDebugBuild) { 1145 // Sanity check that the profiling infos do not have a dangling entry point. 1146 for (ProfilingInfo* info : profiling_infos_) { 1147 DCHECK(info->GetSavedEntryPoint() == nullptr); 1148 } 1149 } 1150 1151 // Mark compiled code that are entrypoints of ArtMethods. Compiled code that is not 1152 // an entry point is either: 1153 // - an osr compiled code, that will be removed if not in a thread call stack. 1154 // - discarded compiled code, that will be removed if not in a thread call stack. 1155 for (const auto& it : method_code_map_) { 1156 ArtMethod* method = it.second; 1157 const void* code_ptr = it.first; 1158 const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1159 if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) { 1160 GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr)); 1161 } 1162 } 1163 1164 // Empty osr method map, as osr compiled code will be deleted (except the ones 1165 // on thread stacks). 1166 osr_code_map_.clear(); 1167 } 1168 1169 // Run a checkpoint on all threads to mark the JIT compiled code they are running. 1170 MarkCompiledCodeOnThreadStacks(self); 1171 1172 // At this point, mutator threads are still running, and entrypoints of methods can 1173 // change. We do know they cannot change to a code cache entry that is not marked, 1174 // therefore we can safely remove those entries. 1175 RemoveUnmarkedCode(self); 1176 1177 if (collect_profiling_info) { 1178 ScopedThreadSuspension sts(self, kSuspended); 1179 MutexLock mu(self, lock_); 1180 // Free all profiling infos of methods not compiled nor being compiled. 1181 auto profiling_kept_end = std::remove_if(profiling_infos_.begin(), profiling_infos_.end(), 1182 [this] (ProfilingInfo* info) NO_THREAD_SAFETY_ANALYSIS { 1183 const void* ptr = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); 1184 // We have previously cleared the ProfilingInfo pointer in the ArtMethod in the hope 1185 // that the compiled code would not get revived. As mutator threads run concurrently, 1186 // they may have revived the compiled code, and now we are in the situation where 1187 // a method has compiled code but no ProfilingInfo. 1188 // We make sure compiled methods have a ProfilingInfo object. It is needed for 1189 // code cache collection. 1190 if (ContainsPc(ptr) && 1191 info->GetMethod()->GetProfilingInfo(kRuntimePointerSize) == nullptr) { 1192 info->GetMethod()->SetProfilingInfo(info); 1193 } else if (info->GetMethod()->GetProfilingInfo(kRuntimePointerSize) != info) { 1194 // No need for this ProfilingInfo object anymore. 1195 FreeData(reinterpret_cast<uint8_t*>(info)); 1196 return true; 1197 } 1198 return false; 1199 }); 1200 profiling_infos_.erase(profiling_kept_end, profiling_infos_.end()); 1201 DCHECK(CheckLiveCompiledCodeHasProfilingInfo()); 1202 } 1203 } 1204 1205 bool JitCodeCache::CheckLiveCompiledCodeHasProfilingInfo() { 1206 ScopedTrace trace(__FUNCTION__); 1207 // Check that methods we have compiled do have a ProfilingInfo object. We would 1208 // have memory leaks of compiled code otherwise. 1209 for (const auto& it : method_code_map_) { 1210 ArtMethod* method = it.second; 1211 if (method->GetProfilingInfo(kRuntimePointerSize) == nullptr) { 1212 const void* code_ptr = it.first; 1213 const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1214 if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) { 1215 // If the code is not dead, then we have a problem. Note that this can even 1216 // happen just after a collection, as mutator threads are running in parallel 1217 // and could deoptimize an existing compiled code. 1218 return false; 1219 } 1220 } 1221 } 1222 return true; 1223 } 1224 1225 OatQuickMethodHeader* JitCodeCache::LookupMethodHeader(uintptr_t pc, ArtMethod* method) { 1226 static_assert(kRuntimeISA != kThumb2, "kThumb2 cannot be a runtime ISA"); 1227 if (kRuntimeISA == kArm) { 1228 // On Thumb-2, the pc is offset by one. 1229 --pc; 1230 } 1231 if (!ContainsPc(reinterpret_cast<const void*>(pc))) { 1232 return nullptr; 1233 } 1234 1235 MutexLock mu(Thread::Current(), lock_); 1236 if (method_code_map_.empty()) { 1237 return nullptr; 1238 } 1239 auto it = method_code_map_.lower_bound(reinterpret_cast<const void*>(pc)); 1240 --it; 1241 1242 const void* code_ptr = it->first; 1243 OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1244 if (!method_header->Contains(pc)) { 1245 return nullptr; 1246 } 1247 if (kIsDebugBuild && method != nullptr) { 1248 // When we are walking the stack to redefine classes and creating obsolete methods it is 1249 // possible that we might have updated the method_code_map by making this method obsolete in a 1250 // previous frame. Therefore we should just check that the non-obsolete version of this method 1251 // is the one we expect. We change to the non-obsolete versions in the error message since the 1252 // obsolete version of the method might not be fully initialized yet. This situation can only 1253 // occur when we are in the process of allocating and setting up obsolete methods. Otherwise 1254 // method and it->second should be identical. (See runtime/openjdkjvmti/ti_redefine.cc for more 1255 // information.) 1256 DCHECK_EQ(it->second->GetNonObsoleteMethod(), method->GetNonObsoleteMethod()) 1257 << ArtMethod::PrettyMethod(method->GetNonObsoleteMethod()) << " " 1258 << ArtMethod::PrettyMethod(it->second->GetNonObsoleteMethod()) << " " 1259 << std::hex << pc; 1260 } 1261 return method_header; 1262 } 1263 1264 OatQuickMethodHeader* JitCodeCache::LookupOsrMethodHeader(ArtMethod* method) { 1265 MutexLock mu(Thread::Current(), lock_); 1266 auto it = osr_code_map_.find(method); 1267 if (it == osr_code_map_.end()) { 1268 return nullptr; 1269 } 1270 return OatQuickMethodHeader::FromCodePointer(it->second); 1271 } 1272 1273 ProfilingInfo* JitCodeCache::AddProfilingInfo(Thread* self, 1274 ArtMethod* method, 1275 const std::vector<uint32_t>& entries, 1276 bool retry_allocation) 1277 // No thread safety analysis as we are using TryLock/Unlock explicitly. 1278 NO_THREAD_SAFETY_ANALYSIS { 1279 ProfilingInfo* info = nullptr; 1280 if (!retry_allocation) { 1281 // If we are allocating for the interpreter, just try to lock, to avoid 1282 // lock contention with the JIT. 1283 if (lock_.ExclusiveTryLock(self)) { 1284 info = AddProfilingInfoInternal(self, method, entries); 1285 lock_.ExclusiveUnlock(self); 1286 } 1287 } else { 1288 { 1289 MutexLock mu(self, lock_); 1290 info = AddProfilingInfoInternal(self, method, entries); 1291 } 1292 1293 if (info == nullptr) { 1294 GarbageCollectCache(self); 1295 MutexLock mu(self, lock_); 1296 info = AddProfilingInfoInternal(self, method, entries); 1297 } 1298 } 1299 return info; 1300 } 1301 1302 ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self ATTRIBUTE_UNUSED, 1303 ArtMethod* method, 1304 const std::vector<uint32_t>& entries) { 1305 size_t profile_info_size = RoundUp( 1306 sizeof(ProfilingInfo) + sizeof(InlineCache) * entries.size(), 1307 sizeof(void*)); 1308 1309 // Check whether some other thread has concurrently created it. 1310 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1311 if (info != nullptr) { 1312 return info; 1313 } 1314 1315 uint8_t* data = AllocateData(profile_info_size); 1316 if (data == nullptr) { 1317 return nullptr; 1318 } 1319 info = new (data) ProfilingInfo(method, entries); 1320 1321 // Make sure other threads see the data in the profiling info object before the 1322 // store in the ArtMethod's ProfilingInfo pointer. 1323 QuasiAtomic::ThreadFenceRelease(); 1324 1325 method->SetProfilingInfo(info); 1326 profiling_infos_.push_back(info); 1327 histogram_profiling_info_memory_use_.AddValue(profile_info_size); 1328 return info; 1329 } 1330 1331 // NO_THREAD_SAFETY_ANALYSIS as this is called from mspace code, at which point the lock 1332 // is already held. 1333 void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_SAFETY_ANALYSIS { 1334 if (code_mspace_ == mspace) { 1335 size_t result = code_end_; 1336 code_end_ += increment; 1337 return reinterpret_cast<void*>(result + code_map_->Begin()); 1338 } else { 1339 DCHECK_EQ(data_mspace_, mspace); 1340 size_t result = data_end_; 1341 data_end_ += increment; 1342 return reinterpret_cast<void*>(result + data_map_->Begin()); 1343 } 1344 } 1345 1346 void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_locations, 1347 std::vector<ProfileMethodInfo>& methods) { 1348 ScopedTrace trace(__FUNCTION__); 1349 MutexLock mu(Thread::Current(), lock_); 1350 uint16_t jit_compile_threshold = Runtime::Current()->GetJITOptions()->GetCompileThreshold(); 1351 for (const ProfilingInfo* info : profiling_infos_) { 1352 ArtMethod* method = info->GetMethod(); 1353 const DexFile* dex_file = method->GetDexFile(); 1354 if (!ContainsElement(dex_base_locations, dex_file->GetBaseLocation())) { 1355 // Skip dex files which are not profiled. 1356 continue; 1357 } 1358 std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches; 1359 1360 // If the method didn't reach the compilation threshold don't save the inline caches. 1361 // They might be incomplete and cause unnecessary deoptimizations. 1362 // If the inline cache is empty the compiler will generate a regular invoke virtual/interface. 1363 if (method->GetCounter() < jit_compile_threshold) { 1364 methods.emplace_back(/*ProfileMethodInfo*/ 1365 MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches); 1366 continue; 1367 } 1368 1369 for (size_t i = 0; i < info->number_of_inline_caches_; ++i) { 1370 std::vector<TypeReference> profile_classes; 1371 const InlineCache& cache = info->cache_[i]; 1372 ArtMethod* caller = info->GetMethod(); 1373 bool is_missing_types = false; 1374 for (size_t k = 0; k < InlineCache::kIndividualCacheSize; k++) { 1375 mirror::Class* cls = cache.classes_[k].Read(); 1376 if (cls == nullptr) { 1377 break; 1378 } 1379 1380 // Check if the receiver is in the boot class path or if it's in the 1381 // same class loader as the caller. If not, skip it, as there is not 1382 // much we can do during AOT. 1383 if (!cls->IsBootStrapClassLoaded() && 1384 caller->GetClassLoader() != cls->GetClassLoader()) { 1385 is_missing_types = true; 1386 continue; 1387 } 1388 1389 const DexFile* class_dex_file = nullptr; 1390 dex::TypeIndex type_index; 1391 1392 if (cls->GetDexCache() == nullptr) { 1393 DCHECK(cls->IsArrayClass()) << cls->PrettyClass(); 1394 // Make a best effort to find the type index in the method's dex file. 1395 // We could search all open dex files but that might turn expensive 1396 // and probably not worth it. 1397 class_dex_file = dex_file; 1398 type_index = cls->FindTypeIndexInOtherDexFile(*dex_file); 1399 } else { 1400 class_dex_file = &(cls->GetDexFile()); 1401 type_index = cls->GetDexTypeIndex(); 1402 } 1403 if (!type_index.IsValid()) { 1404 // Could be a proxy class or an array for which we couldn't find the type index. 1405 is_missing_types = true; 1406 continue; 1407 } 1408 if (ContainsElement(dex_base_locations, class_dex_file->GetBaseLocation())) { 1409 // Only consider classes from the same apk (including multidex). 1410 profile_classes.emplace_back(/*ProfileMethodInfo::ProfileClassReference*/ 1411 class_dex_file, type_index); 1412 } else { 1413 is_missing_types = true; 1414 } 1415 } 1416 if (!profile_classes.empty()) { 1417 inline_caches.emplace_back(/*ProfileMethodInfo::ProfileInlineCache*/ 1418 cache.dex_pc_, is_missing_types, profile_classes); 1419 } 1420 } 1421 methods.emplace_back(/*ProfileMethodInfo*/ 1422 MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches); 1423 } 1424 } 1425 1426 uint64_t JitCodeCache::GetLastUpdateTimeNs() const { 1427 return last_update_time_ns_.LoadAcquire(); 1428 } 1429 1430 bool JitCodeCache::IsOsrCompiled(ArtMethod* method) { 1431 MutexLock mu(Thread::Current(), lock_); 1432 return osr_code_map_.find(method) != osr_code_map_.end(); 1433 } 1434 1435 bool JitCodeCache::NotifyCompilationOf(ArtMethod* method, Thread* self, bool osr) { 1436 if (!osr && ContainsPc(method->GetEntryPointFromQuickCompiledCode())) { 1437 return false; 1438 } 1439 1440 MutexLock mu(self, lock_); 1441 if (osr && (osr_code_map_.find(method) != osr_code_map_.end())) { 1442 return false; 1443 } 1444 1445 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1446 if (info == nullptr) { 1447 VLOG(jit) << method->PrettyMethod() << " needs a ProfilingInfo to be compiled"; 1448 // Because the counter is not atomic, there are some rare cases where we may not hit the 1449 // threshold for creating the ProfilingInfo. Reset the counter now to "correct" this. 1450 ClearMethodCounter(method, /*was_warm*/ false); 1451 return false; 1452 } 1453 1454 if (info->IsMethodBeingCompiled(osr)) { 1455 return false; 1456 } 1457 1458 info->SetIsMethodBeingCompiled(true, osr); 1459 return true; 1460 } 1461 1462 ProfilingInfo* JitCodeCache::NotifyCompilerUse(ArtMethod* method, Thread* self) { 1463 MutexLock mu(self, lock_); 1464 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1465 if (info != nullptr) { 1466 if (!info->IncrementInlineUse()) { 1467 // Overflow of inlining uses, just bail. 1468 return nullptr; 1469 } 1470 } 1471 return info; 1472 } 1473 1474 void JitCodeCache::DoneCompilerUse(ArtMethod* method, Thread* self) { 1475 MutexLock mu(self, lock_); 1476 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1477 DCHECK(info != nullptr); 1478 info->DecrementInlineUse(); 1479 } 1480 1481 void JitCodeCache::DoneCompiling(ArtMethod* method, Thread* self ATTRIBUTE_UNUSED, bool osr) { 1482 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1483 DCHECK(info->IsMethodBeingCompiled(osr)); 1484 info->SetIsMethodBeingCompiled(false, osr); 1485 } 1486 1487 size_t JitCodeCache::GetMemorySizeOfCodePointer(const void* ptr) { 1488 MutexLock mu(Thread::Current(), lock_); 1489 return mspace_usable_size(reinterpret_cast<const void*>(FromCodeToAllocation(ptr))); 1490 } 1491 1492 void JitCodeCache::InvalidateCompiledCodeFor(ArtMethod* method, 1493 const OatQuickMethodHeader* header) { 1494 ProfilingInfo* profiling_info = method->GetProfilingInfo(kRuntimePointerSize); 1495 if ((profiling_info != nullptr) && 1496 (profiling_info->GetSavedEntryPoint() == header->GetEntryPoint())) { 1497 // Prevent future uses of the compiled code. 1498 profiling_info->SetSavedEntryPoint(nullptr); 1499 } 1500 1501 if (method->GetEntryPointFromQuickCompiledCode() == header->GetEntryPoint()) { 1502 // The entrypoint is the one to invalidate, so we just update it to the interpreter entry point 1503 // and clear the counter to get the method Jitted again. 1504 Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( 1505 method, GetQuickToInterpreterBridge()); 1506 ClearMethodCounter(method, /*was_warm*/ profiling_info != nullptr); 1507 } else { 1508 MutexLock mu(Thread::Current(), lock_); 1509 auto it = osr_code_map_.find(method); 1510 if (it != osr_code_map_.end() && OatQuickMethodHeader::FromCodePointer(it->second) == header) { 1511 // Remove the OSR method, to avoid using it again. 1512 osr_code_map_.erase(it); 1513 } 1514 } 1515 } 1516 1517 uint8_t* JitCodeCache::AllocateCode(size_t code_size) { 1518 size_t alignment = GetInstructionSetAlignment(kRuntimeISA); 1519 uint8_t* result = reinterpret_cast<uint8_t*>( 1520 mspace_memalign(code_mspace_, alignment, code_size)); 1521 size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); 1522 // Ensure the header ends up at expected instruction alignment. 1523 DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment); 1524 used_memory_for_code_ += mspace_usable_size(result); 1525 return result; 1526 } 1527 1528 void JitCodeCache::FreeCode(uint8_t* code) { 1529 used_memory_for_code_ -= mspace_usable_size(code); 1530 mspace_free(code_mspace_, code); 1531 } 1532 1533 uint8_t* JitCodeCache::AllocateData(size_t data_size) { 1534 void* result = mspace_malloc(data_mspace_, data_size); 1535 used_memory_for_data_ += mspace_usable_size(result); 1536 return reinterpret_cast<uint8_t*>(result); 1537 } 1538 1539 void JitCodeCache::FreeData(uint8_t* data) { 1540 used_memory_for_data_ -= mspace_usable_size(data); 1541 mspace_free(data_mspace_, data); 1542 } 1543 1544 void JitCodeCache::Dump(std::ostream& os) { 1545 MutexLock mu(Thread::Current(), lock_); 1546 os << "Current JIT code cache size: " << PrettySize(used_memory_for_code_) << "\n" 1547 << "Current JIT data cache size: " << PrettySize(used_memory_for_data_) << "\n" 1548 << "Current JIT capacity: " << PrettySize(current_capacity_) << "\n" 1549 << "Current number of JIT code cache entries: " << method_code_map_.size() << "\n" 1550 << "Total number of JIT compilations: " << number_of_compilations_ << "\n" 1551 << "Total number of JIT compilations for on stack replacement: " 1552 << number_of_osr_compilations_ << "\n" 1553 << "Total number of JIT code cache collections: " << number_of_collections_ << std::endl; 1554 histogram_stack_map_memory_use_.PrintMemoryUse(os); 1555 histogram_code_memory_use_.PrintMemoryUse(os); 1556 histogram_profiling_info_memory_use_.PrintMemoryUse(os); 1557 } 1558 1559 } // namespace jit 1560 } // namespace art 1561
