1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "jit_code_cache.h" 18 19 #include <sstream> 20 21 #include <android-base/logging.h> 22 #include <android-base/unique_fd.h> 23 24 #include "arch/context.h" 25 #include "art_method-inl.h" 26 #include "base/enums.h" 27 #include "base/histogram-inl.h" 28 #include "base/logging.h" // For VLOG. 29 #include "base/membarrier.h" 30 #include "base/memfd.h" 31 #include "base/mem_map.h" 32 #include "base/quasi_atomic.h" 33 #include "base/stl_util.h" 34 #include "base/systrace.h" 35 #include "base/time_utils.h" 36 #include "base/utils.h" 37 #include "cha.h" 38 #include "debugger_interface.h" 39 #include "dex/dex_file_loader.h" 40 #include "dex/method_reference.h" 41 #include "entrypoints/runtime_asm_entrypoints.h" 42 #include "gc/accounting/bitmap-inl.h" 43 #include "gc/allocator/dlmalloc.h" 44 #include "gc/scoped_gc_critical_section.h" 45 #include "handle.h" 46 #include "instrumentation.h" 47 #include "intern_table.h" 48 #include "jit/jit.h" 49 #include "jit/profiling_info.h" 50 #include "linear_alloc.h" 51 #include "oat_file-inl.h" 52 #include "oat_quick_method_header.h" 53 #include "object_callbacks.h" 54 #include "profile/profile_compilation_info.h" 55 #include "scoped_thread_state_change-inl.h" 56 #include "stack.h" 57 #include "thread-current-inl.h" 58 #include "thread_list.h" 59 60 using android::base::unique_fd; 61 62 namespace art { 63 namespace jit { 64 65 static constexpr size_t kCodeSizeLogThreshold = 50 * KB; 66 static constexpr size_t kStackMapSizeLogThreshold = 50 * KB; 67 68 // Data cache will be half of the capacity 69 // Code cache will be the other half of the capacity. 70 // TODO: Make this variable? 71 static constexpr size_t kCodeAndDataCapacityDivider = 2; 72 73 static constexpr int kProtR = PROT_READ; 74 static constexpr int kProtRW = PROT_READ | PROT_WRITE; 75 static constexpr int kProtRWX = PROT_READ | PROT_WRITE | PROT_EXEC; 76 static constexpr int kProtRX = PROT_READ | PROT_EXEC; 77 78 namespace { 79 80 // Translate an address belonging to one memory map into an address in a second. This is useful 81 // when there are two virtual memory ranges for the same physical memory range. 82 template <typename T> 83 T* TranslateAddress(T* src_ptr, const MemMap& src, const MemMap& dst) { 84 CHECK(src.HasAddress(src_ptr)); 85 uint8_t* const raw_src_ptr = reinterpret_cast<uint8_t*>(src_ptr); 86 return reinterpret_cast<T*>(raw_src_ptr - src.Begin() + dst.Begin()); 87 } 88 89 } // namespace 90 91 class JitCodeCache::JniStubKey { 92 public: 93 explicit JniStubKey(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) 94 : shorty_(method->GetShorty()), 95 is_static_(method->IsStatic()), 96 is_fast_native_(method->IsFastNative()), 97 is_critical_native_(method->IsCriticalNative()), 98 is_synchronized_(method->IsSynchronized()) { 99 DCHECK(!(is_fast_native_ && is_critical_native_)); 100 } 101 102 bool operator<(const JniStubKey& rhs) const { 103 if (is_static_ != rhs.is_static_) { 104 return rhs.is_static_; 105 } 106 if (is_synchronized_ != rhs.is_synchronized_) { 107 return rhs.is_synchronized_; 108 } 109 if (is_fast_native_ != rhs.is_fast_native_) { 110 return rhs.is_fast_native_; 111 } 112 if (is_critical_native_ != rhs.is_critical_native_) { 113 return rhs.is_critical_native_; 114 } 115 return strcmp(shorty_, rhs.shorty_) < 0; 116 } 117 118 // Update the shorty to point to another method's shorty. Call this function when removing 119 // the method that references the old shorty from JniCodeData and not removing the entire 120 // JniCodeData; the old shorty may become a dangling pointer when that method is unloaded. 121 void UpdateShorty(ArtMethod* method) const REQUIRES_SHARED(Locks::mutator_lock_) { 122 const char* shorty = method->GetShorty(); 123 DCHECK_STREQ(shorty_, shorty); 124 shorty_ = shorty; 125 } 126 127 private: 128 // The shorty points to a DexFile data and may need to change 129 // to point to the same shorty in a different DexFile. 130 mutable const char* shorty_; 131 132 const bool is_static_; 133 const bool is_fast_native_; 134 const bool is_critical_native_; 135 const bool is_synchronized_; 136 }; 137 138 class JitCodeCache::JniStubData { 139 public: 140 JniStubData() : code_(nullptr), methods_() {} 141 142 void SetCode(const void* code) { 143 DCHECK(code != nullptr); 144 code_ = code; 145 } 146 147 const void* GetCode() const { 148 return code_; 149 } 150 151 bool IsCompiled() const { 152 return GetCode() != nullptr; 153 } 154 155 void AddMethod(ArtMethod* method) { 156 if (!ContainsElement(methods_, method)) { 157 methods_.push_back(method); 158 } 159 } 160 161 const std::vector<ArtMethod*>& GetMethods() const { 162 return methods_; 163 } 164 165 void RemoveMethodsIn(const LinearAlloc& alloc) { 166 auto kept_end = std::remove_if( 167 methods_.begin(), 168 methods_.end(), 169 [&alloc](ArtMethod* method) { return alloc.ContainsUnsafe(method); }); 170 methods_.erase(kept_end, methods_.end()); 171 } 172 173 bool RemoveMethod(ArtMethod* method) { 174 auto it = std::find(methods_.begin(), methods_.end(), method); 175 if (it != methods_.end()) { 176 methods_.erase(it); 177 return true; 178 } else { 179 return false; 180 } 181 } 182 183 void MoveObsoleteMethod(ArtMethod* old_method, ArtMethod* new_method) { 184 std::replace(methods_.begin(), methods_.end(), old_method, new_method); 185 } 186 187 private: 188 const void* code_; 189 std::vector<ArtMethod*> methods_; 190 }; 191 192 bool JitCodeCache::InitializeMappings(bool rwx_memory_allowed, 193 bool is_zygote, 194 std::string* error_msg) { 195 ScopedTrace trace(__PRETTY_FUNCTION__); 196 197 const size_t capacity = max_capacity_; 198 const size_t data_capacity = capacity / kCodeAndDataCapacityDivider; 199 const size_t exec_capacity = capacity - data_capacity; 200 201 // File descriptor enabling dual-view mapping of code section. 202 unique_fd mem_fd; 203 204 // Zygote shouldn't create a shared mapping for JIT, so we cannot use dual view 205 // for it. 206 if (!is_zygote) { 207 // Bionic supports memfd_create, but the call may fail on older kernels. 208 mem_fd = unique_fd(art::memfd_create("/jit-cache", /* flags= */ 0)); 209 if (mem_fd.get() < 0) { 210 std::ostringstream oss; 211 oss << "Failed to initialize dual view JIT. memfd_create() error: " << strerror(errno); 212 if (!rwx_memory_allowed) { 213 // Without using RWX page permissions, the JIT can not fallback to single mapping as it 214 // requires tranitioning the code pages to RWX for updates. 215 *error_msg = oss.str(); 216 return false; 217 } 218 VLOG(jit) << oss.str(); 219 } 220 } 221 222 if (mem_fd.get() >= 0 && ftruncate(mem_fd, capacity) != 0) { 223 std::ostringstream oss; 224 oss << "Failed to initialize memory file: " << strerror(errno); 225 *error_msg = oss.str(); 226 return false; 227 } 228 229 std::string data_cache_name = is_zygote ? "zygote-data-code-cache" : "data-code-cache"; 230 std::string exec_cache_name = is_zygote ? "zygote-jit-code-cache" : "jit-code-cache"; 231 232 std::string error_str; 233 // Map name specific for android_os_Debug.cpp accounting. 234 // Map in low 4gb to simplify accessing root tables for x86_64. 235 // We could do PC-relative addressing to avoid this problem, but that 236 // would require reserving code and data area before submitting, which 237 // means more windows for the code memory to be RWX. 238 int base_flags; 239 MemMap data_pages; 240 if (mem_fd.get() >= 0) { 241 // Dual view of JIT code cache case. Create an initial mapping of data pages large enough 242 // for data and non-writable view of JIT code pages. We use the memory file descriptor to 243 // enable dual mapping - we'll create a second mapping using the descriptor below. The 244 // mappings will look like: 245 // 246 // VA PA 247 // 248 // +---------------+ 249 // | non exec code |\ 250 // +---------------+ \ 251 // : :\ \ 252 // +---------------+.\.+---------------+ 253 // | exec code | \| code | 254 // +---------------+...+---------------+ 255 // | data | | data | 256 // +---------------+...+---------------+ 257 // 258 // In this configuration code updates are written to the non-executable view of the code 259 // cache, and the executable view of the code cache has fixed RX memory protections. 260 // 261 // This memory needs to be mapped shared as the code portions will have two mappings. 262 base_flags = MAP_SHARED; 263 data_pages = MemMap::MapFile( 264 data_capacity + exec_capacity, 265 kProtRW, 266 base_flags, 267 mem_fd, 268 /* start= */ 0, 269 /* low_4gb= */ true, 270 data_cache_name.c_str(), 271 &error_str); 272 } else { 273 // Single view of JIT code cache case. Create an initial mapping of data pages large enough 274 // for data and JIT code pages. The mappings will look like: 275 // 276 // VA PA 277 // 278 // +---------------+...+---------------+ 279 // | exec code | | code | 280 // +---------------+...+---------------+ 281 // | data | | data | 282 // +---------------+...+---------------+ 283 // 284 // In this configuration code updates are written to the executable view of the code cache, 285 // and the executable view of the code cache transitions RX to RWX for the update and then 286 // back to RX after the update. 287 base_flags = MAP_PRIVATE | MAP_ANON; 288 data_pages = MemMap::MapAnonymous( 289 data_cache_name.c_str(), 290 data_capacity + exec_capacity, 291 kProtRW, 292 /* low_4gb= */ true, 293 &error_str); 294 } 295 296 if (!data_pages.IsValid()) { 297 std::ostringstream oss; 298 oss << "Failed to create read write cache: " << error_str << " size=" << capacity; 299 *error_msg = oss.str(); 300 return false; 301 } 302 303 MemMap exec_pages; 304 MemMap non_exec_pages; 305 if (exec_capacity > 0) { 306 uint8_t* const divider = data_pages.Begin() + data_capacity; 307 // Set initial permission for executable view to catch any SELinux permission problems early 308 // (for processes that cannot map WX pages). Otherwise, this region does not need to be 309 // executable as there is no code in the cache yet. 310 exec_pages = data_pages.RemapAtEnd(divider, 311 exec_cache_name.c_str(), 312 kProtRX, 313 base_flags | MAP_FIXED, 314 mem_fd.get(), 315 (mem_fd.get() >= 0) ? data_capacity : 0, 316 &error_str); 317 if (!exec_pages.IsValid()) { 318 std::ostringstream oss; 319 oss << "Failed to create read execute code cache: " << error_str << " size=" << capacity; 320 *error_msg = oss.str(); 321 return false; 322 } 323 324 if (mem_fd.get() >= 0) { 325 // For dual view, create the secondary view of code memory used for updating code. This view 326 // is never executable. 327 std::string name = exec_cache_name + "-rw"; 328 non_exec_pages = MemMap::MapFile(exec_capacity, 329 kProtR, 330 base_flags, 331 mem_fd, 332 /* start= */ data_capacity, 333 /* low_4GB= */ false, 334 name.c_str(), 335 &error_str); 336 if (!non_exec_pages.IsValid()) { 337 static const char* kFailedNxView = "Failed to map non-executable view of JIT code cache"; 338 if (rwx_memory_allowed) { 339 // Log and continue as single view JIT (requires RWX memory). 340 VLOG(jit) << kFailedNxView; 341 } else { 342 *error_msg = kFailedNxView; 343 return false; 344 } 345 } 346 } 347 } else { 348 // Profiling only. No memory for code required. 349 } 350 351 data_pages_ = std::move(data_pages); 352 exec_pages_ = std::move(exec_pages); 353 non_exec_pages_ = std::move(non_exec_pages); 354 return true; 355 } 356 357 JitCodeCache* JitCodeCache::Create(bool used_only_for_profile_data, 358 bool rwx_memory_allowed, 359 bool is_zygote, 360 std::string* error_msg) { 361 // Register for membarrier expedited sync core if JIT will be generating code. 362 if (!used_only_for_profile_data) { 363 if (art::membarrier(art::MembarrierCommand::kRegisterPrivateExpeditedSyncCore) != 0) { 364 // MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE ensures that CPU instruction pipelines are 365 // flushed and it's used when adding code to the JIT. The memory used by the new code may 366 // have just been released and, in theory, the old code could still be in a pipeline. 367 VLOG(jit) << "Kernel does not support membarrier sync-core"; 368 } 369 } 370 371 // Check whether the provided max capacity in options is below 1GB. 372 size_t max_capacity = Runtime::Current()->GetJITOptions()->GetCodeCacheMaxCapacity(); 373 // We need to have 32 bit offsets from method headers in code cache which point to things 374 // in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work. 375 // Ensure we're below 1 GB to be safe. 376 if (max_capacity > 1 * GB) { 377 std::ostringstream oss; 378 oss << "Maxium code cache capacity is limited to 1 GB, " 379 << PrettySize(max_capacity) << " is too big"; 380 *error_msg = oss.str(); 381 return nullptr; 382 } 383 384 size_t initial_capacity = Runtime::Current()->GetJITOptions()->GetCodeCacheInitialCapacity(); 385 386 std::unique_ptr<JitCodeCache> jit_code_cache(new JitCodeCache()); 387 388 MutexLock mu(Thread::Current(), jit_code_cache->lock_); 389 jit_code_cache->InitializeState(initial_capacity, max_capacity); 390 391 // Zygote should never collect code to share the memory with the children. 392 if (is_zygote) { 393 jit_code_cache->garbage_collect_code_ = false; 394 } 395 396 if (!jit_code_cache->InitializeMappings(rwx_memory_allowed, is_zygote, error_msg)) { 397 return nullptr; 398 } 399 400 jit_code_cache->InitializeSpaces(); 401 402 VLOG(jit) << "Created jit code cache: initial capacity=" 403 << PrettySize(initial_capacity) 404 << ", maximum capacity=" 405 << PrettySize(max_capacity); 406 407 return jit_code_cache.release(); 408 } 409 410 JitCodeCache::JitCodeCache() 411 : lock_("Jit code cache", kJitCodeCacheLock), 412 lock_cond_("Jit code cache condition variable", lock_), 413 collection_in_progress_(false), 414 last_collection_increased_code_cache_(false), 415 garbage_collect_code_(true), 416 used_memory_for_data_(0), 417 used_memory_for_code_(0), 418 number_of_compilations_(0), 419 number_of_osr_compilations_(0), 420 number_of_collections_(0), 421 histogram_stack_map_memory_use_("Memory used for stack maps", 16), 422 histogram_code_memory_use_("Memory used for compiled code", 16), 423 histogram_profiling_info_memory_use_("Memory used for profiling info", 16), 424 is_weak_access_enabled_(true), 425 inline_cache_cond_("Jit inline cache condition variable", lock_), 426 zygote_data_pages_(), 427 zygote_exec_pages_(), 428 zygote_data_mspace_(nullptr), 429 zygote_exec_mspace_(nullptr) { 430 } 431 432 void JitCodeCache::InitializeState(size_t initial_capacity, size_t max_capacity) { 433 CHECK_GE(max_capacity, initial_capacity); 434 CHECK(max_capacity <= 1 * GB) << "The max supported size for JIT code cache is 1GB"; 435 // Align both capacities to page size, as that's the unit mspaces use. 436 initial_capacity = RoundDown(initial_capacity, 2 * kPageSize); 437 max_capacity = RoundDown(max_capacity, 2 * kPageSize); 438 439 used_memory_for_data_ = 0; 440 used_memory_for_code_ = 0; 441 number_of_compilations_ = 0; 442 number_of_osr_compilations_ = 0; 443 number_of_collections_ = 0; 444 445 data_pages_ = MemMap(); 446 exec_pages_ = MemMap(); 447 non_exec_pages_ = MemMap(); 448 initial_capacity_ = initial_capacity; 449 max_capacity_ = max_capacity; 450 current_capacity_ = initial_capacity, 451 data_end_ = initial_capacity / kCodeAndDataCapacityDivider; 452 exec_end_ = initial_capacity - data_end_; 453 } 454 455 void JitCodeCache::InitializeSpaces() { 456 // Initialize the data heap 457 data_mspace_ = create_mspace_with_base(data_pages_.Begin(), data_end_, false /*locked*/); 458 CHECK(data_mspace_ != nullptr) << "create_mspace_with_base (data) failed"; 459 460 // Initialize the code heap 461 MemMap* code_heap = nullptr; 462 if (non_exec_pages_.IsValid()) { 463 code_heap = &non_exec_pages_; 464 } else if (exec_pages_.IsValid()) { 465 code_heap = &exec_pages_; 466 } 467 if (code_heap != nullptr) { 468 // Make all pages reserved for the code heap writable. The mspace allocator, that manages the 469 // heap, will take and initialize pages in create_mspace_with_base(). 470 CheckedCall(mprotect, "create code heap", code_heap->Begin(), code_heap->Size(), kProtRW); 471 exec_mspace_ = create_mspace_with_base(code_heap->Begin(), exec_end_, false /*locked*/); 472 CHECK(exec_mspace_ != nullptr) << "create_mspace_with_base (exec) failed"; 473 SetFootprintLimit(initial_capacity_); 474 // Protect pages containing heap metadata. Updates to the code heap toggle write permission to 475 // perform the update and there are no other times write access is required. 476 CheckedCall(mprotect, "protect code heap", code_heap->Begin(), code_heap->Size(), kProtR); 477 } else { 478 exec_mspace_ = nullptr; 479 SetFootprintLimit(initial_capacity_); 480 } 481 } 482 483 JitCodeCache::~JitCodeCache() {} 484 485 bool JitCodeCache::ContainsPc(const void* ptr) const { 486 return exec_pages_.HasAddress(ptr) || zygote_exec_pages_.HasAddress(ptr); 487 } 488 489 bool JitCodeCache::WillExecuteJitCode(ArtMethod* method) { 490 ScopedObjectAccess soa(art::Thread::Current()); 491 ScopedAssertNoThreadSuspension sants(__FUNCTION__); 492 if (ContainsPc(method->GetEntryPointFromQuickCompiledCode())) { 493 return true; 494 } else if (method->GetEntryPointFromQuickCompiledCode() == GetQuickInstrumentationEntryPoint()) { 495 return FindCompiledCodeForInstrumentation(method) != nullptr; 496 } 497 return false; 498 } 499 500 bool JitCodeCache::ContainsMethod(ArtMethod* method) { 501 MutexLock mu(Thread::Current(), lock_); 502 if (UNLIKELY(method->IsNative())) { 503 auto it = jni_stubs_map_.find(JniStubKey(method)); 504 if (it != jni_stubs_map_.end() && 505 it->second.IsCompiled() && 506 ContainsElement(it->second.GetMethods(), method)) { 507 return true; 508 } 509 } else { 510 for (const auto& it : method_code_map_) { 511 if (it.second == method) { 512 return true; 513 } 514 } 515 } 516 return false; 517 } 518 519 const void* JitCodeCache::GetJniStubCode(ArtMethod* method) { 520 DCHECK(method->IsNative()); 521 MutexLock mu(Thread::Current(), lock_); 522 auto it = jni_stubs_map_.find(JniStubKey(method)); 523 if (it != jni_stubs_map_.end()) { 524 JniStubData& data = it->second; 525 if (data.IsCompiled() && ContainsElement(data.GetMethods(), method)) { 526 return data.GetCode(); 527 } 528 } 529 return nullptr; 530 } 531 532 const void* JitCodeCache::FindCompiledCodeForInstrumentation(ArtMethod* method) { 533 // If jit-gc is still on we use the SavedEntryPoint field for doing that and so cannot use it to 534 // find the instrumentation entrypoint. 535 if (LIKELY(GetGarbageCollectCode())) { 536 return nullptr; 537 } 538 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 539 if (info == nullptr) { 540 return nullptr; 541 } 542 // When GC is disabled for trampoline tracing we will use SavedEntrypoint to hold the actual 543 // jit-compiled version of the method. If jit-gc is disabled for other reasons this will just be 544 // nullptr. 545 return info->GetSavedEntryPoint(); 546 } 547 548 const void* JitCodeCache::GetZygoteSavedEntryPoint(ArtMethod* method) { 549 if (Runtime::Current()->IsUsingApexBootImageLocation() && 550 // Currently only applies to boot classpath 551 method->GetDeclaringClass()->GetClassLoader() == nullptr) { 552 const void* entry_point = nullptr; 553 if (method->IsNative()) { 554 const void* code_ptr = GetJniStubCode(method); 555 if (code_ptr != nullptr) { 556 entry_point = OatQuickMethodHeader::FromCodePointer(code_ptr)->GetEntryPoint(); 557 } 558 } else { 559 ProfilingInfo* profiling_info = method->GetProfilingInfo(kRuntimePointerSize); 560 if (profiling_info != nullptr) { 561 entry_point = profiling_info->GetSavedEntryPoint(); 562 } 563 } 564 if (Runtime::Current()->IsZygote() || IsInZygoteExecSpace(entry_point)) { 565 return entry_point; 566 } 567 } 568 return nullptr; 569 } 570 571 class ScopedCodeCacheWrite : ScopedTrace { 572 public: 573 explicit ScopedCodeCacheWrite(const JitCodeCache* const code_cache) 574 : ScopedTrace("ScopedCodeCacheWrite"), 575 code_cache_(code_cache) { 576 ScopedTrace trace("mprotect all"); 577 const MemMap* const updatable_pages = code_cache_->GetUpdatableCodeMapping(); 578 if (updatable_pages != nullptr) { 579 int prot = code_cache_->HasDualCodeMapping() ? kProtRW : kProtRWX; 580 CheckedCall(mprotect, "Cache +W", updatable_pages->Begin(), updatable_pages->Size(), prot); 581 } 582 } 583 584 ~ScopedCodeCacheWrite() { 585 ScopedTrace trace("mprotect code"); 586 const MemMap* const updatable_pages = code_cache_->GetUpdatableCodeMapping(); 587 if (updatable_pages != nullptr) { 588 int prot = code_cache_->HasDualCodeMapping() ? kProtR : kProtRX; 589 CheckedCall(mprotect, "Cache -W", updatable_pages->Begin(), updatable_pages->Size(), prot); 590 } 591 } 592 593 private: 594 const JitCodeCache* const code_cache_; 595 596 DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite); 597 }; 598 599 uint8_t* JitCodeCache::CommitCode(Thread* self, 600 ArtMethod* method, 601 uint8_t* stack_map, 602 uint8_t* roots_data, 603 const uint8_t* code, 604 size_t code_size, 605 size_t data_size, 606 bool osr, 607 const std::vector<Handle<mirror::Object>>& roots, 608 bool has_should_deoptimize_flag, 609 const ArenaSet<ArtMethod*>& cha_single_implementation_list) { 610 uint8_t* result = CommitCodeInternal(self, 611 method, 612 stack_map, 613 roots_data, 614 code, 615 code_size, 616 data_size, 617 osr, 618 roots, 619 has_should_deoptimize_flag, 620 cha_single_implementation_list); 621 if (result == nullptr) { 622 // Retry. 623 GarbageCollectCache(self); 624 result = CommitCodeInternal(self, 625 method, 626 stack_map, 627 roots_data, 628 code, 629 code_size, 630 data_size, 631 osr, 632 roots, 633 has_should_deoptimize_flag, 634 cha_single_implementation_list); 635 } 636 return result; 637 } 638 639 bool JitCodeCache::WaitForPotentialCollectionToComplete(Thread* self) { 640 bool in_collection = false; 641 while (collection_in_progress_) { 642 in_collection = true; 643 lock_cond_.Wait(self); 644 } 645 return in_collection; 646 } 647 648 static size_t GetJitCodeAlignment() { 649 if (kRuntimeISA == InstructionSet::kArm || kRuntimeISA == InstructionSet::kThumb2) { 650 // Some devices with 32-bit ARM kernels need additional JIT code alignment when using dual 651 // view JIT (b/132205399). The alignment returned here coincides with the typical ARM d-cache 652 // line (though the value should be probed ideally). Both the method header and code in the 653 // cache are aligned to this size. Anything less than 64-bytes exhibits the problem. 654 return 64; 655 } 656 return GetInstructionSetAlignment(kRuntimeISA); 657 } 658 659 static uintptr_t FromCodeToAllocation(const void* code) { 660 size_t alignment = GetJitCodeAlignment(); 661 return reinterpret_cast<uintptr_t>(code) - RoundUp(sizeof(OatQuickMethodHeader), alignment); 662 } 663 664 static uint32_t ComputeRootTableSize(uint32_t number_of_roots) { 665 return sizeof(uint32_t) + number_of_roots * sizeof(GcRoot<mirror::Object>); 666 } 667 668 static uint32_t GetNumberOfRoots(const uint8_t* stack_map) { 669 // The length of the table is stored just before the stack map (and therefore at the end of 670 // the table itself), in order to be able to fetch it from a `stack_map` pointer. 671 return reinterpret_cast<const uint32_t*>(stack_map)[-1]; 672 } 673 674 static void FillRootTableLength(uint8_t* roots_data, uint32_t length) { 675 // Store the length of the table at the end. This will allow fetching it from a `stack_map` 676 // pointer. 677 reinterpret_cast<uint32_t*>(roots_data)[length] = length; 678 } 679 680 static const uint8_t* FromStackMapToRoots(const uint8_t* stack_map_data) { 681 return stack_map_data - ComputeRootTableSize(GetNumberOfRoots(stack_map_data)); 682 } 683 684 static void DCheckRootsAreValid(const std::vector<Handle<mirror::Object>>& roots) 685 REQUIRES(!Locks::intern_table_lock_) REQUIRES_SHARED(Locks::mutator_lock_) { 686 if (!kIsDebugBuild) { 687 return; 688 } 689 // Put all roots in `roots_data`. 690 for (Handle<mirror::Object> object : roots) { 691 // Ensure the string is strongly interned. b/32995596 692 if (object->IsString()) { 693 ObjPtr<mirror::String> str = object->AsString(); 694 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 695 CHECK(class_linker->GetInternTable()->LookupStrong(Thread::Current(), str) != nullptr); 696 } 697 } 698 } 699 700 void JitCodeCache::FillRootTable(uint8_t* roots_data, 701 const std::vector<Handle<mirror::Object>>& roots) { 702 GcRoot<mirror::Object>* gc_roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); 703 const uint32_t length = roots.size(); 704 // Put all roots in `roots_data`. 705 for (uint32_t i = 0; i < length; ++i) { 706 ObjPtr<mirror::Object> object = roots[i].Get(); 707 gc_roots[i] = GcRoot<mirror::Object>(object); 708 } 709 } 710 711 static uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr) { 712 OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 713 uint8_t* data = method_header->GetOptimizedCodeInfoPtr(); 714 uint32_t roots = GetNumberOfRoots(data); 715 if (number_of_roots != nullptr) { 716 *number_of_roots = roots; 717 } 718 return data - ComputeRootTableSize(roots); 719 } 720 721 // Use a sentinel for marking entries in the JIT table that have been cleared. 722 // This helps diagnosing in case the compiled code tries to wrongly access such 723 // entries. 724 static mirror::Class* const weak_sentinel = 725 reinterpret_cast<mirror::Class*>(Context::kBadGprBase + 0xff); 726 727 // Helper for the GC to process a weak class in a JIT root table. 728 static inline void ProcessWeakClass(GcRoot<mirror::Class>* root_ptr, 729 IsMarkedVisitor* visitor, 730 mirror::Class* update) 731 REQUIRES_SHARED(Locks::mutator_lock_) { 732 // This does not need a read barrier because this is called by GC. 733 mirror::Class* cls = root_ptr->Read<kWithoutReadBarrier>(); 734 if (cls != nullptr && cls != weak_sentinel) { 735 DCHECK((cls->IsClass<kDefaultVerifyFlags>())); 736 // Look at the classloader of the class to know if it has been unloaded. 737 // This does not need a read barrier because this is called by GC. 738 ObjPtr<mirror::Object> class_loader = 739 cls->GetClassLoader<kDefaultVerifyFlags, kWithoutReadBarrier>(); 740 if (class_loader == nullptr || visitor->IsMarked(class_loader.Ptr()) != nullptr) { 741 // The class loader is live, update the entry if the class has moved. 742 mirror::Class* new_cls = down_cast<mirror::Class*>(visitor->IsMarked(cls)); 743 // Note that new_object can be null for CMS and newly allocated objects. 744 if (new_cls != nullptr && new_cls != cls) { 745 *root_ptr = GcRoot<mirror::Class>(new_cls); 746 } 747 } else { 748 // The class loader is not live, clear the entry. 749 *root_ptr = GcRoot<mirror::Class>(update); 750 } 751 } 752 } 753 754 void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) { 755 MutexLock mu(Thread::Current(), lock_); 756 for (const auto& entry : method_code_map_) { 757 uint32_t number_of_roots = 0; 758 uint8_t* roots_data = GetRootTable(entry.first, &number_of_roots); 759 GcRoot<mirror::Object>* roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); 760 for (uint32_t i = 0; i < number_of_roots; ++i) { 761 // This does not need a read barrier because this is called by GC. 762 mirror::Object* object = roots[i].Read<kWithoutReadBarrier>(); 763 if (object == nullptr || object == weak_sentinel) { 764 // entry got deleted in a previous sweep. 765 } else if (object->IsString<kDefaultVerifyFlags>()) { 766 mirror::Object* new_object = visitor->IsMarked(object); 767 // We know the string is marked because it's a strongly-interned string that 768 // is always alive. The IsMarked implementation of the CMS collector returns 769 // null for newly allocated objects, but we know those haven't moved. Therefore, 770 // only update the entry if we get a different non-null string. 771 // TODO: Do not use IsMarked for j.l.Class, and adjust once we move this method 772 // out of the weak access/creation pause. b/32167580 773 if (new_object != nullptr && new_object != object) { 774 DCHECK(new_object->IsString()); 775 roots[i] = GcRoot<mirror::Object>(new_object); 776 } 777 } else { 778 ProcessWeakClass( 779 reinterpret_cast<GcRoot<mirror::Class>*>(&roots[i]), visitor, weak_sentinel); 780 } 781 } 782 } 783 // Walk over inline caches to clear entries containing unloaded classes. 784 for (ProfilingInfo* info : profiling_infos_) { 785 for (size_t i = 0; i < info->number_of_inline_caches_; ++i) { 786 InlineCache* cache = &info->cache_[i]; 787 for (size_t j = 0; j < InlineCache::kIndividualCacheSize; ++j) { 788 ProcessWeakClass(&cache->classes_[j], visitor, nullptr); 789 } 790 } 791 } 792 } 793 794 void JitCodeCache::FreeCodeAndData(const void* code_ptr) { 795 if (IsInZygoteExecSpace(code_ptr)) { 796 // No need to free, this is shared memory. 797 return; 798 } 799 uintptr_t allocation = FromCodeToAllocation(code_ptr); 800 // Notify native debugger that we are about to remove the code. 801 // It does nothing if we are not using native debugger. 802 RemoveNativeDebugInfoForJit(Thread::Current(), code_ptr); 803 if (OatQuickMethodHeader::FromCodePointer(code_ptr)->IsOptimized()) { 804 FreeData(GetRootTable(code_ptr)); 805 } // else this is a JNI stub without any data. 806 807 uint8_t* code_allocation = reinterpret_cast<uint8_t*>(allocation); 808 if (HasDualCodeMapping()) { 809 code_allocation = TranslateAddress(code_allocation, exec_pages_, non_exec_pages_); 810 } 811 812 FreeCode(code_allocation); 813 } 814 815 void JitCodeCache::FreeAllMethodHeaders( 816 const std::unordered_set<OatQuickMethodHeader*>& method_headers) { 817 // We need to remove entries in method_headers from CHA dependencies 818 // first since once we do FreeCode() below, the memory can be reused 819 // so it's possible for the same method_header to start representing 820 // different compile code. 821 MutexLock mu(Thread::Current(), lock_); 822 { 823 MutexLock mu2(Thread::Current(), *Locks::cha_lock_); 824 Runtime::Current()->GetClassLinker()->GetClassHierarchyAnalysis() 825 ->RemoveDependentsWithMethodHeaders(method_headers); 826 } 827 828 ScopedCodeCacheWrite scc(this); 829 for (const OatQuickMethodHeader* method_header : method_headers) { 830 FreeCodeAndData(method_header->GetCode()); 831 } 832 } 833 834 void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) { 835 ScopedTrace trace(__PRETTY_FUNCTION__); 836 // We use a set to first collect all method_headers whose code need to be 837 // removed. We need to free the underlying code after we remove CHA dependencies 838 // for entries in this set. And it's more efficient to iterate through 839 // the CHA dependency map just once with an unordered_set. 840 std::unordered_set<OatQuickMethodHeader*> method_headers; 841 { 842 MutexLock mu(self, lock_); 843 // We do not check if a code cache GC is in progress, as this method comes 844 // with the classlinker_classes_lock_ held, and suspending ourselves could 845 // lead to a deadlock. 846 { 847 ScopedCodeCacheWrite scc(this); 848 for (auto it = jni_stubs_map_.begin(); it != jni_stubs_map_.end();) { 849 it->second.RemoveMethodsIn(alloc); 850 if (it->second.GetMethods().empty()) { 851 method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->second.GetCode())); 852 it = jni_stubs_map_.erase(it); 853 } else { 854 it->first.UpdateShorty(it->second.GetMethods().front()); 855 ++it; 856 } 857 } 858 for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { 859 if (alloc.ContainsUnsafe(it->second)) { 860 method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first)); 861 it = method_code_map_.erase(it); 862 } else { 863 ++it; 864 } 865 } 866 } 867 for (auto it = osr_code_map_.begin(); it != osr_code_map_.end();) { 868 if (alloc.ContainsUnsafe(it->first)) { 869 // Note that the code has already been pushed to method_headers in the loop 870 // above and is going to be removed in FreeCode() below. 871 it = osr_code_map_.erase(it); 872 } else { 873 ++it; 874 } 875 } 876 for (auto it = profiling_infos_.begin(); it != profiling_infos_.end();) { 877 ProfilingInfo* info = *it; 878 if (alloc.ContainsUnsafe(info->GetMethod())) { 879 info->GetMethod()->SetProfilingInfo(nullptr); 880 FreeData(reinterpret_cast<uint8_t*>(info)); 881 it = profiling_infos_.erase(it); 882 } else { 883 ++it; 884 } 885 } 886 } 887 FreeAllMethodHeaders(method_headers); 888 } 889 890 bool JitCodeCache::IsWeakAccessEnabled(Thread* self) const { 891 return kUseReadBarrier 892 ? self->GetWeakRefAccessEnabled() 893 : is_weak_access_enabled_.load(std::memory_order_seq_cst); 894 } 895 896 void JitCodeCache::WaitUntilInlineCacheAccessible(Thread* self) { 897 if (IsWeakAccessEnabled(self)) { 898 return; 899 } 900 ScopedThreadSuspension sts(self, kWaitingWeakGcRootRead); 901 MutexLock mu(self, lock_); 902 while (!IsWeakAccessEnabled(self)) { 903 inline_cache_cond_.Wait(self); 904 } 905 } 906 907 void JitCodeCache::BroadcastForInlineCacheAccess() { 908 Thread* self = Thread::Current(); 909 MutexLock mu(self, lock_); 910 inline_cache_cond_.Broadcast(self); 911 } 912 913 void JitCodeCache::AllowInlineCacheAccess() { 914 DCHECK(!kUseReadBarrier); 915 is_weak_access_enabled_.store(true, std::memory_order_seq_cst); 916 BroadcastForInlineCacheAccess(); 917 } 918 919 void JitCodeCache::DisallowInlineCacheAccess() { 920 DCHECK(!kUseReadBarrier); 921 is_weak_access_enabled_.store(false, std::memory_order_seq_cst); 922 } 923 924 void JitCodeCache::CopyInlineCacheInto(const InlineCache& ic, 925 Handle<mirror::ObjectArray<mirror::Class>> array) { 926 WaitUntilInlineCacheAccessible(Thread::Current()); 927 // Note that we don't need to lock `lock_` here, the compiler calling 928 // this method has already ensured the inline cache will not be deleted. 929 for (size_t in_cache = 0, in_array = 0; 930 in_cache < InlineCache::kIndividualCacheSize; 931 ++in_cache) { 932 mirror::Class* object = ic.classes_[in_cache].Read(); 933 if (object != nullptr) { 934 array->Set(in_array++, object); 935 } 936 } 937 } 938 939 static void ClearMethodCounter(ArtMethod* method, bool was_warm) 940 REQUIRES_SHARED(Locks::mutator_lock_) { 941 if (was_warm) { 942 method->SetPreviouslyWarm(); 943 } 944 // We reset the counter to 1 so that the profile knows that the method was executed at least once. 945 // This is required for layout purposes. 946 // We also need to make sure we'll pass the warmup threshold again, so we set to 0 if 947 // the warmup threshold is 1. 948 uint16_t jit_warmup_threshold = Runtime::Current()->GetJITOptions()->GetWarmupThreshold(); 949 method->SetCounter(std::min(jit_warmup_threshold - 1, 1)); 950 } 951 952 void JitCodeCache::WaitForPotentialCollectionToCompleteRunnable(Thread* self) { 953 while (collection_in_progress_) { 954 lock_.Unlock(self); 955 { 956 ScopedThreadSuspension sts(self, kSuspended); 957 MutexLock mu(self, lock_); 958 WaitForPotentialCollectionToComplete(self); 959 } 960 lock_.Lock(self); 961 } 962 } 963 964 const MemMap* JitCodeCache::GetUpdatableCodeMapping() const { 965 if (HasDualCodeMapping()) { 966 return &non_exec_pages_; 967 } else if (HasCodeMapping()) { 968 return &exec_pages_; 969 } else { 970 return nullptr; 971 } 972 } 973 974 uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, 975 ArtMethod* method, 976 uint8_t* stack_map, 977 uint8_t* roots_data, 978 const uint8_t* code, 979 size_t code_size, 980 size_t data_size, 981 bool osr, 982 const std::vector<Handle<mirror::Object>>& roots, 983 bool has_should_deoptimize_flag, 984 const ArenaSet<ArtMethod*>& 985 cha_single_implementation_list) { 986 DCHECK(!method->IsNative() || !osr); 987 988 if (!method->IsNative()) { 989 // We need to do this before grabbing the lock_ because it needs to be able to see the string 990 // InternTable. Native methods do not have roots. 991 DCheckRootsAreValid(roots); 992 } 993 994 OatQuickMethodHeader* method_header = nullptr; 995 uint8_t* nox_memory = nullptr; 996 uint8_t* code_ptr = nullptr; 997 998 MutexLock mu(self, lock_); 999 // We need to make sure that there will be no jit-gcs going on and wait for any ongoing one to 1000 // finish. 1001 WaitForPotentialCollectionToCompleteRunnable(self); 1002 { 1003 ScopedCodeCacheWrite scc(this); 1004 1005 size_t alignment = GetJitCodeAlignment(); 1006 // Ensure the header ends up at expected instruction alignment. 1007 size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); 1008 size_t total_size = header_size + code_size; 1009 1010 // AllocateCode allocates memory in non-executable region for alignment header and code. The 1011 // header size may include alignment padding. 1012 nox_memory = AllocateCode(total_size); 1013 if (nox_memory == nullptr) { 1014 return nullptr; 1015 } 1016 1017 // code_ptr points to non-executable code. 1018 code_ptr = nox_memory + header_size; 1019 std::copy(code, code + code_size, code_ptr); 1020 method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1021 1022 // From here code_ptr points to executable code. 1023 if (HasDualCodeMapping()) { 1024 code_ptr = TranslateAddress(code_ptr, non_exec_pages_, exec_pages_); 1025 } 1026 1027 new (method_header) OatQuickMethodHeader( 1028 (stack_map != nullptr) ? code_ptr - stack_map : 0u, 1029 code_size); 1030 1031 DCHECK(!Runtime::Current()->IsAotCompiler()); 1032 if (has_should_deoptimize_flag) { 1033 method_header->SetHasShouldDeoptimizeFlag(); 1034 } 1035 1036 // Update method_header pointer to executable code region. 1037 if (HasDualCodeMapping()) { 1038 method_header = TranslateAddress(method_header, non_exec_pages_, exec_pages_); 1039 } 1040 1041 // Both instruction and data caches need flushing to the point of unification where both share 1042 // a common view of memory. Flushing the data cache ensures the dirty cachelines from the 1043 // newly added code are written out to the point of unification. Flushing the instruction 1044 // cache ensures the newly written code will be fetched from the point of unification before 1045 // use. Memory in the code cache is re-cycled as code is added and removed. The flushes 1046 // prevent stale code from residing in the instruction cache. 1047 // 1048 // Caches are flushed before write permission is removed because some ARMv8 Qualcomm kernels 1049 // may trigger a segfault if a page fault occurs when requesting a cache maintenance 1050 // operation. This is a kernel bug that we need to work around until affected devices 1051 // (e.g. Nexus 5X and 6P) stop being supported or their kernels are fixed. 1052 // 1053 // For reference, this behavior is caused by this commit: 1054 // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c 1055 // 1056 bool cache_flush_success = true; 1057 if (HasDualCodeMapping()) { 1058 // Flush the data cache lines associated with the non-executable copy of the code just added. 1059 cache_flush_success = FlushCpuCaches(nox_memory, nox_memory + total_size); 1060 } 1061 1062 // Invalidate i-cache for the executable mapping. 1063 if (cache_flush_success) { 1064 uint8_t* x_memory = reinterpret_cast<uint8_t*>(FromCodeToAllocation(code_ptr)); 1065 cache_flush_success = FlushCpuCaches(x_memory, x_memory + total_size); 1066 } 1067 1068 // If flushing the cache has failed, reject the allocation because we can't guarantee 1069 // correctness of the instructions present in the processor caches. 1070 if (!cache_flush_success) { 1071 PLOG(ERROR) << "Cache flush failed for JIT code, code not committed."; 1072 FreeCode(nox_memory); 1073 return nullptr; 1074 } 1075 1076 // Ensure CPU instruction pipelines are flushed for all cores. This is necessary for 1077 // correctness as code may still be in instruction pipelines despite the i-cache flush. It is 1078 // not safe to assume that changing permissions with mprotect (RX->RWX->RX) will cause a TLB 1079 // shootdown (incidentally invalidating the CPU pipelines by sending an IPI to all cores to 1080 // notify them of the TLB invalidation). Some architectures, notably ARM and ARM64, have 1081 // hardware support that broadcasts TLB invalidations and so their kernels have no software 1082 // based TLB shootdown. The sync-core flavor of membarrier was introduced in Linux 4.16 to 1083 // address this (see mbarrier(2)). The membarrier here will fail on prior kernels and on 1084 // platforms lacking the appropriate support. 1085 art::membarrier(art::MembarrierCommand::kPrivateExpeditedSyncCore); 1086 1087 number_of_compilations_++; 1088 } 1089 1090 // We need to update the entry point in the runnable state for the instrumentation. 1091 { 1092 // The following needs to be guarded by cha_lock_ also. Otherwise it's possible that the 1093 // compiled code is considered invalidated by some class linking, but below we still make the 1094 // compiled code valid for the method. Need cha_lock_ for checking all single-implementation 1095 // flags and register dependencies. 1096 MutexLock cha_mu(self, *Locks::cha_lock_); 1097 bool single_impl_still_valid = true; 1098 for (ArtMethod* single_impl : cha_single_implementation_list) { 1099 if (!single_impl->HasSingleImplementation()) { 1100 // Simply discard the compiled code. Clear the counter so that it may be recompiled later. 1101 // Hopefully the class hierarchy will be more stable when compilation is retried. 1102 single_impl_still_valid = false; 1103 ClearMethodCounter(method, /*was_warm=*/ false); 1104 break; 1105 } 1106 } 1107 1108 // Discard the code if any single-implementation assumptions are now invalid. 1109 if (!single_impl_still_valid) { 1110 VLOG(jit) << "JIT discarded jitted code due to invalid single-implementation assumptions."; 1111 return nullptr; 1112 } 1113 DCHECK(cha_single_implementation_list.empty() || !Runtime::Current()->IsJavaDebuggable()) 1114 << "Should not be using cha on debuggable apps/runs!"; 1115 1116 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 1117 for (ArtMethod* single_impl : cha_single_implementation_list) { 1118 class_linker->GetClassHierarchyAnalysis()->AddDependency(single_impl, method, method_header); 1119 } 1120 1121 if (UNLIKELY(method->IsNative())) { 1122 auto it = jni_stubs_map_.find(JniStubKey(method)); 1123 DCHECK(it != jni_stubs_map_.end()) 1124 << "Entry inserted in NotifyCompilationOf() should be alive."; 1125 JniStubData* data = &it->second; 1126 DCHECK(ContainsElement(data->GetMethods(), method)) 1127 << "Entry inserted in NotifyCompilationOf() should contain this method."; 1128 data->SetCode(code_ptr); 1129 instrumentation::Instrumentation* instrum = Runtime::Current()->GetInstrumentation(); 1130 for (ArtMethod* m : data->GetMethods()) { 1131 if (!class_linker->IsQuickResolutionStub(m->GetEntryPointFromQuickCompiledCode())) { 1132 instrum->UpdateMethodsCode(m, method_header->GetEntryPoint()); 1133 } 1134 } 1135 } else { 1136 // Fill the root table before updating the entry point. 1137 DCHECK_EQ(FromStackMapToRoots(stack_map), roots_data); 1138 DCHECK_LE(roots_data, stack_map); 1139 FillRootTable(roots_data, roots); 1140 { 1141 // Flush data cache, as compiled code references literals in it. 1142 // TODO(oth): establish whether this is necessary. 1143 if (!FlushCpuCaches(roots_data, roots_data + data_size)) { 1144 PLOG(ERROR) << "Cache flush failed for JIT data, code not committed."; 1145 ScopedCodeCacheWrite scc(this); 1146 FreeCode(nox_memory); 1147 return nullptr; 1148 } 1149 } 1150 method_code_map_.Put(code_ptr, method); 1151 if (osr) { 1152 number_of_osr_compilations_++; 1153 osr_code_map_.Put(method, code_ptr); 1154 } else if (class_linker->IsQuickResolutionStub( 1155 method->GetEntryPointFromQuickCompiledCode())) { 1156 // This situation currently only occurs in the jit-zygote mode. 1157 DCHECK(Runtime::Current()->IsZygote()); 1158 DCHECK(Runtime::Current()->IsUsingApexBootImageLocation()); 1159 DCHECK(method->GetProfilingInfo(kRuntimePointerSize) != nullptr); 1160 DCHECK(method->GetDeclaringClass()->GetClassLoader() == nullptr); 1161 // Save the entrypoint, so it can be fethed later once the class is 1162 // initialized. 1163 method->GetProfilingInfo(kRuntimePointerSize)->SetSavedEntryPoint( 1164 method_header->GetEntryPoint()); 1165 } else { 1166 Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( 1167 method, method_header->GetEntryPoint()); 1168 } 1169 } 1170 VLOG(jit) 1171 << "JIT added (osr=" << std::boolalpha << osr << std::noboolalpha << ") " 1172 << ArtMethod::PrettyMethod(method) << "@" << method 1173 << " ccache_size=" << PrettySize(CodeCacheSizeLocked()) << ": " 1174 << " dcache_size=" << PrettySize(DataCacheSizeLocked()) << ": " 1175 << reinterpret_cast<const void*>(method_header->GetEntryPoint()) << "," 1176 << reinterpret_cast<const void*>(method_header->GetEntryPoint() + 1177 method_header->GetCodeSize()); 1178 histogram_code_memory_use_.AddValue(code_size); 1179 if (code_size > kCodeSizeLogThreshold) { 1180 LOG(INFO) << "JIT allocated " 1181 << PrettySize(code_size) 1182 << " for compiled code of " 1183 << ArtMethod::PrettyMethod(method); 1184 } 1185 } 1186 1187 return reinterpret_cast<uint8_t*>(method_header); 1188 } 1189 1190 size_t JitCodeCache::CodeCacheSize() { 1191 MutexLock mu(Thread::Current(), lock_); 1192 return CodeCacheSizeLocked(); 1193 } 1194 1195 bool JitCodeCache::RemoveMethod(ArtMethod* method, bool release_memory) { 1196 // This function is used only for testing and only with non-native methods. 1197 CHECK(!method->IsNative()); 1198 1199 MutexLock mu(Thread::Current(), lock_); 1200 1201 bool osr = osr_code_map_.find(method) != osr_code_map_.end(); 1202 bool in_cache = RemoveMethodLocked(method, release_memory); 1203 1204 if (!in_cache) { 1205 return false; 1206 } 1207 1208 method->SetCounter(0); 1209 Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( 1210 method, GetQuickToInterpreterBridge()); 1211 VLOG(jit) 1212 << "JIT removed (osr=" << std::boolalpha << osr << std::noboolalpha << ") " 1213 << ArtMethod::PrettyMethod(method) << "@" << method 1214 << " ccache_size=" << PrettySize(CodeCacheSizeLocked()) << ": " 1215 << " dcache_size=" << PrettySize(DataCacheSizeLocked()); 1216 return true; 1217 } 1218 1219 bool JitCodeCache::RemoveMethodLocked(ArtMethod* method, bool release_memory) { 1220 if (LIKELY(!method->IsNative())) { 1221 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1222 if (info != nullptr) { 1223 RemoveElement(profiling_infos_, info); 1224 } 1225 method->SetProfilingInfo(nullptr); 1226 } 1227 1228 bool in_cache = false; 1229 ScopedCodeCacheWrite ccw(this); 1230 if (UNLIKELY(method->IsNative())) { 1231 auto it = jni_stubs_map_.find(JniStubKey(method)); 1232 if (it != jni_stubs_map_.end() && it->second.RemoveMethod(method)) { 1233 in_cache = true; 1234 if (it->second.GetMethods().empty()) { 1235 if (release_memory) { 1236 FreeCodeAndData(it->second.GetCode()); 1237 } 1238 jni_stubs_map_.erase(it); 1239 } else { 1240 it->first.UpdateShorty(it->second.GetMethods().front()); 1241 } 1242 } 1243 } else { 1244 for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { 1245 if (it->second == method) { 1246 in_cache = true; 1247 if (release_memory) { 1248 FreeCodeAndData(it->first); 1249 } 1250 it = method_code_map_.erase(it); 1251 } else { 1252 ++it; 1253 } 1254 } 1255 1256 auto osr_it = osr_code_map_.find(method); 1257 if (osr_it != osr_code_map_.end()) { 1258 osr_code_map_.erase(osr_it); 1259 } 1260 } 1261 1262 return in_cache; 1263 } 1264 1265 // This notifies the code cache that the given method has been redefined and that it should remove 1266 // any cached information it has on the method. All threads must be suspended before calling this 1267 // method. The compiled code for the method (if there is any) must not be in any threads call stack. 1268 void JitCodeCache::NotifyMethodRedefined(ArtMethod* method) { 1269 MutexLock mu(Thread::Current(), lock_); 1270 RemoveMethodLocked(method, /* release_memory= */ true); 1271 } 1272 1273 // This invalidates old_method. Once this function returns one can no longer use old_method to 1274 // execute code unless it is fixed up. This fixup will happen later in the process of installing a 1275 // class redefinition. 1276 // TODO We should add some info to ArtMethod to note that 'old_method' has been invalidated and 1277 // shouldn't be used since it is no longer logically in the jit code cache. 1278 // TODO We should add DCHECKS that validate that the JIT is paused when this method is entered. 1279 void JitCodeCache::MoveObsoleteMethod(ArtMethod* old_method, ArtMethod* new_method) { 1280 MutexLock mu(Thread::Current(), lock_); 1281 if (old_method->IsNative()) { 1282 // Update methods in jni_stubs_map_. 1283 for (auto& entry : jni_stubs_map_) { 1284 JniStubData& data = entry.second; 1285 data.MoveObsoleteMethod(old_method, new_method); 1286 } 1287 return; 1288 } 1289 // Update ProfilingInfo to the new one and remove it from the old_method. 1290 if (old_method->GetProfilingInfo(kRuntimePointerSize) != nullptr) { 1291 DCHECK_EQ(old_method->GetProfilingInfo(kRuntimePointerSize)->GetMethod(), old_method); 1292 ProfilingInfo* info = old_method->GetProfilingInfo(kRuntimePointerSize); 1293 old_method->SetProfilingInfo(nullptr); 1294 // Since the JIT should be paused and all threads suspended by the time this is called these 1295 // checks should always pass. 1296 DCHECK(!info->IsInUseByCompiler()); 1297 new_method->SetProfilingInfo(info); 1298 // Get rid of the old saved entrypoint if it is there. 1299 info->SetSavedEntryPoint(nullptr); 1300 info->method_ = new_method; 1301 } 1302 // Update method_code_map_ to point to the new method. 1303 for (auto& it : method_code_map_) { 1304 if (it.second == old_method) { 1305 it.second = new_method; 1306 } 1307 } 1308 // Update osr_code_map_ to point to the new method. 1309 auto code_map = osr_code_map_.find(old_method); 1310 if (code_map != osr_code_map_.end()) { 1311 osr_code_map_.Put(new_method, code_map->second); 1312 osr_code_map_.erase(old_method); 1313 } 1314 } 1315 1316 void JitCodeCache::ClearEntryPointsInZygoteExecSpace() { 1317 MutexLock mu(Thread::Current(), lock_); 1318 // Iterate over profiling infos to know which methods may have been JITted. Note that 1319 // to be JITted, a method must have a profiling info. 1320 for (ProfilingInfo* info : profiling_infos_) { 1321 ArtMethod* method = info->GetMethod(); 1322 if (IsInZygoteExecSpace(method->GetEntryPointFromQuickCompiledCode())) { 1323 method->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge()); 1324 } 1325 // If zygote does method tracing, or in some configuration where 1326 // the JIT zygote does GC, we also need to clear the saved entry point 1327 // in the profiling info. 1328 if (IsInZygoteExecSpace(info->GetSavedEntryPoint())) { 1329 info->SetSavedEntryPoint(nullptr); 1330 } 1331 } 1332 } 1333 1334 size_t JitCodeCache::CodeCacheSizeLocked() { 1335 return used_memory_for_code_; 1336 } 1337 1338 size_t JitCodeCache::DataCacheSize() { 1339 MutexLock mu(Thread::Current(), lock_); 1340 return DataCacheSizeLocked(); 1341 } 1342 1343 size_t JitCodeCache::DataCacheSizeLocked() { 1344 return used_memory_for_data_; 1345 } 1346 1347 void JitCodeCache::ClearData(Thread* self, 1348 uint8_t* stack_map_data, 1349 uint8_t* roots_data) { 1350 DCHECK_EQ(FromStackMapToRoots(stack_map_data), roots_data); 1351 MutexLock mu(self, lock_); 1352 FreeData(reinterpret_cast<uint8_t*>(roots_data)); 1353 } 1354 1355 size_t JitCodeCache::ReserveData(Thread* self, 1356 size_t stack_map_size, 1357 size_t number_of_roots, 1358 ArtMethod* method, 1359 uint8_t** stack_map_data, 1360 uint8_t** roots_data) { 1361 size_t table_size = ComputeRootTableSize(number_of_roots); 1362 size_t size = RoundUp(stack_map_size + table_size, sizeof(void*)); 1363 uint8_t* result = nullptr; 1364 1365 { 1366 ScopedThreadSuspension sts(self, kSuspended); 1367 MutexLock mu(self, lock_); 1368 WaitForPotentialCollectionToComplete(self); 1369 result = AllocateData(size); 1370 } 1371 1372 if (result == nullptr) { 1373 // Retry. 1374 GarbageCollectCache(self); 1375 ScopedThreadSuspension sts(self, kSuspended); 1376 MutexLock mu(self, lock_); 1377 WaitForPotentialCollectionToComplete(self); 1378 result = AllocateData(size); 1379 } 1380 1381 MutexLock mu(self, lock_); 1382 histogram_stack_map_memory_use_.AddValue(size); 1383 if (size > kStackMapSizeLogThreshold) { 1384 LOG(INFO) << "JIT allocated " 1385 << PrettySize(size) 1386 << " for stack maps of " 1387 << ArtMethod::PrettyMethod(method); 1388 } 1389 if (result != nullptr) { 1390 *roots_data = result; 1391 *stack_map_data = result + table_size; 1392 FillRootTableLength(*roots_data, number_of_roots); 1393 return size; 1394 } else { 1395 *roots_data = nullptr; 1396 *stack_map_data = nullptr; 1397 return 0; 1398 } 1399 } 1400 1401 class MarkCodeClosure final : public Closure { 1402 public: 1403 MarkCodeClosure(JitCodeCache* code_cache, CodeCacheBitmap* bitmap, Barrier* barrier) 1404 : code_cache_(code_cache), bitmap_(bitmap), barrier_(barrier) {} 1405 1406 void Run(Thread* thread) override REQUIRES_SHARED(Locks::mutator_lock_) { 1407 ScopedTrace trace(__PRETTY_FUNCTION__); 1408 DCHECK(thread == Thread::Current() || thread->IsSuspended()); 1409 StackVisitor::WalkStack( 1410 [&](const art::StackVisitor* stack_visitor) { 1411 const OatQuickMethodHeader* method_header = 1412 stack_visitor->GetCurrentOatQuickMethodHeader(); 1413 if (method_header == nullptr) { 1414 return true; 1415 } 1416 const void* code = method_header->GetCode(); 1417 if (code_cache_->ContainsPc(code) && !code_cache_->IsInZygoteExecSpace(code)) { 1418 // Use the atomic set version, as multiple threads are executing this code. 1419 bitmap_->AtomicTestAndSet(FromCodeToAllocation(code)); 1420 } 1421 return true; 1422 }, 1423 thread, 1424 /* context= */ nullptr, 1425 art::StackVisitor::StackWalkKind::kSkipInlinedFrames); 1426 1427 if (kIsDebugBuild) { 1428 // The stack walking code queries the side instrumentation stack if it 1429 // sees an instrumentation exit pc, so the JIT code of methods in that stack 1430 // must have been seen. We sanity check this below. 1431 for (const instrumentation::InstrumentationStackFrame& frame 1432 : *thread->GetInstrumentationStack()) { 1433 // The 'method_' in InstrumentationStackFrame is the one that has return_pc_ in 1434 // its stack frame, it is not the method owning return_pc_. We just pass null to 1435 // LookupMethodHeader: the method is only checked against in debug builds. 1436 OatQuickMethodHeader* method_header = 1437 code_cache_->LookupMethodHeader(frame.return_pc_, /* method= */ nullptr); 1438 if (method_header != nullptr) { 1439 const void* code = method_header->GetCode(); 1440 CHECK(bitmap_->Test(FromCodeToAllocation(code))); 1441 } 1442 } 1443 } 1444 barrier_->Pass(Thread::Current()); 1445 } 1446 1447 private: 1448 JitCodeCache* const code_cache_; 1449 CodeCacheBitmap* const bitmap_; 1450 Barrier* const barrier_; 1451 }; 1452 1453 void JitCodeCache::NotifyCollectionDone(Thread* self) { 1454 collection_in_progress_ = false; 1455 lock_cond_.Broadcast(self); 1456 } 1457 1458 void JitCodeCache::SetFootprintLimit(size_t new_footprint) { 1459 size_t data_space_footprint = new_footprint / kCodeAndDataCapacityDivider; 1460 DCHECK(IsAlignedParam(data_space_footprint, kPageSize)); 1461 DCHECK_EQ(data_space_footprint * kCodeAndDataCapacityDivider, new_footprint); 1462 mspace_set_footprint_limit(data_mspace_, data_space_footprint); 1463 if (HasCodeMapping()) { 1464 ScopedCodeCacheWrite scc(this); 1465 mspace_set_footprint_limit(exec_mspace_, new_footprint - data_space_footprint); 1466 } 1467 } 1468 1469 bool JitCodeCache::IncreaseCodeCacheCapacity() { 1470 if (current_capacity_ == max_capacity_) { 1471 return false; 1472 } 1473 1474 // Double the capacity if we're below 1MB, or increase it by 1MB if 1475 // we're above. 1476 if (current_capacity_ < 1 * MB) { 1477 current_capacity_ *= 2; 1478 } else { 1479 current_capacity_ += 1 * MB; 1480 } 1481 if (current_capacity_ > max_capacity_) { 1482 current_capacity_ = max_capacity_; 1483 } 1484 1485 VLOG(jit) << "Increasing code cache capacity to " << PrettySize(current_capacity_); 1486 1487 SetFootprintLimit(current_capacity_); 1488 1489 return true; 1490 } 1491 1492 void JitCodeCache::MarkCompiledCodeOnThreadStacks(Thread* self) { 1493 Barrier barrier(0); 1494 size_t threads_running_checkpoint = 0; 1495 MarkCodeClosure closure(this, GetLiveBitmap(), &barrier); 1496 threads_running_checkpoint = Runtime::Current()->GetThreadList()->RunCheckpoint(&closure); 1497 // Now that we have run our checkpoint, move to a suspended state and wait 1498 // for other threads to run the checkpoint. 1499 ScopedThreadSuspension sts(self, kSuspended); 1500 if (threads_running_checkpoint != 0) { 1501 barrier.Increment(self, threads_running_checkpoint); 1502 } 1503 } 1504 1505 bool JitCodeCache::ShouldDoFullCollection() { 1506 if (current_capacity_ == max_capacity_) { 1507 // Always do a full collection when the code cache is full. 1508 return true; 1509 } else if (current_capacity_ < kReservedCapacity) { 1510 // Always do partial collection when the code cache size is below the reserved 1511 // capacity. 1512 return false; 1513 } else if (last_collection_increased_code_cache_) { 1514 // This time do a full collection. 1515 return true; 1516 } else { 1517 // This time do a partial collection. 1518 return false; 1519 } 1520 } 1521 1522 void JitCodeCache::GarbageCollectCache(Thread* self) { 1523 ScopedTrace trace(__FUNCTION__); 1524 // Wait for an existing collection, or let everyone know we are starting one. 1525 { 1526 ScopedThreadSuspension sts(self, kSuspended); 1527 MutexLock mu(self, lock_); 1528 if (!garbage_collect_code_) { 1529 IncreaseCodeCacheCapacity(); 1530 return; 1531 } else if (WaitForPotentialCollectionToComplete(self)) { 1532 return; 1533 } else { 1534 number_of_collections_++; 1535 live_bitmap_.reset(CodeCacheBitmap::Create( 1536 "code-cache-bitmap", 1537 reinterpret_cast<uintptr_t>(exec_pages_.Begin()), 1538 reinterpret_cast<uintptr_t>(exec_pages_.Begin() + current_capacity_ / 2))); 1539 collection_in_progress_ = true; 1540 } 1541 } 1542 1543 TimingLogger logger("JIT code cache timing logger", true, VLOG_IS_ON(jit)); 1544 { 1545 TimingLogger::ScopedTiming st("Code cache collection", &logger); 1546 1547 bool do_full_collection = false; 1548 { 1549 MutexLock mu(self, lock_); 1550 do_full_collection = ShouldDoFullCollection(); 1551 } 1552 1553 VLOG(jit) << "Do " 1554 << (do_full_collection ? "full" : "partial") 1555 << " code cache collection, code=" 1556 << PrettySize(CodeCacheSize()) 1557 << ", data=" << PrettySize(DataCacheSize()); 1558 1559 DoCollection(self, /* collect_profiling_info= */ do_full_collection); 1560 1561 VLOG(jit) << "After code cache collection, code=" 1562 << PrettySize(CodeCacheSize()) 1563 << ", data=" << PrettySize(DataCacheSize()); 1564 1565 { 1566 MutexLock mu(self, lock_); 1567 1568 // Increase the code cache only when we do partial collections. 1569 // TODO: base this strategy on how full the code cache is? 1570 if (do_full_collection) { 1571 last_collection_increased_code_cache_ = false; 1572 } else { 1573 last_collection_increased_code_cache_ = true; 1574 IncreaseCodeCacheCapacity(); 1575 } 1576 1577 bool next_collection_will_be_full = ShouldDoFullCollection(); 1578 1579 // Start polling the liveness of compiled code to prepare for the next full collection. 1580 if (next_collection_will_be_full) { 1581 // Save the entry point of methods we have compiled, and update the entry 1582 // point of those methods to the interpreter. If the method is invoked, the 1583 // interpreter will update its entry point to the compiled code and call it. 1584 for (ProfilingInfo* info : profiling_infos_) { 1585 const void* entry_point = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); 1586 if (!IsInZygoteDataSpace(info) && ContainsPc(entry_point)) { 1587 info->SetSavedEntryPoint(entry_point); 1588 // Don't call Instrumentation::UpdateMethodsCode(), as it can check the declaring 1589 // class of the method. We may be concurrently running a GC which makes accessing 1590 // the class unsafe. We know it is OK to bypass the instrumentation as we've just 1591 // checked that the current entry point is JIT compiled code. 1592 info->GetMethod()->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge()); 1593 } 1594 } 1595 1596 DCHECK(CheckLiveCompiledCodeHasProfilingInfo()); 1597 1598 // Change entry points of native methods back to the GenericJNI entrypoint. 1599 for (const auto& entry : jni_stubs_map_) { 1600 const JniStubData& data = entry.second; 1601 if (!data.IsCompiled() || IsInZygoteExecSpace(data.GetCode())) { 1602 continue; 1603 } 1604 // Make sure a single invocation of the GenericJNI trampoline tries to recompile. 1605 uint16_t new_counter = Runtime::Current()->GetJit()->HotMethodThreshold() - 1u; 1606 const OatQuickMethodHeader* method_header = 1607 OatQuickMethodHeader::FromCodePointer(data.GetCode()); 1608 for (ArtMethod* method : data.GetMethods()) { 1609 if (method->GetEntryPointFromQuickCompiledCode() == method_header->GetEntryPoint()) { 1610 // Don't call Instrumentation::UpdateMethodsCode(), same as for normal methods above. 1611 method->SetCounter(new_counter); 1612 method->SetEntryPointFromQuickCompiledCode(GetQuickGenericJniStub()); 1613 } 1614 } 1615 } 1616 } 1617 live_bitmap_.reset(nullptr); 1618 NotifyCollectionDone(self); 1619 } 1620 } 1621 Runtime::Current()->GetJit()->AddTimingLogger(logger); 1622 } 1623 1624 void JitCodeCache::RemoveUnmarkedCode(Thread* self) { 1625 ScopedTrace trace(__FUNCTION__); 1626 std::unordered_set<OatQuickMethodHeader*> method_headers; 1627 { 1628 MutexLock mu(self, lock_); 1629 ScopedCodeCacheWrite scc(this); 1630 // Iterate over all compiled code and remove entries that are not marked. 1631 for (auto it = jni_stubs_map_.begin(); it != jni_stubs_map_.end();) { 1632 JniStubData* data = &it->second; 1633 if (IsInZygoteExecSpace(data->GetCode()) || 1634 !data->IsCompiled() || 1635 GetLiveBitmap()->Test(FromCodeToAllocation(data->GetCode()))) { 1636 ++it; 1637 } else { 1638 method_headers.insert(OatQuickMethodHeader::FromCodePointer(data->GetCode())); 1639 it = jni_stubs_map_.erase(it); 1640 } 1641 } 1642 for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { 1643 const void* code_ptr = it->first; 1644 uintptr_t allocation = FromCodeToAllocation(code_ptr); 1645 if (IsInZygoteExecSpace(code_ptr) || GetLiveBitmap()->Test(allocation)) { 1646 ++it; 1647 } else { 1648 OatQuickMethodHeader* header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1649 method_headers.insert(header); 1650 it = method_code_map_.erase(it); 1651 } 1652 } 1653 } 1654 FreeAllMethodHeaders(method_headers); 1655 } 1656 1657 bool JitCodeCache::GetGarbageCollectCode() { 1658 MutexLock mu(Thread::Current(), lock_); 1659 return garbage_collect_code_; 1660 } 1661 1662 void JitCodeCache::SetGarbageCollectCode(bool value) { 1663 Thread* self = Thread::Current(); 1664 MutexLock mu(self, lock_); 1665 if (garbage_collect_code_ != value) { 1666 if (garbage_collect_code_) { 1667 // When dynamically disabling the garbage collection, we neee 1668 // to make sure that a potential current collection is finished, and also 1669 // clear the saved entry point in profiling infos to avoid dangling pointers. 1670 WaitForPotentialCollectionToComplete(self); 1671 for (ProfilingInfo* info : profiling_infos_) { 1672 info->SetSavedEntryPoint(nullptr); 1673 } 1674 } 1675 // Update the flag while holding the lock to ensure no thread will try to GC. 1676 garbage_collect_code_ = value; 1677 } 1678 } 1679 1680 void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { 1681 ScopedTrace trace(__FUNCTION__); 1682 { 1683 MutexLock mu(self, lock_); 1684 if (collect_profiling_info) { 1685 // Clear the profiling info of methods that do not have compiled code as entrypoint. 1686 // Also remove the saved entry point from the ProfilingInfo objects. 1687 for (ProfilingInfo* info : profiling_infos_) { 1688 const void* ptr = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); 1689 if (!ContainsPc(ptr) && !info->IsInUseByCompiler() && !IsInZygoteDataSpace(info)) { 1690 info->GetMethod()->SetProfilingInfo(nullptr); 1691 } 1692 1693 if (info->GetSavedEntryPoint() != nullptr) { 1694 info->SetSavedEntryPoint(nullptr); 1695 // We are going to move this method back to interpreter. Clear the counter now to 1696 // give it a chance to be hot again. 1697 ClearMethodCounter(info->GetMethod(), /*was_warm=*/ true); 1698 } 1699 } 1700 } else if (kIsDebugBuild) { 1701 // Sanity check that the profiling infos do not have a dangling entry point. 1702 for (ProfilingInfo* info : profiling_infos_) { 1703 DCHECK(!Runtime::Current()->IsZygote()); 1704 const void* entry_point = info->GetSavedEntryPoint(); 1705 DCHECK(entry_point == nullptr || IsInZygoteExecSpace(entry_point)); 1706 } 1707 } 1708 1709 // Mark compiled code that are entrypoints of ArtMethods. Compiled code that is not 1710 // an entry point is either: 1711 // - an osr compiled code, that will be removed if not in a thread call stack. 1712 // - discarded compiled code, that will be removed if not in a thread call stack. 1713 for (const auto& entry : jni_stubs_map_) { 1714 const JniStubData& data = entry.second; 1715 const void* code_ptr = data.GetCode(); 1716 if (IsInZygoteExecSpace(code_ptr)) { 1717 continue; 1718 } 1719 const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1720 for (ArtMethod* method : data.GetMethods()) { 1721 if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) { 1722 GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr)); 1723 break; 1724 } 1725 } 1726 } 1727 for (const auto& it : method_code_map_) { 1728 ArtMethod* method = it.second; 1729 const void* code_ptr = it.first; 1730 if (IsInZygoteExecSpace(code_ptr)) { 1731 continue; 1732 } 1733 const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1734 if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) { 1735 GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr)); 1736 } 1737 } 1738 1739 // Empty osr method map, as osr compiled code will be deleted (except the ones 1740 // on thread stacks). 1741 osr_code_map_.clear(); 1742 } 1743 1744 // Run a checkpoint on all threads to mark the JIT compiled code they are running. 1745 MarkCompiledCodeOnThreadStacks(self); 1746 1747 // At this point, mutator threads are still running, and entrypoints of methods can 1748 // change. We do know they cannot change to a code cache entry that is not marked, 1749 // therefore we can safely remove those entries. 1750 RemoveUnmarkedCode(self); 1751 1752 if (collect_profiling_info) { 1753 MutexLock mu(self, lock_); 1754 // Free all profiling infos of methods not compiled nor being compiled. 1755 auto profiling_kept_end = std::remove_if(profiling_infos_.begin(), profiling_infos_.end(), 1756 [this] (ProfilingInfo* info) NO_THREAD_SAFETY_ANALYSIS { 1757 const void* ptr = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); 1758 // We have previously cleared the ProfilingInfo pointer in the ArtMethod in the hope 1759 // that the compiled code would not get revived. As mutator threads run concurrently, 1760 // they may have revived the compiled code, and now we are in the situation where 1761 // a method has compiled code but no ProfilingInfo. 1762 // We make sure compiled methods have a ProfilingInfo object. It is needed for 1763 // code cache collection. 1764 if (ContainsPc(ptr) && 1765 info->GetMethod()->GetProfilingInfo(kRuntimePointerSize) == nullptr) { 1766 info->GetMethod()->SetProfilingInfo(info); 1767 } else if (info->GetMethod()->GetProfilingInfo(kRuntimePointerSize) != info) { 1768 // No need for this ProfilingInfo object anymore. 1769 FreeData(reinterpret_cast<uint8_t*>(info)); 1770 return true; 1771 } 1772 return false; 1773 }); 1774 profiling_infos_.erase(profiling_kept_end, profiling_infos_.end()); 1775 DCHECK(CheckLiveCompiledCodeHasProfilingInfo()); 1776 } 1777 } 1778 1779 bool JitCodeCache::CheckLiveCompiledCodeHasProfilingInfo() { 1780 ScopedTrace trace(__FUNCTION__); 1781 // Check that methods we have compiled do have a ProfilingInfo object. We would 1782 // have memory leaks of compiled code otherwise. 1783 for (const auto& it : method_code_map_) { 1784 ArtMethod* method = it.second; 1785 if (method->GetProfilingInfo(kRuntimePointerSize) == nullptr) { 1786 const void* code_ptr = it.first; 1787 const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1788 if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) { 1789 // If the code is not dead, then we have a problem. Note that this can even 1790 // happen just after a collection, as mutator threads are running in parallel 1791 // and could deoptimize an existing compiled code. 1792 return false; 1793 } 1794 } 1795 } 1796 return true; 1797 } 1798 1799 OatQuickMethodHeader* JitCodeCache::LookupMethodHeader(uintptr_t pc, ArtMethod* method) { 1800 static_assert(kRuntimeISA != InstructionSet::kThumb2, "kThumb2 cannot be a runtime ISA"); 1801 if (kRuntimeISA == InstructionSet::kArm) { 1802 // On Thumb-2, the pc is offset by one. 1803 --pc; 1804 } 1805 if (!ContainsPc(reinterpret_cast<const void*>(pc))) { 1806 return nullptr; 1807 } 1808 1809 if (!kIsDebugBuild) { 1810 // Called with null `method` only from MarkCodeClosure::Run() in debug build. 1811 CHECK(method != nullptr); 1812 } 1813 1814 MutexLock mu(Thread::Current(), lock_); 1815 OatQuickMethodHeader* method_header = nullptr; 1816 ArtMethod* found_method = nullptr; // Only for DCHECK(), not for JNI stubs. 1817 if (method != nullptr && UNLIKELY(method->IsNative())) { 1818 auto it = jni_stubs_map_.find(JniStubKey(method)); 1819 if (it == jni_stubs_map_.end() || !ContainsElement(it->second.GetMethods(), method)) { 1820 return nullptr; 1821 } 1822 const void* code_ptr = it->second.GetCode(); 1823 method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1824 if (!method_header->Contains(pc)) { 1825 return nullptr; 1826 } 1827 } else { 1828 auto it = method_code_map_.lower_bound(reinterpret_cast<const void*>(pc)); 1829 if (it != method_code_map_.begin()) { 1830 --it; 1831 const void* code_ptr = it->first; 1832 if (OatQuickMethodHeader::FromCodePointer(code_ptr)->Contains(pc)) { 1833 method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); 1834 found_method = it->second; 1835 } 1836 } 1837 if (method_header == nullptr && method == nullptr) { 1838 // Scan all compiled JNI stubs as well. This slow search is used only 1839 // for checks in debug build, for release builds the `method` is not null. 1840 for (auto&& entry : jni_stubs_map_) { 1841 const JniStubData& data = entry.second; 1842 if (data.IsCompiled() && 1843 OatQuickMethodHeader::FromCodePointer(data.GetCode())->Contains(pc)) { 1844 method_header = OatQuickMethodHeader::FromCodePointer(data.GetCode()); 1845 } 1846 } 1847 } 1848 if (method_header == nullptr) { 1849 return nullptr; 1850 } 1851 } 1852 1853 if (kIsDebugBuild && method != nullptr && !method->IsNative()) { 1854 // When we are walking the stack to redefine classes and creating obsolete methods it is 1855 // possible that we might have updated the method_code_map by making this method obsolete in a 1856 // previous frame. Therefore we should just check that the non-obsolete version of this method 1857 // is the one we expect. We change to the non-obsolete versions in the error message since the 1858 // obsolete version of the method might not be fully initialized yet. This situation can only 1859 // occur when we are in the process of allocating and setting up obsolete methods. Otherwise 1860 // method and it->second should be identical. (See openjdkjvmti/ti_redefine.cc for more 1861 // information.) 1862 DCHECK_EQ(found_method->GetNonObsoleteMethod(), method->GetNonObsoleteMethod()) 1863 << ArtMethod::PrettyMethod(method->GetNonObsoleteMethod()) << " " 1864 << ArtMethod::PrettyMethod(found_method->GetNonObsoleteMethod()) << " " 1865 << std::hex << pc; 1866 } 1867 return method_header; 1868 } 1869 1870 OatQuickMethodHeader* JitCodeCache::LookupOsrMethodHeader(ArtMethod* method) { 1871 MutexLock mu(Thread::Current(), lock_); 1872 auto it = osr_code_map_.find(method); 1873 if (it == osr_code_map_.end()) { 1874 return nullptr; 1875 } 1876 return OatQuickMethodHeader::FromCodePointer(it->second); 1877 } 1878 1879 ProfilingInfo* JitCodeCache::AddProfilingInfo(Thread* self, 1880 ArtMethod* method, 1881 const std::vector<uint32_t>& entries, 1882 bool retry_allocation) 1883 // No thread safety analysis as we are using TryLock/Unlock explicitly. 1884 NO_THREAD_SAFETY_ANALYSIS { 1885 ProfilingInfo* info = nullptr; 1886 if (!retry_allocation) { 1887 // If we are allocating for the interpreter, just try to lock, to avoid 1888 // lock contention with the JIT. 1889 if (lock_.ExclusiveTryLock(self)) { 1890 info = AddProfilingInfoInternal(self, method, entries); 1891 lock_.ExclusiveUnlock(self); 1892 } 1893 } else { 1894 { 1895 MutexLock mu(self, lock_); 1896 info = AddProfilingInfoInternal(self, method, entries); 1897 } 1898 1899 if (info == nullptr) { 1900 GarbageCollectCache(self); 1901 MutexLock mu(self, lock_); 1902 info = AddProfilingInfoInternal(self, method, entries); 1903 } 1904 } 1905 return info; 1906 } 1907 1908 ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self ATTRIBUTE_UNUSED, 1909 ArtMethod* method, 1910 const std::vector<uint32_t>& entries) { 1911 size_t profile_info_size = RoundUp( 1912 sizeof(ProfilingInfo) + sizeof(InlineCache) * entries.size(), 1913 sizeof(void*)); 1914 1915 // Check whether some other thread has concurrently created it. 1916 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 1917 if (info != nullptr) { 1918 return info; 1919 } 1920 1921 uint8_t* data = AllocateData(profile_info_size); 1922 if (data == nullptr) { 1923 return nullptr; 1924 } 1925 info = new (data) ProfilingInfo(method, entries); 1926 1927 // Make sure other threads see the data in the profiling info object before the 1928 // store in the ArtMethod's ProfilingInfo pointer. 1929 std::atomic_thread_fence(std::memory_order_release); 1930 1931 method->SetProfilingInfo(info); 1932 profiling_infos_.push_back(info); 1933 histogram_profiling_info_memory_use_.AddValue(profile_info_size); 1934 return info; 1935 } 1936 1937 // NO_THREAD_SAFETY_ANALYSIS as this is called from mspace code, at which point the lock 1938 // is already held. 1939 void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_SAFETY_ANALYSIS { 1940 if (mspace == exec_mspace_) { 1941 DCHECK(exec_mspace_ != nullptr); 1942 const MemMap* const code_pages = GetUpdatableCodeMapping(); 1943 void* result = code_pages->Begin() + exec_end_; 1944 exec_end_ += increment; 1945 return result; 1946 } else { 1947 DCHECK_EQ(data_mspace_, mspace); 1948 void* result = data_pages_.Begin() + data_end_; 1949 data_end_ += increment; 1950 return result; 1951 } 1952 } 1953 1954 void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_locations, 1955 std::vector<ProfileMethodInfo>& methods) { 1956 Thread* self = Thread::Current(); 1957 WaitUntilInlineCacheAccessible(self); 1958 MutexLock mu(self, lock_); 1959 ScopedTrace trace(__FUNCTION__); 1960 uint16_t jit_compile_threshold = Runtime::Current()->GetJITOptions()->GetCompileThreshold(); 1961 for (const ProfilingInfo* info : profiling_infos_) { 1962 ArtMethod* method = info->GetMethod(); 1963 const DexFile* dex_file = method->GetDexFile(); 1964 const std::string base_location = DexFileLoader::GetBaseLocation(dex_file->GetLocation()); 1965 if (!ContainsElement(dex_base_locations, base_location)) { 1966 // Skip dex files which are not profiled. 1967 continue; 1968 } 1969 std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches; 1970 1971 // If the method didn't reach the compilation threshold don't save the inline caches. 1972 // They might be incomplete and cause unnecessary deoptimizations. 1973 // If the inline cache is empty the compiler will generate a regular invoke virtual/interface. 1974 if (method->GetCounter() < jit_compile_threshold) { 1975 methods.emplace_back(/*ProfileMethodInfo*/ 1976 MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches); 1977 continue; 1978 } 1979 1980 for (size_t i = 0; i < info->number_of_inline_caches_; ++i) { 1981 std::vector<TypeReference> profile_classes; 1982 const InlineCache& cache = info->cache_[i]; 1983 ArtMethod* caller = info->GetMethod(); 1984 bool is_missing_types = false; 1985 for (size_t k = 0; k < InlineCache::kIndividualCacheSize; k++) { 1986 mirror::Class* cls = cache.classes_[k].Read(); 1987 if (cls == nullptr) { 1988 break; 1989 } 1990 1991 // Check if the receiver is in the boot class path or if it's in the 1992 // same class loader as the caller. If not, skip it, as there is not 1993 // much we can do during AOT. 1994 if (!cls->IsBootStrapClassLoaded() && 1995 caller->GetClassLoader() != cls->GetClassLoader()) { 1996 is_missing_types = true; 1997 continue; 1998 } 1999 2000 const DexFile* class_dex_file = nullptr; 2001 dex::TypeIndex type_index; 2002 2003 if (cls->GetDexCache() == nullptr) { 2004 DCHECK(cls->IsArrayClass()) << cls->PrettyClass(); 2005 // Make a best effort to find the type index in the method's dex file. 2006 // We could search all open dex files but that might turn expensive 2007 // and probably not worth it. 2008 class_dex_file = dex_file; 2009 type_index = cls->FindTypeIndexInOtherDexFile(*dex_file); 2010 } else { 2011 class_dex_file = &(cls->GetDexFile()); 2012 type_index = cls->GetDexTypeIndex(); 2013 } 2014 if (!type_index.IsValid()) { 2015 // Could be a proxy class or an array for which we couldn't find the type index. 2016 is_missing_types = true; 2017 continue; 2018 } 2019 if (ContainsElement(dex_base_locations, 2020 DexFileLoader::GetBaseLocation(class_dex_file->GetLocation()))) { 2021 // Only consider classes from the same apk (including multidex). 2022 profile_classes.emplace_back(/*ProfileMethodInfo::ProfileClassReference*/ 2023 class_dex_file, type_index); 2024 } else { 2025 is_missing_types = true; 2026 } 2027 } 2028 if (!profile_classes.empty()) { 2029 inline_caches.emplace_back(/*ProfileMethodInfo::ProfileInlineCache*/ 2030 cache.dex_pc_, is_missing_types, profile_classes); 2031 } 2032 } 2033 methods.emplace_back(/*ProfileMethodInfo*/ 2034 MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches); 2035 } 2036 } 2037 2038 bool JitCodeCache::IsOsrCompiled(ArtMethod* method) { 2039 MutexLock mu(Thread::Current(), lock_); 2040 return osr_code_map_.find(method) != osr_code_map_.end(); 2041 } 2042 2043 bool JitCodeCache::NotifyCompilationOf(ArtMethod* method, Thread* self, bool osr) { 2044 if (!osr && ContainsPc(method->GetEntryPointFromQuickCompiledCode())) { 2045 return false; 2046 } 2047 2048 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 2049 if (class_linker->IsQuickResolutionStub(method->GetEntryPointFromQuickCompiledCode())) { 2050 if (!Runtime::Current()->IsUsingApexBootImageLocation() || !Runtime::Current()->IsZygote()) { 2051 // Unless we're running as zygote in the jitzygote experiment, we currently don't save 2052 // the JIT compiled code if we cannot update the entrypoint due to having the resolution stub. 2053 VLOG(jit) << "Not compiling " 2054 << method->PrettyMethod() 2055 << " because it has the resolution stub"; 2056 // Give it a new chance to be hot. 2057 ClearMethodCounter(method, /*was_warm=*/ false); 2058 return false; 2059 } 2060 } 2061 2062 MutexLock mu(self, lock_); 2063 if (osr && (osr_code_map_.find(method) != osr_code_map_.end())) { 2064 return false; 2065 } 2066 2067 if (UNLIKELY(method->IsNative())) { 2068 JniStubKey key(method); 2069 auto it = jni_stubs_map_.find(key); 2070 bool new_compilation = false; 2071 if (it == jni_stubs_map_.end()) { 2072 // Create a new entry to mark the stub as being compiled. 2073 it = jni_stubs_map_.Put(key, JniStubData{}); 2074 new_compilation = true; 2075 } 2076 JniStubData* data = &it->second; 2077 data->AddMethod(method); 2078 if (data->IsCompiled()) { 2079 OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(data->GetCode()); 2080 const void* entrypoint = method_header->GetEntryPoint(); 2081 // Update also entrypoints of other methods held by the JniStubData. 2082 // We could simply update the entrypoint of `method` but if the last JIT GC has 2083 // changed these entrypoints to GenericJNI in preparation for a full GC, we may 2084 // as well change them back as this stub shall not be collected anyway and this 2085 // can avoid a few expensive GenericJNI calls. 2086 instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation(); 2087 for (ArtMethod* m : data->GetMethods()) { 2088 // Call the dedicated method instead of the more generic UpdateMethodsCode, because 2089 // `m` might be in the process of being deleted. 2090 if (!class_linker->IsQuickResolutionStub(m->GetEntryPointFromQuickCompiledCode())) { 2091 instrumentation->UpdateNativeMethodsCodeToJitCode(m, entrypoint); 2092 } 2093 } 2094 if (collection_in_progress_) { 2095 if (!IsInZygoteExecSpace(data->GetCode())) { 2096 GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(data->GetCode())); 2097 } 2098 } 2099 } 2100 return new_compilation; 2101 } else { 2102 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 2103 if (info == nullptr) { 2104 VLOG(jit) << method->PrettyMethod() << " needs a ProfilingInfo to be compiled"; 2105 // Because the counter is not atomic, there are some rare cases where we may not hit the 2106 // threshold for creating the ProfilingInfo. Reset the counter now to "correct" this. 2107 ClearMethodCounter(method, /*was_warm=*/ false); 2108 return false; 2109 } 2110 2111 if (info->IsMethodBeingCompiled(osr)) { 2112 return false; 2113 } 2114 2115 info->SetIsMethodBeingCompiled(true, osr); 2116 return true; 2117 } 2118 } 2119 2120 ProfilingInfo* JitCodeCache::NotifyCompilerUse(ArtMethod* method, Thread* self) { 2121 MutexLock mu(self, lock_); 2122 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 2123 if (info != nullptr) { 2124 if (!info->IncrementInlineUse()) { 2125 // Overflow of inlining uses, just bail. 2126 return nullptr; 2127 } 2128 } 2129 return info; 2130 } 2131 2132 void JitCodeCache::DoneCompilerUse(ArtMethod* method, Thread* self) { 2133 MutexLock mu(self, lock_); 2134 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 2135 DCHECK(info != nullptr); 2136 info->DecrementInlineUse(); 2137 } 2138 2139 void JitCodeCache::DoneCompiling(ArtMethod* method, Thread* self, bool osr) { 2140 DCHECK_EQ(Thread::Current(), self); 2141 MutexLock mu(self, lock_); 2142 if (UNLIKELY(method->IsNative())) { 2143 auto it = jni_stubs_map_.find(JniStubKey(method)); 2144 DCHECK(it != jni_stubs_map_.end()); 2145 JniStubData* data = &it->second; 2146 DCHECK(ContainsElement(data->GetMethods(), method)); 2147 if (UNLIKELY(!data->IsCompiled())) { 2148 // Failed to compile; the JNI compiler never fails, but the cache may be full. 2149 jni_stubs_map_.erase(it); // Remove the entry added in NotifyCompilationOf(). 2150 } // else CommitCodeInternal() updated entrypoints of all methods in the JniStubData. 2151 } else { 2152 ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); 2153 DCHECK(info->IsMethodBeingCompiled(osr)); 2154 info->SetIsMethodBeingCompiled(false, osr); 2155 } 2156 } 2157 2158 void JitCodeCache::InvalidateCompiledCodeFor(ArtMethod* method, 2159 const OatQuickMethodHeader* header) { 2160 DCHECK(!method->IsNative()); 2161 ProfilingInfo* profiling_info = method->GetProfilingInfo(kRuntimePointerSize); 2162 const void* method_entrypoint = method->GetEntryPointFromQuickCompiledCode(); 2163 if ((profiling_info != nullptr) && 2164 (profiling_info->GetSavedEntryPoint() == header->GetEntryPoint())) { 2165 // When instrumentation is set, the actual entrypoint is the one in the profiling info. 2166 method_entrypoint = profiling_info->GetSavedEntryPoint(); 2167 // Prevent future uses of the compiled code. 2168 profiling_info->SetSavedEntryPoint(nullptr); 2169 } 2170 2171 // Clear the method counter if we are running jitted code since we might want to jit this again in 2172 // the future. 2173 if (method_entrypoint == header->GetEntryPoint()) { 2174 // The entrypoint is the one to invalidate, so we just update it to the interpreter entry point 2175 // and clear the counter to get the method Jitted again. 2176 Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( 2177 method, GetQuickToInterpreterBridge()); 2178 ClearMethodCounter(method, /*was_warm=*/ profiling_info != nullptr); 2179 } else { 2180 MutexLock mu(Thread::Current(), lock_); 2181 auto it = osr_code_map_.find(method); 2182 if (it != osr_code_map_.end() && OatQuickMethodHeader::FromCodePointer(it->second) == header) { 2183 // Remove the OSR method, to avoid using it again. 2184 osr_code_map_.erase(it); 2185 } 2186 } 2187 } 2188 2189 uint8_t* JitCodeCache::AllocateCode(size_t allocation_size) { 2190 // Each allocation should be on its own set of cache lines. The allocation must be large enough 2191 // for header, code, and any padding. 2192 size_t alignment = GetJitCodeAlignment(); 2193 uint8_t* result = reinterpret_cast<uint8_t*>( 2194 mspace_memalign(exec_mspace_, alignment, allocation_size)); 2195 size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); 2196 // Ensure the header ends up at expected instruction alignment. 2197 DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment); 2198 used_memory_for_code_ += mspace_usable_size(result); 2199 return result; 2200 } 2201 2202 void JitCodeCache::FreeCode(uint8_t* code) { 2203 if (IsInZygoteExecSpace(code)) { 2204 // No need to free, this is shared memory. 2205 return; 2206 } 2207 used_memory_for_code_ -= mspace_usable_size(code); 2208 mspace_free(exec_mspace_, code); 2209 } 2210 2211 uint8_t* JitCodeCache::AllocateData(size_t data_size) { 2212 void* result = mspace_malloc(data_mspace_, data_size); 2213 used_memory_for_data_ += mspace_usable_size(result); 2214 return reinterpret_cast<uint8_t*>(result); 2215 } 2216 2217 void JitCodeCache::FreeData(uint8_t* data) { 2218 if (IsInZygoteDataSpace(data)) { 2219 // No need to free, this is shared memory. 2220 return; 2221 } 2222 used_memory_for_data_ -= mspace_usable_size(data); 2223 mspace_free(data_mspace_, data); 2224 } 2225 2226 void JitCodeCache::Dump(std::ostream& os) { 2227 MutexLock mu(Thread::Current(), lock_); 2228 os << "Current JIT code cache size: " << PrettySize(used_memory_for_code_) << "\n" 2229 << "Current JIT data cache size: " << PrettySize(used_memory_for_data_) << "\n" 2230 << "Current JIT mini-debug-info size: " << PrettySize(GetJitMiniDebugInfoMemUsage()) << "\n" 2231 << "Current JIT capacity: " << PrettySize(current_capacity_) << "\n" 2232 << "Current number of JIT JNI stub entries: " << jni_stubs_map_.size() << "\n" 2233 << "Current number of JIT code cache entries: " << method_code_map_.size() << "\n" 2234 << "Total number of JIT compilations: " << number_of_compilations_ << "\n" 2235 << "Total number of JIT compilations for on stack replacement: " 2236 << number_of_osr_compilations_ << "\n" 2237 << "Total number of JIT code cache collections: " << number_of_collections_ << std::endl; 2238 histogram_stack_map_memory_use_.PrintMemoryUse(os); 2239 histogram_code_memory_use_.PrintMemoryUse(os); 2240 histogram_profiling_info_memory_use_.PrintMemoryUse(os); 2241 } 2242 2243 void JitCodeCache::PostForkChildAction(bool is_system_server, bool is_zygote) { 2244 if (is_zygote) { 2245 // Don't transition if this is for a child zygote. 2246 return; 2247 } 2248 MutexLock mu(Thread::Current(), lock_); 2249 2250 zygote_data_pages_ = std::move(data_pages_); 2251 zygote_exec_pages_ = std::move(exec_pages_); 2252 zygote_data_mspace_ = data_mspace_; 2253 zygote_exec_mspace_ = exec_mspace_; 2254 2255 size_t initial_capacity = Runtime::Current()->GetJITOptions()->GetCodeCacheInitialCapacity(); 2256 size_t max_capacity = Runtime::Current()->GetJITOptions()->GetCodeCacheMaxCapacity(); 2257 2258 InitializeState(initial_capacity, max_capacity); 2259 2260 std::string error_msg; 2261 if (!InitializeMappings(/* rwx_memory_allowed= */ !is_system_server, is_zygote, &error_msg)) { 2262 LOG(WARNING) << "Could not reset JIT state after zygote fork: " << error_msg; 2263 return; 2264 } 2265 2266 InitializeSpaces(); 2267 } 2268 2269 } // namespace jit 2270 } // namespace art 2271