1 /* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_RUNTIME_GC_HEAP_H_ 18 #define ART_RUNTIME_GC_HEAP_H_ 19 20 #include <iosfwd> 21 #include <string> 22 #include <unordered_set> 23 #include <vector> 24 25 #include <android-base/logging.h> 26 27 #include "allocator_type.h" 28 #include "base/atomic.h" 29 #include "base/macros.h" 30 #include "base/mutex.h" 31 #include "base/runtime_debug.h" 32 #include "base/safe_map.h" 33 #include "base/time_utils.h" 34 #include "gc/collector/gc_type.h" 35 #include "gc/collector/iteration.h" 36 #include "gc/collector_type.h" 37 #include "gc/gc_cause.h" 38 #include "gc/space/image_space_loading_order.h" 39 #include "gc/space/large_object_space.h" 40 #include "handle.h" 41 #include "obj_ptr.h" 42 #include "offsets.h" 43 #include "process_state.h" 44 #include "read_barrier_config.h" 45 #include "runtime_globals.h" 46 #include "verify_object.h" 47 48 namespace art { 49 50 class ConditionVariable; 51 enum class InstructionSet; 52 class IsMarkedVisitor; 53 class Mutex; 54 class RootVisitor; 55 class StackVisitor; 56 class Thread; 57 class ThreadPool; 58 class TimingLogger; 59 class VariableSizedHandleScope; 60 61 namespace mirror { 62 class Class; 63 class Object; 64 } // namespace mirror 65 66 namespace gc { 67 68 class AllocationListener; 69 class AllocRecordObjectMap; 70 class GcPauseListener; 71 class ReferenceProcessor; 72 class TaskProcessor; 73 class Verification; 74 75 namespace accounting { 76 template <typename T> class AtomicStack; 77 typedef AtomicStack<mirror::Object> ObjectStack; 78 class CardTable; 79 class HeapBitmap; 80 class ModUnionTable; 81 class ReadBarrierTable; 82 class RememberedSet; 83 } // namespace accounting 84 85 namespace collector { 86 class ConcurrentCopying; 87 class GarbageCollector; 88 class MarkSweep; 89 class SemiSpace; 90 } // namespace collector 91 92 namespace allocator { 93 class RosAlloc; 94 } // namespace allocator 95 96 namespace space { 97 class AllocSpace; 98 class BumpPointerSpace; 99 class ContinuousMemMapAllocSpace; 100 class DiscontinuousSpace; 101 class DlMallocSpace; 102 class ImageSpace; 103 class LargeObjectSpace; 104 class MallocSpace; 105 class RegionSpace; 106 class RosAllocSpace; 107 class Space; 108 class ZygoteSpace; 109 } // namespace space 110 111 enum HomogeneousSpaceCompactResult { 112 // Success. 113 kSuccess, 114 // Reject due to disabled moving GC. 115 kErrorReject, 116 // Unsupported due to the current configuration. 117 kErrorUnsupported, 118 // System is shutting down. 119 kErrorVMShuttingDown, 120 }; 121 122 // If true, use rosalloc/RosAllocSpace instead of dlmalloc/DlMallocSpace 123 static constexpr bool kUseRosAlloc = true; 124 125 // If true, use thread-local allocation stack. 126 static constexpr bool kUseThreadLocalAllocationStack = true; 127 128 class Heap { 129 public: 130 static constexpr size_t kDefaultStartingSize = kPageSize; 131 static constexpr size_t kDefaultInitialSize = 2 * MB; 132 static constexpr size_t kDefaultMaximumSize = 256 * MB; 133 static constexpr size_t kDefaultNonMovingSpaceCapacity = 64 * MB; 134 static constexpr size_t kDefaultMaxFree = 2 * MB; 135 static constexpr size_t kDefaultMinFree = kDefaultMaxFree / 4; 136 static constexpr size_t kDefaultLongPauseLogThreshold = MsToNs(5); 137 static constexpr size_t kDefaultLongGCLogThreshold = MsToNs(100); 138 static constexpr size_t kDefaultTLABSize = 32 * KB; 139 static constexpr double kDefaultTargetUtilization = 0.5; 140 static constexpr double kDefaultHeapGrowthMultiplier = 2.0; 141 // Primitive arrays larger than this size are put in the large object space. 142 static constexpr size_t kMinLargeObjectThreshold = 3 * kPageSize; 143 static constexpr size_t kDefaultLargeObjectThreshold = kMinLargeObjectThreshold; 144 // Whether or not parallel GC is enabled. If not, then we never create the thread pool. 145 static constexpr bool kDefaultEnableParallelGC = false; 146 static uint8_t* const kPreferredAllocSpaceBegin; 147 148 // Whether or not we use the free list large object space. Only use it if USE_ART_LOW_4G_ALLOCATOR 149 // since this means that we have to use the slow msync loop in MemMap::MapAnonymous. 150 static constexpr space::LargeObjectSpaceType kDefaultLargeObjectSpaceType = 151 USE_ART_LOW_4G_ALLOCATOR ? 152 space::LargeObjectSpaceType::kFreeList 153 : space::LargeObjectSpaceType::kMap; 154 155 // Used so that we don't overflow the allocation time atomic integer. 156 static constexpr size_t kTimeAdjust = 1024; 157 158 // Client should call NotifyNativeAllocation every kNotifyNativeInterval allocations. 159 // Should be chosen so that time_to_call_mallinfo / kNotifyNativeInterval is on the same order 160 // as object allocation time. time_to_call_mallinfo seems to be on the order of 1 usec. 161 #ifdef __ANDROID__ 162 static constexpr uint32_t kNotifyNativeInterval = 32; 163 #else 164 // Some host mallinfo() implementations are slow. And memory is less scarce. 165 static constexpr uint32_t kNotifyNativeInterval = 128; 166 #endif 167 168 // RegisterNativeAllocation checks immediately whether GC is needed if size exceeds the 169 // following. kCheckImmediatelyThreshold * kNotifyNativeInterval should be small enough to 170 // make it safe to allocate that many bytes between checks. 171 static constexpr size_t kCheckImmediatelyThreshold = 300000; 172 173 // How often we allow heap trimming to happen (nanoseconds). 174 static constexpr uint64_t kHeapTrimWait = MsToNs(5000); 175 // How long we wait after a transition request to perform a collector transition (nanoseconds). 176 static constexpr uint64_t kCollectorTransitionWait = MsToNs(5000); 177 // Whether the transition-wait applies or not. Zero wait will stress the 178 // transition code and collector, but increases jank probability. 179 DECLARE_RUNTIME_DEBUG_FLAG(kStressCollectorTransition); 180 181 // Create a heap with the requested sizes. The possible empty 182 // image_file_names names specify Spaces to load based on 183 // ImageWriter output. 184 Heap(size_t initial_size, 185 size_t growth_limit, 186 size_t min_free, 187 size_t max_free, 188 double target_utilization, 189 double foreground_heap_growth_multiplier, 190 size_t capacity, 191 size_t non_moving_space_capacity, 192 const std::vector<std::string>& boot_class_path, 193 const std::vector<std::string>& boot_class_path_locations, 194 const std::string& image_file_name, 195 InstructionSet image_instruction_set, 196 CollectorType foreground_collector_type, 197 CollectorType background_collector_type, 198 space::LargeObjectSpaceType large_object_space_type, 199 size_t large_object_threshold, 200 size_t parallel_gc_threads, 201 size_t conc_gc_threads, 202 bool low_memory_mode, 203 size_t long_pause_threshold, 204 size_t long_gc_threshold, 205 bool ignore_target_footprint, 206 bool use_tlab, 207 bool verify_pre_gc_heap, 208 bool verify_pre_sweeping_heap, 209 bool verify_post_gc_heap, 210 bool verify_pre_gc_rosalloc, 211 bool verify_pre_sweeping_rosalloc, 212 bool verify_post_gc_rosalloc, 213 bool gc_stress_mode, 214 bool measure_gc_performance, 215 bool use_homogeneous_space_compaction, 216 bool use_generational_cc, 217 uint64_t min_interval_homogeneous_space_compaction_by_oom, 218 bool dump_region_info_before_gc, 219 bool dump_region_info_after_gc, 220 space::ImageSpaceLoadingOrder image_space_loading_order); 221 222 ~Heap(); 223 224 // Allocates and initializes storage for an object instance. 225 template <bool kInstrumented, typename PreFenceVisitor> 226 mirror::Object* AllocObject(Thread* self, 227 ObjPtr<mirror::Class> klass, 228 size_t num_bytes, 229 const PreFenceVisitor& pre_fence_visitor) 230 REQUIRES_SHARED(Locks::mutator_lock_) 231 REQUIRES(!*gc_complete_lock_, 232 !*pending_task_lock_, 233 !*backtrace_lock_, 234 !Roles::uninterruptible_) { 235 return AllocObjectWithAllocator<kInstrumented, true>(self, 236 klass, 237 num_bytes, 238 GetCurrentAllocator(), 239 pre_fence_visitor); 240 } 241 242 template <bool kInstrumented, typename PreFenceVisitor> 243 mirror::Object* AllocNonMovableObject(Thread* self, 244 ObjPtr<mirror::Class> klass, 245 size_t num_bytes, 246 const PreFenceVisitor& pre_fence_visitor) 247 REQUIRES_SHARED(Locks::mutator_lock_) 248 REQUIRES(!*gc_complete_lock_, 249 !*pending_task_lock_, 250 !*backtrace_lock_, 251 !Roles::uninterruptible_) { 252 return AllocObjectWithAllocator<kInstrumented, true>(self, 253 klass, 254 num_bytes, 255 GetCurrentNonMovingAllocator(), 256 pre_fence_visitor); 257 } 258 259 template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor> 260 ALWAYS_INLINE mirror::Object* AllocObjectWithAllocator(Thread* self, 261 ObjPtr<mirror::Class> klass, 262 size_t byte_count, 263 AllocatorType allocator, 264 const PreFenceVisitor& pre_fence_visitor) 265 REQUIRES_SHARED(Locks::mutator_lock_) 266 REQUIRES(!*gc_complete_lock_, 267 !*pending_task_lock_, 268 !*backtrace_lock_, 269 !Roles::uninterruptible_); 270 271 AllocatorType GetCurrentAllocator() const { 272 return current_allocator_; 273 } 274 275 AllocatorType GetCurrentNonMovingAllocator() const { 276 return current_non_moving_allocator_; 277 } 278 279 // Visit all of the live objects in the heap. 280 template <typename Visitor> 281 ALWAYS_INLINE void VisitObjects(Visitor&& visitor) 282 REQUIRES_SHARED(Locks::mutator_lock_) 283 REQUIRES(!Locks::heap_bitmap_lock_, !*gc_complete_lock_); 284 template <typename Visitor> 285 ALWAYS_INLINE void VisitObjectsPaused(Visitor&& visitor) 286 REQUIRES(Locks::mutator_lock_, !Locks::heap_bitmap_lock_, !*gc_complete_lock_); 287 288 void CheckPreconditionsForAllocObject(ObjPtr<mirror::Class> c, size_t byte_count) 289 REQUIRES_SHARED(Locks::mutator_lock_); 290 291 // Inform the garbage collector of a non-malloc allocated native memory that might become 292 // reclaimable in the future as a result of Java garbage collection. 293 void RegisterNativeAllocation(JNIEnv* env, size_t bytes) 294 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 295 void RegisterNativeFree(JNIEnv* env, size_t bytes); 296 297 // Notify the garbage collector of malloc allocations that might be reclaimable 298 // as a result of Java garbage collection. Each such call represents approximately 299 // kNotifyNativeInterval such allocations. 300 void NotifyNativeAllocations(JNIEnv* env) 301 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 302 303 uint32_t GetNotifyNativeInterval() { 304 return kNotifyNativeInterval; 305 } 306 307 // Change the allocator, updates entrypoints. 308 void ChangeAllocator(AllocatorType allocator) 309 REQUIRES(Locks::mutator_lock_, !Locks::runtime_shutdown_lock_); 310 311 // Transition the garbage collector during runtime, may copy objects from one space to another. 312 void TransitionCollector(CollectorType collector_type) REQUIRES(!*gc_complete_lock_); 313 314 // Change the collector to be one of the possible options (MS, CMS, SS). 315 void ChangeCollector(CollectorType collector_type) 316 REQUIRES(Locks::mutator_lock_); 317 318 // The given reference is believed to be to an object in the Java heap, check the soundness of it. 319 // TODO: NO_THREAD_SAFETY_ANALYSIS since we call this everywhere and it is impossible to find a 320 // proper lock ordering for it. 321 void VerifyObjectBody(ObjPtr<mirror::Object> o) NO_THREAD_SAFETY_ANALYSIS; 322 323 // Check sanity of all live references. 324 void VerifyHeap() REQUIRES(!Locks::heap_bitmap_lock_); 325 // Returns how many failures occured. 326 size_t VerifyHeapReferences(bool verify_referents = true) 327 REQUIRES(Locks::mutator_lock_, !*gc_complete_lock_); 328 bool VerifyMissingCardMarks() 329 REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); 330 331 // A weaker test than IsLiveObject or VerifyObject that doesn't require the heap lock, 332 // and doesn't abort on error, allowing the caller to report more 333 // meaningful diagnostics. 334 bool IsValidObjectAddress(const void* obj) const REQUIRES_SHARED(Locks::mutator_lock_); 335 336 // Faster alternative to IsHeapAddress since finding if an object is in the large object space is 337 // very slow. 338 bool IsNonDiscontinuousSpaceHeapAddress(const void* addr) const 339 REQUIRES_SHARED(Locks::mutator_lock_); 340 341 // Returns true if 'obj' is a live heap object, false otherwise (including for invalid addresses). 342 // Requires the heap lock to be held. 343 bool IsLiveObjectLocked(ObjPtr<mirror::Object> obj, 344 bool search_allocation_stack = true, 345 bool search_live_stack = true, 346 bool sorted = false) 347 REQUIRES_SHARED(Locks::heap_bitmap_lock_, Locks::mutator_lock_); 348 349 // Returns true if there is any chance that the object (obj) will move. 350 bool IsMovableObject(ObjPtr<mirror::Object> obj) const REQUIRES_SHARED(Locks::mutator_lock_); 351 352 // Enables us to compacting GC until objects are released. 353 void IncrementDisableMovingGC(Thread* self) REQUIRES(!*gc_complete_lock_); 354 void DecrementDisableMovingGC(Thread* self) REQUIRES(!*gc_complete_lock_); 355 356 // Temporarily disable thread flip for JNI critical calls. 357 void IncrementDisableThreadFlip(Thread* self) REQUIRES(!*thread_flip_lock_); 358 void DecrementDisableThreadFlip(Thread* self) REQUIRES(!*thread_flip_lock_); 359 void ThreadFlipBegin(Thread* self) REQUIRES(!*thread_flip_lock_); 360 void ThreadFlipEnd(Thread* self) REQUIRES(!*thread_flip_lock_); 361 362 // Clear all of the mark bits, doesn't clear bitmaps which have the same live bits as mark bits. 363 // Mutator lock is required for GetContinuousSpaces. 364 void ClearMarkedObjects() 365 REQUIRES(Locks::heap_bitmap_lock_) 366 REQUIRES_SHARED(Locks::mutator_lock_); 367 368 // Initiates an explicit garbage collection. 369 void CollectGarbage(bool clear_soft_references, GcCause cause = kGcCauseExplicit) 370 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 371 372 // Does a concurrent GC, should only be called by the GC daemon thread 373 // through runtime. 374 void ConcurrentGC(Thread* self, GcCause cause, bool force_full) 375 REQUIRES(!Locks::runtime_shutdown_lock_, !*gc_complete_lock_, !*pending_task_lock_); 376 377 // Implements VMDebug.countInstancesOfClass and JDWP VM_InstanceCount. 378 // The boolean decides whether to use IsAssignableFrom or == when comparing classes. 379 void CountInstances(const std::vector<Handle<mirror::Class>>& classes, 380 bool use_is_assignable_from, 381 uint64_t* counts) 382 REQUIRES(!Locks::heap_bitmap_lock_, !*gc_complete_lock_) 383 REQUIRES_SHARED(Locks::mutator_lock_); 384 385 // Implements VMDebug.getInstancesOfClasses and JDWP RT_Instances. 386 void GetInstances(VariableSizedHandleScope& scope, 387 Handle<mirror::Class> c, 388 bool use_is_assignable_from, 389 int32_t max_count, 390 std::vector<Handle<mirror::Object>>& instances) 391 REQUIRES(!Locks::heap_bitmap_lock_, !*gc_complete_lock_) 392 REQUIRES_SHARED(Locks::mutator_lock_); 393 394 // Implements JDWP OR_ReferringObjects. 395 void GetReferringObjects(VariableSizedHandleScope& scope, 396 Handle<mirror::Object> o, 397 int32_t max_count, 398 std::vector<Handle<mirror::Object>>& referring_objects) 399 REQUIRES(!Locks::heap_bitmap_lock_, !*gc_complete_lock_) 400 REQUIRES_SHARED(Locks::mutator_lock_); 401 402 // Removes the growth limit on the alloc space so it may grow to its maximum capacity. Used to 403 // implement dalvik.system.VMRuntime.clearGrowthLimit. 404 void ClearGrowthLimit(); 405 406 // Make the current growth limit the new maximum capacity, unmaps pages at the end of spaces 407 // which will never be used. Used to implement dalvik.system.VMRuntime.clampGrowthLimit. 408 void ClampGrowthLimit() REQUIRES(!Locks::heap_bitmap_lock_); 409 410 // Target ideal heap utilization ratio, implements 411 // dalvik.system.VMRuntime.getTargetHeapUtilization. 412 double GetTargetHeapUtilization() const { 413 return target_utilization_; 414 } 415 416 // Data structure memory usage tracking. 417 void RegisterGCAllocation(size_t bytes); 418 void RegisterGCDeAllocation(size_t bytes); 419 420 // Set the heap's private space pointers to be the same as the space based on it's type. Public 421 // due to usage by tests. 422 void SetSpaceAsDefault(space::ContinuousSpace* continuous_space) 423 REQUIRES(!Locks::heap_bitmap_lock_); 424 void AddSpace(space::Space* space) 425 REQUIRES(!Locks::heap_bitmap_lock_) 426 REQUIRES(Locks::mutator_lock_); 427 void RemoveSpace(space::Space* space) 428 REQUIRES(!Locks::heap_bitmap_lock_) 429 REQUIRES(Locks::mutator_lock_); 430 431 double GetPreGcWeightedAllocatedBytes() const { 432 return pre_gc_weighted_allocated_bytes_; 433 } 434 435 double GetPostGcWeightedAllocatedBytes() const { 436 return post_gc_weighted_allocated_bytes_; 437 } 438 439 void CalculatePreGcWeightedAllocatedBytes(); 440 void CalculatePostGcWeightedAllocatedBytes(); 441 uint64_t GetTotalGcCpuTime(); 442 443 uint64_t GetProcessCpuStartTime() const { 444 return process_cpu_start_time_ns_; 445 } 446 447 uint64_t GetPostGCLastProcessCpuTime() const { 448 return post_gc_last_process_cpu_time_ns_; 449 } 450 451 // Set target ideal heap utilization ratio, implements 452 // dalvik.system.VMRuntime.setTargetHeapUtilization. 453 void SetTargetHeapUtilization(float target); 454 455 // For the alloc space, sets the maximum number of bytes that the heap is allowed to allocate 456 // from the system. Doesn't allow the space to exceed its growth limit. 457 void SetIdealFootprint(size_t max_allowed_footprint); 458 459 // Blocks the caller until the garbage collector becomes idle and returns the type of GC we 460 // waited for. 461 collector::GcType WaitForGcToComplete(GcCause cause, Thread* self) REQUIRES(!*gc_complete_lock_); 462 463 // Update the heap's process state to a new value, may cause compaction to occur. 464 void UpdateProcessState(ProcessState old_process_state, ProcessState new_process_state) 465 REQUIRES(!*pending_task_lock_, !*gc_complete_lock_); 466 467 bool HaveContinuousSpaces() const NO_THREAD_SAFETY_ANALYSIS { 468 // No lock since vector empty is thread safe. 469 return !continuous_spaces_.empty(); 470 } 471 472 const std::vector<space::ContinuousSpace*>& GetContinuousSpaces() const 473 REQUIRES_SHARED(Locks::mutator_lock_) { 474 return continuous_spaces_; 475 } 476 477 const std::vector<space::DiscontinuousSpace*>& GetDiscontinuousSpaces() const { 478 return discontinuous_spaces_; 479 } 480 481 const collector::Iteration* GetCurrentGcIteration() const { 482 return ¤t_gc_iteration_; 483 } 484 collector::Iteration* GetCurrentGcIteration() { 485 return ¤t_gc_iteration_; 486 } 487 488 // Enable verification of object references when the runtime is sufficiently initialized. 489 void EnableObjectValidation() { 490 verify_object_mode_ = kVerifyObjectSupport; 491 if (verify_object_mode_ > kVerifyObjectModeDisabled) { 492 VerifyHeap(); 493 } 494 } 495 496 // Disable object reference verification for image writing. 497 void DisableObjectValidation() { 498 verify_object_mode_ = kVerifyObjectModeDisabled; 499 } 500 501 // Other checks may be performed if we know the heap should be in a sane state. 502 bool IsObjectValidationEnabled() const { 503 return verify_object_mode_ > kVerifyObjectModeDisabled; 504 } 505 506 // Returns true if low memory mode is enabled. 507 bool IsLowMemoryMode() const { 508 return low_memory_mode_; 509 } 510 511 // Returns the heap growth multiplier, this affects how much we grow the heap after a GC. 512 // Scales heap growth, min free, and max free. 513 double HeapGrowthMultiplier() const; 514 515 // Freed bytes can be negative in cases where we copy objects from a compacted space to a 516 // free-list backed space. 517 void RecordFree(uint64_t freed_objects, int64_t freed_bytes); 518 519 // Record the bytes freed by thread-local buffer revoke. 520 void RecordFreeRevoke(); 521 522 accounting::CardTable* GetCardTable() const { 523 return card_table_.get(); 524 } 525 526 accounting::ReadBarrierTable* GetReadBarrierTable() const { 527 return rb_table_.get(); 528 } 529 530 void AddFinalizerReference(Thread* self, ObjPtr<mirror::Object>* object); 531 532 // Returns the number of bytes currently allocated. 533 // The result should be treated as an approximation, if it is being concurrently updated. 534 size_t GetBytesAllocated() const { 535 return num_bytes_allocated_.load(std::memory_order_relaxed); 536 } 537 538 bool GetUseGenerationalCC() const { 539 return use_generational_cc_; 540 } 541 542 // Returns the number of objects currently allocated. 543 size_t GetObjectsAllocated() const 544 REQUIRES(!Locks::heap_bitmap_lock_); 545 546 // Returns the total number of objects allocated since the heap was created. 547 uint64_t GetObjectsAllocatedEver() const; 548 549 // Returns the total number of bytes allocated since the heap was created. 550 uint64_t GetBytesAllocatedEver() const; 551 552 // Returns the total number of objects freed since the heap was created. 553 uint64_t GetObjectsFreedEver() const { 554 return total_objects_freed_ever_; 555 } 556 557 // Returns the total number of bytes freed since the heap was created. 558 uint64_t GetBytesFreedEver() const { 559 return total_bytes_freed_ever_; 560 } 561 562 space::RegionSpace* GetRegionSpace() const { 563 return region_space_; 564 } 565 566 // Implements java.lang.Runtime.maxMemory, returning the maximum amount of memory a program can 567 // consume. For a regular VM this would relate to the -Xmx option and would return -1 if no Xmx 568 // were specified. Android apps start with a growth limit (small heap size) which is 569 // cleared/extended for large apps. 570 size_t GetMaxMemory() const { 571 // There are some race conditions in the allocation code that can cause bytes allocated to 572 // become larger than growth_limit_ in rare cases. 573 return std::max(GetBytesAllocated(), growth_limit_); 574 } 575 576 // Implements java.lang.Runtime.totalMemory, returning approximate amount of memory currently 577 // consumed by an application. 578 size_t GetTotalMemory() const; 579 580 // Returns approximately how much free memory we have until the next GC happens. 581 size_t GetFreeMemoryUntilGC() const { 582 return UnsignedDifference(target_footprint_.load(std::memory_order_relaxed), 583 GetBytesAllocated()); 584 } 585 586 // Returns approximately how much free memory we have until the next OOME happens. 587 size_t GetFreeMemoryUntilOOME() const { 588 return UnsignedDifference(growth_limit_, GetBytesAllocated()); 589 } 590 591 // Returns how much free memory we have until we need to grow the heap to perform an allocation. 592 // Similar to GetFreeMemoryUntilGC. Implements java.lang.Runtime.freeMemory. 593 size_t GetFreeMemory() const { 594 return UnsignedDifference(GetTotalMemory(), 595 num_bytes_allocated_.load(std::memory_order_relaxed)); 596 } 597 598 // Get the space that corresponds to an object's address. Current implementation searches all 599 // spaces in turn. If fail_ok is false then failing to find a space will cause an abort. 600 // TODO: consider using faster data structure like binary tree. 601 space::ContinuousSpace* FindContinuousSpaceFromObject(ObjPtr<mirror::Object>, bool fail_ok) const 602 REQUIRES_SHARED(Locks::mutator_lock_); 603 604 space::ContinuousSpace* FindContinuousSpaceFromAddress(const mirror::Object* addr) const 605 REQUIRES_SHARED(Locks::mutator_lock_); 606 607 space::DiscontinuousSpace* FindDiscontinuousSpaceFromObject(ObjPtr<mirror::Object>, 608 bool fail_ok) const 609 REQUIRES_SHARED(Locks::mutator_lock_); 610 611 space::Space* FindSpaceFromObject(ObjPtr<mirror::Object> obj, bool fail_ok) const 612 REQUIRES_SHARED(Locks::mutator_lock_); 613 614 space::Space* FindSpaceFromAddress(const void* ptr) const 615 REQUIRES_SHARED(Locks::mutator_lock_); 616 617 std::string DumpSpaceNameFromAddress(const void* addr) const 618 REQUIRES_SHARED(Locks::mutator_lock_); 619 620 void DumpForSigQuit(std::ostream& os) REQUIRES(!*gc_complete_lock_); 621 622 // Do a pending collector transition. 623 void DoPendingCollectorTransition() REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 624 625 // Deflate monitors, ... and trim the spaces. 626 void Trim(Thread* self) REQUIRES(!*gc_complete_lock_); 627 628 void RevokeThreadLocalBuffers(Thread* thread); 629 void RevokeRosAllocThreadLocalBuffers(Thread* thread); 630 void RevokeAllThreadLocalBuffers(); 631 void AssertThreadLocalBuffersAreRevoked(Thread* thread); 632 void AssertAllBumpPointerSpaceThreadLocalBuffersAreRevoked(); 633 void RosAllocVerification(TimingLogger* timings, const char* name) 634 REQUIRES(Locks::mutator_lock_); 635 636 accounting::HeapBitmap* GetLiveBitmap() REQUIRES_SHARED(Locks::heap_bitmap_lock_) { 637 return live_bitmap_.get(); 638 } 639 640 accounting::HeapBitmap* GetMarkBitmap() REQUIRES_SHARED(Locks::heap_bitmap_lock_) { 641 return mark_bitmap_.get(); 642 } 643 644 accounting::ObjectStack* GetLiveStack() REQUIRES_SHARED(Locks::heap_bitmap_lock_) { 645 return live_stack_.get(); 646 } 647 648 void PreZygoteFork() NO_THREAD_SAFETY_ANALYSIS; 649 650 // Mark and empty stack. 651 void FlushAllocStack() 652 REQUIRES_SHARED(Locks::mutator_lock_) 653 REQUIRES(Locks::heap_bitmap_lock_); 654 655 // Revoke all the thread-local allocation stacks. 656 void RevokeAllThreadLocalAllocationStacks(Thread* self) 657 REQUIRES(Locks::mutator_lock_, !Locks::runtime_shutdown_lock_, !Locks::thread_list_lock_); 658 659 // Mark all the objects in the allocation stack in the specified bitmap. 660 // TODO: Refactor? 661 void MarkAllocStack(accounting::SpaceBitmap<kObjectAlignment>* bitmap1, 662 accounting::SpaceBitmap<kObjectAlignment>* bitmap2, 663 accounting::SpaceBitmap<kLargeObjectAlignment>* large_objects, 664 accounting::ObjectStack* stack) 665 REQUIRES_SHARED(Locks::mutator_lock_) 666 REQUIRES(Locks::heap_bitmap_lock_); 667 668 // Mark the specified allocation stack as live. 669 void MarkAllocStackAsLive(accounting::ObjectStack* stack) 670 REQUIRES_SHARED(Locks::mutator_lock_) 671 REQUIRES(Locks::heap_bitmap_lock_); 672 673 // Unbind any bound bitmaps. 674 void UnBindBitmaps() 675 REQUIRES(Locks::heap_bitmap_lock_) 676 REQUIRES_SHARED(Locks::mutator_lock_); 677 678 // Returns the boot image spaces. There may be multiple boot image spaces. 679 const std::vector<space::ImageSpace*>& GetBootImageSpaces() const { 680 return boot_image_spaces_; 681 } 682 683 bool ObjectIsInBootImageSpace(ObjPtr<mirror::Object> obj) const 684 REQUIRES_SHARED(Locks::mutator_lock_); 685 686 bool IsInBootImageOatFile(const void* p) const 687 REQUIRES_SHARED(Locks::mutator_lock_); 688 689 void GetBootImagesSize(uint32_t* boot_image_begin, 690 uint32_t* boot_image_end, 691 uint32_t* boot_oat_begin, 692 uint32_t* boot_oat_end); 693 694 // Permenantly disable moving garbage collection. 695 void DisableMovingGc() REQUIRES(!*gc_complete_lock_); 696 697 space::DlMallocSpace* GetDlMallocSpace() const { 698 return dlmalloc_space_; 699 } 700 701 space::RosAllocSpace* GetRosAllocSpace() const { 702 return rosalloc_space_; 703 } 704 705 // Return the corresponding rosalloc space. 706 space::RosAllocSpace* GetRosAllocSpace(gc::allocator::RosAlloc* rosalloc) const 707 REQUIRES_SHARED(Locks::mutator_lock_); 708 709 space::MallocSpace* GetNonMovingSpace() const { 710 return non_moving_space_; 711 } 712 713 space::LargeObjectSpace* GetLargeObjectsSpace() const { 714 return large_object_space_; 715 } 716 717 // Returns the free list space that may contain movable objects (the 718 // one that's not the non-moving space), either rosalloc_space_ or 719 // dlmalloc_space_. 720 space::MallocSpace* GetPrimaryFreeListSpace() { 721 if (kUseRosAlloc) { 722 DCHECK(rosalloc_space_ != nullptr); 723 // reinterpret_cast is necessary as the space class hierarchy 724 // isn't known (#included) yet here. 725 return reinterpret_cast<space::MallocSpace*>(rosalloc_space_); 726 } else { 727 DCHECK(dlmalloc_space_ != nullptr); 728 return reinterpret_cast<space::MallocSpace*>(dlmalloc_space_); 729 } 730 } 731 732 void DumpSpaces(std::ostream& stream) const REQUIRES_SHARED(Locks::mutator_lock_); 733 std::string DumpSpaces() const REQUIRES_SHARED(Locks::mutator_lock_); 734 735 // GC performance measuring 736 void DumpGcPerformanceInfo(std::ostream& os) 737 REQUIRES(!*gc_complete_lock_); 738 void ResetGcPerformanceInfo() REQUIRES(!*gc_complete_lock_); 739 740 // Thread pool. 741 void CreateThreadPool(); 742 void DeleteThreadPool(); 743 ThreadPool* GetThreadPool() { 744 return thread_pool_.get(); 745 } 746 size_t GetParallelGCThreadCount() const { 747 return parallel_gc_threads_; 748 } 749 size_t GetConcGCThreadCount() const { 750 return conc_gc_threads_; 751 } 752 accounting::ModUnionTable* FindModUnionTableFromSpace(space::Space* space); 753 void AddModUnionTable(accounting::ModUnionTable* mod_union_table); 754 755 accounting::RememberedSet* FindRememberedSetFromSpace(space::Space* space); 756 void AddRememberedSet(accounting::RememberedSet* remembered_set); 757 // Also deletes the remebered set. 758 void RemoveRememberedSet(space::Space* space); 759 760 bool IsCompilingBoot() const; 761 bool HasBootImageSpace() const { 762 return !boot_image_spaces_.empty(); 763 } 764 765 ReferenceProcessor* GetReferenceProcessor() { 766 return reference_processor_.get(); 767 } 768 TaskProcessor* GetTaskProcessor() { 769 return task_processor_.get(); 770 } 771 772 bool HasZygoteSpace() const { 773 return zygote_space_ != nullptr; 774 } 775 776 // Returns the active concurrent copying collector. 777 collector::ConcurrentCopying* ConcurrentCopyingCollector() { 778 if (use_generational_cc_) { 779 DCHECK((active_concurrent_copying_collector_ == concurrent_copying_collector_) || 780 (active_concurrent_copying_collector_ == young_concurrent_copying_collector_)); 781 } else { 782 DCHECK_EQ(active_concurrent_copying_collector_, concurrent_copying_collector_); 783 } 784 return active_concurrent_copying_collector_; 785 } 786 787 CollectorType CurrentCollectorType() { 788 return collector_type_; 789 } 790 791 bool IsGcConcurrentAndMoving() const { 792 if (IsGcConcurrent() && IsMovingGc(collector_type_)) { 793 // Assume no transition when a concurrent moving collector is used. 794 DCHECK_EQ(collector_type_, foreground_collector_type_); 795 return true; 796 } 797 return false; 798 } 799 800 bool IsMovingGCDisabled(Thread* self) REQUIRES(!*gc_complete_lock_) { 801 MutexLock mu(self, *gc_complete_lock_); 802 return disable_moving_gc_count_ > 0; 803 } 804 805 // Request an asynchronous trim. 806 void RequestTrim(Thread* self) REQUIRES(!*pending_task_lock_); 807 808 // Request asynchronous GC. 809 void RequestConcurrentGC(Thread* self, GcCause cause, bool force_full) 810 REQUIRES(!*pending_task_lock_); 811 812 // Whether or not we may use a garbage collector, used so that we only create collectors we need. 813 bool MayUseCollector(CollectorType type) const; 814 815 // Used by tests to reduce timinig-dependent flakiness in OOME behavior. 816 void SetMinIntervalHomogeneousSpaceCompactionByOom(uint64_t interval) { 817 min_interval_homogeneous_space_compaction_by_oom_ = interval; 818 } 819 820 // Helpers for android.os.Debug.getRuntimeStat(). 821 uint64_t GetGcCount() const; 822 uint64_t GetGcTime() const; 823 uint64_t GetBlockingGcCount() const; 824 uint64_t GetBlockingGcTime() const; 825 void DumpGcCountRateHistogram(std::ostream& os) const REQUIRES(!*gc_complete_lock_); 826 void DumpBlockingGcCountRateHistogram(std::ostream& os) const REQUIRES(!*gc_complete_lock_); 827 828 // Allocation tracking support 829 // Callers to this function use double-checked locking to ensure safety on allocation_records_ 830 bool IsAllocTrackingEnabled() const { 831 return alloc_tracking_enabled_.load(std::memory_order_relaxed); 832 } 833 834 void SetAllocTrackingEnabled(bool enabled) REQUIRES(Locks::alloc_tracker_lock_) { 835 alloc_tracking_enabled_.store(enabled, std::memory_order_relaxed); 836 } 837 838 // Return the current stack depth of allocation records. 839 size_t GetAllocTrackerStackDepth() const { 840 return alloc_record_depth_; 841 } 842 843 // Return the current stack depth of allocation records. 844 void SetAllocTrackerStackDepth(size_t alloc_record_depth) { 845 alloc_record_depth_ = alloc_record_depth; 846 } 847 848 AllocRecordObjectMap* GetAllocationRecords() const REQUIRES(Locks::alloc_tracker_lock_) { 849 return allocation_records_.get(); 850 } 851 852 void SetAllocationRecords(AllocRecordObjectMap* records) 853 REQUIRES(Locks::alloc_tracker_lock_); 854 855 void VisitAllocationRecords(RootVisitor* visitor) const 856 REQUIRES_SHARED(Locks::mutator_lock_) 857 REQUIRES(!Locks::alloc_tracker_lock_); 858 859 void SweepAllocationRecords(IsMarkedVisitor* visitor) const 860 REQUIRES_SHARED(Locks::mutator_lock_) 861 REQUIRES(!Locks::alloc_tracker_lock_); 862 863 void DisallowNewAllocationRecords() const 864 REQUIRES_SHARED(Locks::mutator_lock_) 865 REQUIRES(!Locks::alloc_tracker_lock_); 866 867 void AllowNewAllocationRecords() const 868 REQUIRES_SHARED(Locks::mutator_lock_) 869 REQUIRES(!Locks::alloc_tracker_lock_); 870 871 void BroadcastForNewAllocationRecords() const 872 REQUIRES(!Locks::alloc_tracker_lock_); 873 874 void DisableGCForShutdown() REQUIRES(!*gc_complete_lock_); 875 876 // Create a new alloc space and compact default alloc space to it. 877 HomogeneousSpaceCompactResult PerformHomogeneousSpaceCompact() REQUIRES(!*gc_complete_lock_); 878 bool SupportHomogeneousSpaceCompactAndCollectorTransitions() const; 879 880 // Install an allocation listener. 881 void SetAllocationListener(AllocationListener* l); 882 // Remove an allocation listener. Note: the listener must not be deleted, as for performance 883 // reasons, we assume it stays valid when we read it (so that we don't require a lock). 884 void RemoveAllocationListener(); 885 886 // Install a gc pause listener. 887 void SetGcPauseListener(GcPauseListener* l); 888 // Get the currently installed gc pause listener, or null. 889 GcPauseListener* GetGcPauseListener() { 890 return gc_pause_listener_.load(std::memory_order_acquire); 891 } 892 // Remove a gc pause listener. Note: the listener must not be deleted, as for performance 893 // reasons, we assume it stays valid when we read it (so that we don't require a lock). 894 void RemoveGcPauseListener(); 895 896 const Verification* GetVerification() const; 897 898 void PostForkChildAction(Thread* self); 899 900 private: 901 class ConcurrentGCTask; 902 class CollectorTransitionTask; 903 class HeapTrimTask; 904 class TriggerPostForkCCGcTask; 905 906 // Compact source space to target space. Returns the collector used. 907 collector::GarbageCollector* Compact(space::ContinuousMemMapAllocSpace* target_space, 908 space::ContinuousMemMapAllocSpace* source_space, 909 GcCause gc_cause) 910 REQUIRES(Locks::mutator_lock_); 911 912 void LogGC(GcCause gc_cause, collector::GarbageCollector* collector); 913 void StartGC(Thread* self, GcCause cause, CollectorType collector_type) 914 REQUIRES(!*gc_complete_lock_); 915 void FinishGC(Thread* self, collector::GcType gc_type) REQUIRES(!*gc_complete_lock_); 916 917 double CalculateGcWeightedAllocatedBytes(uint64_t gc_last_process_cpu_time_ns, 918 uint64_t current_process_cpu_time) const; 919 920 // Create a mem map with a preferred base address. 921 static MemMap MapAnonymousPreferredAddress(const char* name, 922 uint8_t* request_begin, 923 size_t capacity, 924 std::string* out_error_str); 925 926 bool SupportHSpaceCompaction() const { 927 // Returns true if we can do hspace compaction 928 return main_space_backup_ != nullptr; 929 } 930 931 // Size_t saturating arithmetic 932 static ALWAYS_INLINE size_t UnsignedDifference(size_t x, size_t y) { 933 return x > y ? x - y : 0; 934 } 935 static ALWAYS_INLINE size_t UnsignedSum(size_t x, size_t y) { 936 return x + y >= x ? x + y : std::numeric_limits<size_t>::max(); 937 } 938 939 static ALWAYS_INLINE bool AllocatorHasAllocationStack(AllocatorType allocator_type) { 940 return 941 allocator_type != kAllocatorTypeRegionTLAB && 942 allocator_type != kAllocatorTypeBumpPointer && 943 allocator_type != kAllocatorTypeTLAB && 944 allocator_type != kAllocatorTypeRegion; 945 } 946 static ALWAYS_INLINE bool AllocatorMayHaveConcurrentGC(AllocatorType allocator_type) { 947 if (kUseReadBarrier) { 948 // Read barrier may have the TLAB allocator but is always concurrent. TODO: clean this up. 949 return true; 950 } 951 return 952 allocator_type != kAllocatorTypeTLAB && 953 allocator_type != kAllocatorTypeBumpPointer; 954 } 955 static bool IsMovingGc(CollectorType collector_type) { 956 return 957 collector_type == kCollectorTypeCC || 958 collector_type == kCollectorTypeSS || 959 collector_type == kCollectorTypeGSS || 960 collector_type == kCollectorTypeCCBackground || 961 collector_type == kCollectorTypeHomogeneousSpaceCompact; 962 } 963 bool ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_count) const 964 REQUIRES_SHARED(Locks::mutator_lock_); 965 966 // Checks whether we should garbage collect: 967 ALWAYS_INLINE bool ShouldConcurrentGCForJava(size_t new_num_bytes_allocated); 968 float NativeMemoryOverTarget(size_t current_native_bytes, bool is_gc_concurrent); 969 ALWAYS_INLINE void CheckConcurrentGCForJava(Thread* self, 970 size_t new_num_bytes_allocated, 971 ObjPtr<mirror::Object>* obj) 972 REQUIRES_SHARED(Locks::mutator_lock_) 973 REQUIRES(!*pending_task_lock_, !*gc_complete_lock_); 974 void CheckGCForNative(Thread* self) 975 REQUIRES(!*pending_task_lock_, !*gc_complete_lock_); 976 977 accounting::ObjectStack* GetMarkStack() { 978 return mark_stack_.get(); 979 } 980 981 // We don't force this to be inlined since it is a slow path. 982 template <bool kInstrumented, typename PreFenceVisitor> 983 mirror::Object* AllocLargeObject(Thread* self, 984 ObjPtr<mirror::Class>* klass, 985 size_t byte_count, 986 const PreFenceVisitor& pre_fence_visitor) 987 REQUIRES_SHARED(Locks::mutator_lock_) 988 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_, !*backtrace_lock_); 989 990 // Handles Allocate()'s slow allocation path with GC involved after 991 // an initial allocation attempt failed. 992 mirror::Object* AllocateInternalWithGc(Thread* self, 993 AllocatorType allocator, 994 bool instrumented, 995 size_t num_bytes, 996 size_t* bytes_allocated, 997 size_t* usable_size, 998 size_t* bytes_tl_bulk_allocated, 999 ObjPtr<mirror::Class>* klass) 1000 REQUIRES(!Locks::thread_suspend_count_lock_, !*gc_complete_lock_, !*pending_task_lock_) 1001 REQUIRES_SHARED(Locks::mutator_lock_); 1002 1003 // Allocate into a specific space. 1004 mirror::Object* AllocateInto(Thread* self, 1005 space::AllocSpace* space, 1006 ObjPtr<mirror::Class> c, 1007 size_t bytes) 1008 REQUIRES_SHARED(Locks::mutator_lock_); 1009 1010 // Need to do this with mutators paused so that somebody doesn't accidentally allocate into the 1011 // wrong space. 1012 void SwapSemiSpaces() REQUIRES(Locks::mutator_lock_); 1013 1014 // Try to allocate a number of bytes, this function never does any GCs. Needs to be inlined so 1015 // that the switch statement is constant optimized in the entrypoints. 1016 template <const bool kInstrumented, const bool kGrow> 1017 ALWAYS_INLINE mirror::Object* TryToAllocate(Thread* self, 1018 AllocatorType allocator_type, 1019 size_t alloc_size, 1020 size_t* bytes_allocated, 1021 size_t* usable_size, 1022 size_t* bytes_tl_bulk_allocated) 1023 REQUIRES_SHARED(Locks::mutator_lock_); 1024 1025 mirror::Object* AllocWithNewTLAB(Thread* self, 1026 size_t alloc_size, 1027 bool grow, 1028 size_t* bytes_allocated, 1029 size_t* usable_size, 1030 size_t* bytes_tl_bulk_allocated) 1031 REQUIRES_SHARED(Locks::mutator_lock_); 1032 1033 void ThrowOutOfMemoryError(Thread* self, size_t byte_count, AllocatorType allocator_type) 1034 REQUIRES_SHARED(Locks::mutator_lock_); 1035 1036 // Are we out of memory, and thus should force a GC or fail? 1037 // For concurrent collectors, out of memory is defined by growth_limit_. 1038 // For nonconcurrent collectors it is defined by target_footprint_ unless grow is 1039 // set. If grow is set, the limit is growth_limit_ and we adjust target_footprint_ 1040 // to accomodate the allocation. 1041 ALWAYS_INLINE bool IsOutOfMemoryOnAllocation(AllocatorType allocator_type, 1042 size_t alloc_size, 1043 bool grow); 1044 1045 // Run the finalizers. If timeout is non zero, then we use the VMRuntime version. 1046 void RunFinalization(JNIEnv* env, uint64_t timeout); 1047 1048 // Blocks the caller until the garbage collector becomes idle and returns the type of GC we 1049 // waited for. 1050 collector::GcType WaitForGcToCompleteLocked(GcCause cause, Thread* self) 1051 REQUIRES(gc_complete_lock_); 1052 1053 void RequestCollectorTransition(CollectorType desired_collector_type, uint64_t delta_time) 1054 REQUIRES(!*pending_task_lock_); 1055 1056 void RequestConcurrentGCAndSaveObject(Thread* self, bool force_full, ObjPtr<mirror::Object>* obj) 1057 REQUIRES_SHARED(Locks::mutator_lock_) 1058 REQUIRES(!*pending_task_lock_); 1059 bool IsGCRequestPending() const; 1060 1061 // Sometimes CollectGarbageInternal decides to run a different Gc than you requested. Returns 1062 // which type of Gc was actually ran. 1063 collector::GcType CollectGarbageInternal(collector::GcType gc_plan, 1064 GcCause gc_cause, 1065 bool clear_soft_references) 1066 REQUIRES(!*gc_complete_lock_, !Locks::heap_bitmap_lock_, !Locks::thread_suspend_count_lock_, 1067 !*pending_task_lock_); 1068 1069 void PreGcVerification(collector::GarbageCollector* gc) 1070 REQUIRES(!Locks::mutator_lock_, !*gc_complete_lock_); 1071 void PreGcVerificationPaused(collector::GarbageCollector* gc) 1072 REQUIRES(Locks::mutator_lock_, !*gc_complete_lock_); 1073 void PrePauseRosAllocVerification(collector::GarbageCollector* gc) 1074 REQUIRES(Locks::mutator_lock_); 1075 void PreSweepingGcVerification(collector::GarbageCollector* gc) 1076 REQUIRES(Locks::mutator_lock_, !Locks::heap_bitmap_lock_, !*gc_complete_lock_); 1077 void PostGcVerification(collector::GarbageCollector* gc) 1078 REQUIRES(!Locks::mutator_lock_, !*gc_complete_lock_); 1079 void PostGcVerificationPaused(collector::GarbageCollector* gc) 1080 REQUIRES(Locks::mutator_lock_, !*gc_complete_lock_); 1081 1082 // Find a collector based on GC type. 1083 collector::GarbageCollector* FindCollectorByGcType(collector::GcType gc_type); 1084 1085 // Create the main free list malloc space, either a RosAlloc space or DlMalloc space. 1086 void CreateMainMallocSpace(MemMap&& mem_map, 1087 size_t initial_size, 1088 size_t growth_limit, 1089 size_t capacity); 1090 1091 // Create a malloc space based on a mem map. Does not set the space as default. 1092 space::MallocSpace* CreateMallocSpaceFromMemMap(MemMap&& mem_map, 1093 size_t initial_size, 1094 size_t growth_limit, 1095 size_t capacity, 1096 const char* name, 1097 bool can_move_objects); 1098 1099 // Given the current contents of the alloc space, increase the allowed heap footprint to match 1100 // the target utilization ratio. This should only be called immediately after a full garbage 1101 // collection. bytes_allocated_before_gc is used to measure bytes / second for the period which 1102 // the GC was run. 1103 void GrowForUtilization(collector::GarbageCollector* collector_ran, 1104 size_t bytes_allocated_before_gc = 0); 1105 1106 size_t GetPercentFree(); 1107 1108 // Swap the allocation stack with the live stack. 1109 void SwapStacks() REQUIRES_SHARED(Locks::mutator_lock_); 1110 1111 // Clear cards and update the mod union table. When process_alloc_space_cards is true, 1112 // if clear_alloc_space_cards is true, then we clear cards instead of ageing them. We do 1113 // not process the alloc space if process_alloc_space_cards is false. 1114 void ProcessCards(TimingLogger* timings, 1115 bool use_rem_sets, 1116 bool process_alloc_space_cards, 1117 bool clear_alloc_space_cards) 1118 REQUIRES_SHARED(Locks::mutator_lock_); 1119 1120 // Push an object onto the allocation stack. 1121 void PushOnAllocationStack(Thread* self, ObjPtr<mirror::Object>* obj) 1122 REQUIRES_SHARED(Locks::mutator_lock_) 1123 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 1124 void PushOnAllocationStackWithInternalGC(Thread* self, ObjPtr<mirror::Object>* obj) 1125 REQUIRES_SHARED(Locks::mutator_lock_) 1126 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 1127 void PushOnThreadLocalAllocationStackWithInternalGC(Thread* thread, ObjPtr<mirror::Object>* obj) 1128 REQUIRES_SHARED(Locks::mutator_lock_) 1129 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_); 1130 1131 void ClearConcurrentGCRequest(); 1132 void ClearPendingTrim(Thread* self) REQUIRES(!*pending_task_lock_); 1133 void ClearPendingCollectorTransition(Thread* self) REQUIRES(!*pending_task_lock_); 1134 1135 // What kind of concurrency behavior is the runtime after? Currently true for concurrent mark 1136 // sweep GC, false for other GC types. 1137 bool IsGcConcurrent() const ALWAYS_INLINE { 1138 return collector_type_ == kCollectorTypeCC || 1139 collector_type_ == kCollectorTypeCMS || 1140 collector_type_ == kCollectorTypeCCBackground; 1141 } 1142 1143 // Trim the managed and native spaces by releasing unused memory back to the OS. 1144 void TrimSpaces(Thread* self) REQUIRES(!*gc_complete_lock_); 1145 1146 // Trim 0 pages at the end of reference tables. 1147 void TrimIndirectReferenceTables(Thread* self); 1148 1149 template <typename Visitor> 1150 ALWAYS_INLINE void VisitObjectsInternal(Visitor&& visitor) 1151 REQUIRES_SHARED(Locks::mutator_lock_) 1152 REQUIRES(!Locks::heap_bitmap_lock_, !*gc_complete_lock_); 1153 template <typename Visitor> 1154 ALWAYS_INLINE void VisitObjectsInternalRegionSpace(Visitor&& visitor) 1155 REQUIRES(Locks::mutator_lock_, !Locks::heap_bitmap_lock_, !*gc_complete_lock_); 1156 1157 void UpdateGcCountRateHistograms() REQUIRES(gc_complete_lock_); 1158 1159 // GC stress mode attempts to do one GC per unique backtrace. 1160 void CheckGcStressMode(Thread* self, ObjPtr<mirror::Object>* obj) 1161 REQUIRES_SHARED(Locks::mutator_lock_) 1162 REQUIRES(!*gc_complete_lock_, !*pending_task_lock_, !*backtrace_lock_); 1163 1164 collector::GcType NonStickyGcType() const { 1165 return HasZygoteSpace() ? collector::kGcTypePartial : collector::kGcTypeFull; 1166 } 1167 1168 // Return the amount of space we allow for native memory when deciding whether to 1169 // collect. We collect when a weighted sum of Java memory plus native memory exceeds 1170 // the similarly weighted sum of the Java heap size target and this value. 1171 ALWAYS_INLINE size_t NativeAllocationGcWatermark() const { 1172 // We keep the traditional limit of max_free_ in place for small heaps, 1173 // but allow it to be adjusted upward for large heaps to limit GC overhead. 1174 return target_footprint_.load(std::memory_order_relaxed) / 8 + max_free_; 1175 } 1176 1177 ALWAYS_INLINE void IncrementNumberOfBytesFreedRevoke(size_t freed_bytes_revoke); 1178 1179 void TraceHeapSize(size_t heap_size); 1180 1181 // Remove a vlog code from heap-inl.h which is transitively included in half the world. 1182 static void VlogHeapGrowth(size_t max_allowed_footprint, size_t new_footprint, size_t alloc_size); 1183 1184 // Return our best approximation of the number of bytes of native memory that 1185 // are currently in use, and could possibly be reclaimed as an indirect result 1186 // of a garbage collection. 1187 size_t GetNativeBytes(); 1188 1189 // All-known continuous spaces, where objects lie within fixed bounds. 1190 std::vector<space::ContinuousSpace*> continuous_spaces_ GUARDED_BY(Locks::mutator_lock_); 1191 1192 // All-known discontinuous spaces, where objects may be placed throughout virtual memory. 1193 std::vector<space::DiscontinuousSpace*> discontinuous_spaces_ GUARDED_BY(Locks::mutator_lock_); 1194 1195 // All-known alloc spaces, where objects may be or have been allocated. 1196 std::vector<space::AllocSpace*> alloc_spaces_; 1197 1198 // A space where non-movable objects are allocated, when compaction is enabled it contains 1199 // Classes, ArtMethods, ArtFields, and non moving objects. 1200 space::MallocSpace* non_moving_space_; 1201 1202 // Space which we use for the kAllocatorTypeROSAlloc. 1203 space::RosAllocSpace* rosalloc_space_; 1204 1205 // Space which we use for the kAllocatorTypeDlMalloc. 1206 space::DlMallocSpace* dlmalloc_space_; 1207 1208 // The main space is the space which the GC copies to and from on process state updates. This 1209 // space is typically either the dlmalloc_space_ or the rosalloc_space_. 1210 space::MallocSpace* main_space_; 1211 1212 // The large object space we are currently allocating into. 1213 space::LargeObjectSpace* large_object_space_; 1214 1215 // The card table, dirtied by the write barrier. 1216 std::unique_ptr<accounting::CardTable> card_table_; 1217 1218 std::unique_ptr<accounting::ReadBarrierTable> rb_table_; 1219 1220 // A mod-union table remembers all of the references from the it's space to other spaces. 1221 AllocationTrackingSafeMap<space::Space*, accounting::ModUnionTable*, kAllocatorTagHeap> 1222 mod_union_tables_; 1223 1224 // A remembered set remembers all of the references from the it's space to the target space. 1225 AllocationTrackingSafeMap<space::Space*, accounting::RememberedSet*, kAllocatorTagHeap> 1226 remembered_sets_; 1227 1228 // The current collector type. 1229 CollectorType collector_type_; 1230 // Which collector we use when the app is in the foreground. 1231 CollectorType foreground_collector_type_; 1232 // Which collector we will use when the app is notified of a transition to background. 1233 CollectorType background_collector_type_; 1234 // Desired collector type, heap trimming daemon transitions the heap if it is != collector_type_. 1235 CollectorType desired_collector_type_; 1236 1237 // Lock which guards pending tasks. 1238 Mutex* pending_task_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 1239 1240 // How many GC threads we may use for paused parts of garbage collection. 1241 const size_t parallel_gc_threads_; 1242 1243 // How many GC threads we may use for unpaused parts of garbage collection. 1244 const size_t conc_gc_threads_; 1245 1246 // Boolean for if we are in low memory mode. 1247 const bool low_memory_mode_; 1248 1249 // If we get a pause longer than long pause log threshold, then we print out the GC after it 1250 // finishes. 1251 const size_t long_pause_log_threshold_; 1252 1253 // If we get a GC longer than long GC log threshold, then we print out the GC after it finishes. 1254 const size_t long_gc_log_threshold_; 1255 1256 // Starting time of the new process; meant to be used for measuring total process CPU time. 1257 uint64_t process_cpu_start_time_ns_; 1258 1259 // Last time (before and after) GC started; meant to be used to measure the 1260 // duration between two GCs. 1261 uint64_t pre_gc_last_process_cpu_time_ns_; 1262 uint64_t post_gc_last_process_cpu_time_ns_; 1263 1264 // allocated_bytes * (current_process_cpu_time - [pre|post]_gc_last_process_cpu_time) 1265 double pre_gc_weighted_allocated_bytes_; 1266 double post_gc_weighted_allocated_bytes_; 1267 1268 // If we ignore the target footprint it lets the heap grow until it hits the heap capacity, this 1269 // is useful for benchmarking since it reduces time spent in GC to a low %. 1270 const bool ignore_target_footprint_; 1271 1272 // Lock which guards zygote space creation. 1273 Mutex zygote_creation_lock_; 1274 1275 // Non-null iff we have a zygote space. Doesn't contain the large objects allocated before 1276 // zygote space creation. 1277 space::ZygoteSpace* zygote_space_; 1278 1279 // Minimum allocation size of large object. 1280 size_t large_object_threshold_; 1281 1282 // Guards access to the state of GC, associated conditional variable is used to signal when a GC 1283 // completes. 1284 Mutex* gc_complete_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 1285 std::unique_ptr<ConditionVariable> gc_complete_cond_ GUARDED_BY(gc_complete_lock_); 1286 1287 // Used to synchronize between JNI critical calls and the thread flip of the CC collector. 1288 Mutex* thread_flip_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 1289 std::unique_ptr<ConditionVariable> thread_flip_cond_ GUARDED_BY(thread_flip_lock_); 1290 // This counter keeps track of how many threads are currently in a JNI critical section. This is 1291 // incremented once per thread even with nested enters. 1292 size_t disable_thread_flip_count_ GUARDED_BY(thread_flip_lock_); 1293 bool thread_flip_running_ GUARDED_BY(thread_flip_lock_); 1294 1295 // Reference processor; 1296 std::unique_ptr<ReferenceProcessor> reference_processor_; 1297 1298 // Task processor, proxies heap trim requests to the daemon threads. 1299 std::unique_ptr<TaskProcessor> task_processor_; 1300 1301 // Collector type of the running GC. 1302 volatile CollectorType collector_type_running_ GUARDED_BY(gc_complete_lock_); 1303 1304 // Cause of the last running GC. 1305 volatile GcCause last_gc_cause_ GUARDED_BY(gc_complete_lock_); 1306 1307 // The thread currently running the GC. 1308 volatile Thread* thread_running_gc_ GUARDED_BY(gc_complete_lock_); 1309 1310 // Last Gc type we ran. Used by WaitForConcurrentGc to know which Gc was waited on. 1311 volatile collector::GcType last_gc_type_ GUARDED_BY(gc_complete_lock_); 1312 collector::GcType next_gc_type_; 1313 1314 // Maximum size that the heap can reach. 1315 size_t capacity_; 1316 1317 // The size the heap is limited to. This is initially smaller than capacity, but for largeHeap 1318 // programs it is "cleared" making it the same as capacity. 1319 // Only weakly enforced for simultaneous allocations. 1320 size_t growth_limit_; 1321 1322 // Target size (as in maximum allocatable bytes) for the heap. Weakly enforced as a limit for 1323 // non-concurrent GC. Used as a guideline for computing concurrent_start_bytes_ in the 1324 // concurrent GC case. 1325 Atomic<size_t> target_footprint_; 1326 1327 // When num_bytes_allocated_ exceeds this amount then a concurrent GC should be requested so that 1328 // it completes ahead of an allocation failing. 1329 // A multiple of this is also used to determine when to trigger a GC in response to native 1330 // allocation. 1331 size_t concurrent_start_bytes_; 1332 1333 // Since the heap was created, how many bytes have been freed. 1334 uint64_t total_bytes_freed_ever_; 1335 1336 // Since the heap was created, how many objects have been freed. 1337 uint64_t total_objects_freed_ever_; 1338 1339 // Number of bytes currently allocated and not yet reclaimed. Includes active 1340 // TLABS in their entirety, even if they have not yet been parceled out. 1341 Atomic<size_t> num_bytes_allocated_; 1342 1343 // Number of registered native bytes allocated. Adjusted after each RegisterNativeAllocation and 1344 // RegisterNativeFree. Used to help determine when to trigger GC for native allocations. Should 1345 // not include bytes allocated through the system malloc, since those are implicitly included. 1346 Atomic<size_t> native_bytes_registered_; 1347 1348 // Approximately the smallest value of GetNativeBytes() we've seen since the last GC. 1349 Atomic<size_t> old_native_bytes_allocated_; 1350 1351 // Total number of native objects of which we were notified since the beginning of time, mod 2^32. 1352 // Allows us to check for GC only roughly every kNotifyNativeInterval allocations. 1353 Atomic<uint32_t> native_objects_notified_; 1354 1355 // Number of bytes freed by thread local buffer revokes. This will 1356 // cancel out the ahead-of-time bulk counting of bytes allocated in 1357 // rosalloc thread-local buffers. It is temporarily accumulated 1358 // here to be subtracted from num_bytes_allocated_ later at the next 1359 // GC. 1360 Atomic<size_t> num_bytes_freed_revoke_; 1361 1362 // Info related to the current or previous GC iteration. 1363 collector::Iteration current_gc_iteration_; 1364 1365 // Heap verification flags. 1366 const bool verify_missing_card_marks_; 1367 const bool verify_system_weaks_; 1368 const bool verify_pre_gc_heap_; 1369 const bool verify_pre_sweeping_heap_; 1370 const bool verify_post_gc_heap_; 1371 const bool verify_mod_union_table_; 1372 bool verify_pre_gc_rosalloc_; 1373 bool verify_pre_sweeping_rosalloc_; 1374 bool verify_post_gc_rosalloc_; 1375 const bool gc_stress_mode_; 1376 1377 // RAII that temporarily disables the rosalloc verification during 1378 // the zygote fork. 1379 class ScopedDisableRosAllocVerification { 1380 private: 1381 Heap* const heap_; 1382 const bool orig_verify_pre_gc_; 1383 const bool orig_verify_pre_sweeping_; 1384 const bool orig_verify_post_gc_; 1385 1386 public: 1387 explicit ScopedDisableRosAllocVerification(Heap* heap) 1388 : heap_(heap), 1389 orig_verify_pre_gc_(heap_->verify_pre_gc_rosalloc_), 1390 orig_verify_pre_sweeping_(heap_->verify_pre_sweeping_rosalloc_), 1391 orig_verify_post_gc_(heap_->verify_post_gc_rosalloc_) { 1392 heap_->verify_pre_gc_rosalloc_ = false; 1393 heap_->verify_pre_sweeping_rosalloc_ = false; 1394 heap_->verify_post_gc_rosalloc_ = false; 1395 } 1396 ~ScopedDisableRosAllocVerification() { 1397 heap_->verify_pre_gc_rosalloc_ = orig_verify_pre_gc_; 1398 heap_->verify_pre_sweeping_rosalloc_ = orig_verify_pre_sweeping_; 1399 heap_->verify_post_gc_rosalloc_ = orig_verify_post_gc_; 1400 } 1401 }; 1402 1403 // Parallel GC data structures. 1404 std::unique_ptr<ThreadPool> thread_pool_; 1405 1406 // A bitmap that is set corresponding to the known live objects since the last GC cycle. 1407 std::unique_ptr<accounting::HeapBitmap> live_bitmap_ GUARDED_BY(Locks::heap_bitmap_lock_); 1408 // A bitmap that is set corresponding to the marked objects in the current GC cycle. 1409 std::unique_ptr<accounting::HeapBitmap> mark_bitmap_ GUARDED_BY(Locks::heap_bitmap_lock_); 1410 1411 // Mark stack that we reuse to avoid re-allocating the mark stack. 1412 std::unique_ptr<accounting::ObjectStack> mark_stack_; 1413 1414 // Allocation stack, new allocations go here so that we can do sticky mark bits. This enables us 1415 // to use the live bitmap as the old mark bitmap. 1416 const size_t max_allocation_stack_size_; 1417 std::unique_ptr<accounting::ObjectStack> allocation_stack_; 1418 1419 // Second allocation stack so that we can process allocation with the heap unlocked. 1420 std::unique_ptr<accounting::ObjectStack> live_stack_; 1421 1422 // Allocator type. 1423 AllocatorType current_allocator_; 1424 const AllocatorType current_non_moving_allocator_; 1425 1426 // Which GCs we run in order when an allocation fails. 1427 std::vector<collector::GcType> gc_plan_; 1428 1429 // Bump pointer spaces. 1430 space::BumpPointerSpace* bump_pointer_space_; 1431 // Temp space is the space which the semispace collector copies to. 1432 space::BumpPointerSpace* temp_space_; 1433 1434 // Region space, used by the concurrent collector. 1435 space::RegionSpace* region_space_; 1436 1437 // Minimum free guarantees that you always have at least min_free_ free bytes after growing for 1438 // utilization, regardless of target utilization ratio. 1439 const size_t min_free_; 1440 1441 // The ideal maximum free size, when we grow the heap for utilization. 1442 const size_t max_free_; 1443 1444 // Target ideal heap utilization ratio. 1445 double target_utilization_; 1446 1447 // How much more we grow the heap when we are a foreground app instead of background. 1448 double foreground_heap_growth_multiplier_; 1449 1450 // Total time which mutators are paused or waiting for GC to complete. 1451 uint64_t total_wait_time_; 1452 1453 // The current state of heap verification, may be enabled or disabled. 1454 VerifyObjectMode verify_object_mode_; 1455 1456 // Compacting GC disable count, prevents compacting GC from running iff > 0. 1457 size_t disable_moving_gc_count_ GUARDED_BY(gc_complete_lock_); 1458 1459 std::vector<collector::GarbageCollector*> garbage_collectors_; 1460 collector::SemiSpace* semi_space_collector_; 1461 collector::ConcurrentCopying* active_concurrent_copying_collector_; 1462 collector::ConcurrentCopying* young_concurrent_copying_collector_; 1463 collector::ConcurrentCopying* concurrent_copying_collector_; 1464 1465 const bool is_running_on_memory_tool_; 1466 const bool use_tlab_; 1467 1468 // Pointer to the space which becomes the new main space when we do homogeneous space compaction. 1469 // Use unique_ptr since the space is only added during the homogeneous compaction phase. 1470 std::unique_ptr<space::MallocSpace> main_space_backup_; 1471 1472 // Minimal interval allowed between two homogeneous space compactions caused by OOM. 1473 uint64_t min_interval_homogeneous_space_compaction_by_oom_; 1474 1475 // Times of the last homogeneous space compaction caused by OOM. 1476 uint64_t last_time_homogeneous_space_compaction_by_oom_; 1477 1478 // Saved OOMs by homogeneous space compaction. 1479 Atomic<size_t> count_delayed_oom_; 1480 1481 // Count for requested homogeneous space compaction. 1482 Atomic<size_t> count_requested_homogeneous_space_compaction_; 1483 1484 // Count for ignored homogeneous space compaction. 1485 Atomic<size_t> count_ignored_homogeneous_space_compaction_; 1486 1487 // Count for performed homogeneous space compaction. 1488 Atomic<size_t> count_performed_homogeneous_space_compaction_; 1489 1490 // Whether or not a concurrent GC is pending. 1491 Atomic<bool> concurrent_gc_pending_; 1492 1493 // Active tasks which we can modify (change target time, desired collector type, etc..). 1494 CollectorTransitionTask* pending_collector_transition_ GUARDED_BY(pending_task_lock_); 1495 HeapTrimTask* pending_heap_trim_ GUARDED_BY(pending_task_lock_); 1496 1497 // Whether or not we use homogeneous space compaction to avoid OOM errors. 1498 bool use_homogeneous_space_compaction_for_oom_; 1499 1500 // If true, enable generational collection when using the Concurrent Copying 1501 // (CC) collector, i.e. use sticky-bit CC for minor collections and (full) CC 1502 // for major collections. Set in Heap constructor. 1503 const bool use_generational_cc_; 1504 1505 // True if the currently running collection has made some thread wait. 1506 bool running_collection_is_blocking_ GUARDED_BY(gc_complete_lock_); 1507 // The number of blocking GC runs. 1508 uint64_t blocking_gc_count_; 1509 // The total duration of blocking GC runs. 1510 uint64_t blocking_gc_time_; 1511 // The duration of the window for the GC count rate histograms. 1512 static constexpr uint64_t kGcCountRateHistogramWindowDuration = MsToNs(10 * 1000); // 10s. 1513 // Maximum number of missed histogram windows for which statistics will be collected. 1514 static constexpr uint64_t kGcCountRateHistogramMaxNumMissedWindows = 100; 1515 // The last time when the GC count rate histograms were updated. 1516 // This is rounded by kGcCountRateHistogramWindowDuration (a multiple of 10s). 1517 uint64_t last_update_time_gc_count_rate_histograms_; 1518 // The running count of GC runs in the last window. 1519 uint64_t gc_count_last_window_; 1520 // The running count of blocking GC runs in the last window. 1521 uint64_t blocking_gc_count_last_window_; 1522 // The maximum number of buckets in the GC count rate histograms. 1523 static constexpr size_t kGcCountRateMaxBucketCount = 200; 1524 // The histogram of the number of GC invocations per window duration. 1525 Histogram<uint64_t> gc_count_rate_histogram_ GUARDED_BY(gc_complete_lock_); 1526 // The histogram of the number of blocking GC invocations per window duration. 1527 Histogram<uint64_t> blocking_gc_count_rate_histogram_ GUARDED_BY(gc_complete_lock_); 1528 1529 // Allocation tracking support 1530 Atomic<bool> alloc_tracking_enabled_; 1531 std::unique_ptr<AllocRecordObjectMap> allocation_records_; 1532 size_t alloc_record_depth_; 1533 1534 // GC stress related data structures. 1535 Mutex* backtrace_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 1536 // Debugging variables, seen backtraces vs unique backtraces. 1537 Atomic<uint64_t> seen_backtrace_count_; 1538 Atomic<uint64_t> unique_backtrace_count_; 1539 // Stack trace hashes that we already saw, 1540 std::unordered_set<uint64_t> seen_backtraces_ GUARDED_BY(backtrace_lock_); 1541 1542 // We disable GC when we are shutting down the runtime in case there are daemon threads still 1543 // allocating. 1544 bool gc_disabled_for_shutdown_ GUARDED_BY(gc_complete_lock_); 1545 1546 // Turned on by -XX:DumpRegionInfoBeforeGC and -XX:DumpRegionInfoAfterGC to 1547 // emit region info before and after each GC cycle. 1548 bool dump_region_info_before_gc_; 1549 bool dump_region_info_after_gc_; 1550 1551 // Boot image spaces. 1552 std::vector<space::ImageSpace*> boot_image_spaces_; 1553 1554 // An installed allocation listener. 1555 Atomic<AllocationListener*> alloc_listener_; 1556 // An installed GC Pause listener. 1557 Atomic<GcPauseListener*> gc_pause_listener_; 1558 1559 std::unique_ptr<Verification> verification_; 1560 1561 friend class CollectorTransitionTask; 1562 friend class collector::GarbageCollector; 1563 friend class collector::ConcurrentCopying; 1564 friend class collector::MarkSweep; 1565 friend class collector::SemiSpace; 1566 friend class GCCriticalSection; 1567 friend class ReferenceQueue; 1568 friend class ScopedGCCriticalSection; 1569 friend class VerifyReferenceCardVisitor; 1570 friend class VerifyReferenceVisitor; 1571 friend class VerifyObjectVisitor; 1572 1573 DISALLOW_IMPLICIT_CONSTRUCTORS(Heap); 1574 }; 1575 1576 } // namespace gc 1577 } // namespace art 1578 1579 #endif // ART_RUNTIME_GC_HEAP_H_ 1580
