1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_RUNTIME_GC_HEAP_INL_H_ 18 #define ART_RUNTIME_GC_HEAP_INL_H_ 19 20 #include "heap.h" 21 22 #include "allocation_listener.h" 23 #include "base/quasi_atomic.h" 24 #include "base/time_utils.h" 25 #include "gc/accounting/atomic_stack.h" 26 #include "gc/accounting/card_table-inl.h" 27 #include "gc/allocation_record.h" 28 #include "gc/collector/semi_space.h" 29 #include "gc/space/bump_pointer_space-inl.h" 30 #include "gc/space/dlmalloc_space-inl.h" 31 #include "gc/space/large_object_space.h" 32 #include "gc/space/region_space-inl.h" 33 #include "gc/space/rosalloc_space-inl.h" 34 #include "handle_scope-inl.h" 35 #include "obj_ptr-inl.h" 36 #include "runtime.h" 37 #include "thread-inl.h" 38 #include "verify_object.h" 39 #include "write_barrier-inl.h" 40 41 namespace art { 42 namespace gc { 43 44 template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor> 45 inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, 46 ObjPtr<mirror::Class> klass, 47 size_t byte_count, 48 AllocatorType allocator, 49 const PreFenceVisitor& pre_fence_visitor) { 50 if (kIsDebugBuild) { 51 CheckPreconditionsForAllocObject(klass, byte_count); 52 // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are 53 // done in the runnable state where suspension is expected. 54 CHECK_EQ(self->GetState(), kRunnable); 55 self->AssertThreadSuspensionIsAllowable(); 56 self->AssertNoPendingException(); 57 // Make sure to preserve klass. 58 StackHandleScope<1> hs(self); 59 HandleWrapperObjPtr<mirror::Class> h = hs.NewHandleWrapper(&klass); 60 self->PoisonObjectPointers(); 61 } 62 // Need to check that we aren't the large object allocator since the large object allocation code 63 // path includes this function. If we didn't check we would have an infinite loop. 64 ObjPtr<mirror::Object> obj; 65 if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) { 66 obj = AllocLargeObject<kInstrumented, PreFenceVisitor>(self, &klass, byte_count, 67 pre_fence_visitor); 68 if (obj != nullptr) { 69 return obj.Ptr(); 70 } else { 71 // There should be an OOM exception, since we are retrying, clear it. 72 self->ClearException(); 73 } 74 // If the large object allocation failed, try to use the normal spaces (main space, 75 // non moving space). This can happen if there is significant virtual address space 76 // fragmentation. 77 } 78 // bytes allocated for the (individual) object. 79 size_t bytes_allocated; 80 size_t usable_size; 81 size_t new_num_bytes_allocated = 0; 82 if (IsTLABAllocator(allocator)) { 83 byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment); 84 } 85 // If we have a thread local allocation we don't need to update bytes allocated. 86 if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) { 87 obj = self->AllocTlab(byte_count); 88 DCHECK(obj != nullptr) << "AllocTlab can't fail"; 89 obj->SetClass(klass); 90 if (kUseBakerReadBarrier) { 91 obj->AssertReadBarrierState(); 92 } 93 bytes_allocated = byte_count; 94 usable_size = bytes_allocated; 95 pre_fence_visitor(obj, usable_size); 96 QuasiAtomic::ThreadFenceForConstructor(); 97 } else if ( 98 !kInstrumented && allocator == kAllocatorTypeRosAlloc && 99 (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) != nullptr && 100 LIKELY(obj != nullptr)) { 101 DCHECK(!is_running_on_memory_tool_); 102 obj->SetClass(klass); 103 if (kUseBakerReadBarrier) { 104 obj->AssertReadBarrierState(); 105 } 106 usable_size = bytes_allocated; 107 pre_fence_visitor(obj, usable_size); 108 QuasiAtomic::ThreadFenceForConstructor(); 109 } else { 110 // Bytes allocated that includes bulk thread-local buffer allocations in addition to direct 111 // non-TLAB object allocations. 112 size_t bytes_tl_bulk_allocated = 0u; 113 obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated, 114 &usable_size, &bytes_tl_bulk_allocated); 115 if (UNLIKELY(obj == nullptr)) { 116 // AllocateInternalWithGc can cause thread suspension, if someone instruments the entrypoints 117 // or changes the allocator in a suspend point here, we need to retry the allocation. 118 obj = AllocateInternalWithGc(self, 119 allocator, 120 kInstrumented, 121 byte_count, 122 &bytes_allocated, 123 &usable_size, 124 &bytes_tl_bulk_allocated, &klass); 125 if (obj == nullptr) { 126 // The only way that we can get a null return if there is no pending exception is if the 127 // allocator or instrumentation changed. 128 if (!self->IsExceptionPending()) { 129 // AllocObject will pick up the new allocator type, and instrumented as true is the safe 130 // default. 131 return AllocObject</*kInstrumented=*/true>(self, 132 klass, 133 byte_count, 134 pre_fence_visitor); 135 } 136 return nullptr; 137 } 138 } 139 DCHECK_GT(bytes_allocated, 0u); 140 DCHECK_GT(usable_size, 0u); 141 obj->SetClass(klass); 142 if (kUseBakerReadBarrier) { 143 obj->AssertReadBarrierState(); 144 } 145 if (collector::SemiSpace::kUseRememberedSet && UNLIKELY(allocator == kAllocatorTypeNonMoving)) { 146 // (Note this if statement will be constant folded away for the 147 // fast-path quick entry points.) Because SetClass() has no write 148 // barrier, if a non-moving space allocation, we need a write 149 // barrier as the class pointer may point to the bump pointer 150 // space (where the class pointer is an "old-to-young" reference, 151 // though rare) under the GSS collector with the remembered set 152 // enabled. We don't need this for kAllocatorTypeRosAlloc/DlMalloc 153 // cases because we don't directly allocate into the main alloc 154 // space (besides promotions) under the SS/GSS collector. 155 WriteBarrier::ForFieldWrite(obj, mirror::Object::ClassOffset(), klass); 156 } 157 pre_fence_visitor(obj, usable_size); 158 QuasiAtomic::ThreadFenceForConstructor(); 159 if (bytes_tl_bulk_allocated > 0) { 160 size_t num_bytes_allocated_before = 161 num_bytes_allocated_.fetch_add(bytes_tl_bulk_allocated, std::memory_order_relaxed); 162 new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated; 163 // Only trace when we get an increase in the number of bytes allocated. This happens when 164 // obtaining a new TLAB and isn't often enough to hurt performance according to golem. 165 TraceHeapSize(new_num_bytes_allocated); 166 } 167 } 168 if (kIsDebugBuild && Runtime::Current()->IsStarted()) { 169 CHECK_LE(obj->SizeOf(), usable_size); 170 } 171 // TODO: Deprecate. 172 if (kInstrumented) { 173 if (Runtime::Current()->HasStatsEnabled()) { 174 RuntimeStats* thread_stats = self->GetStats(); 175 ++thread_stats->allocated_objects; 176 thread_stats->allocated_bytes += bytes_allocated; 177 RuntimeStats* global_stats = Runtime::Current()->GetStats(); 178 ++global_stats->allocated_objects; 179 global_stats->allocated_bytes += bytes_allocated; 180 } 181 } else { 182 DCHECK(!Runtime::Current()->HasStatsEnabled()); 183 } 184 if (kInstrumented) { 185 if (IsAllocTrackingEnabled()) { 186 // allocation_records_ is not null since it never becomes null after allocation tracking is 187 // enabled. 188 DCHECK(allocation_records_ != nullptr); 189 allocation_records_->RecordAllocation(self, &obj, bytes_allocated); 190 } 191 AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst); 192 if (l != nullptr) { 193 // Same as above. We assume that a listener that was once stored will never be deleted. 194 // Otherwise we'd have to perform this under a lock. 195 l->ObjectAllocated(self, &obj, bytes_allocated); 196 } 197 } else { 198 DCHECK(!IsAllocTrackingEnabled()); 199 } 200 if (AllocatorHasAllocationStack(allocator)) { 201 PushOnAllocationStack(self, &obj); 202 } 203 if (kInstrumented) { 204 if (gc_stress_mode_) { 205 CheckGcStressMode(self, &obj); 206 } 207 } else { 208 DCHECK(!gc_stress_mode_); 209 } 210 // IsGcConcurrent() isn't known at compile time so we can optimize by not checking it for 211 // the BumpPointer or TLAB allocators. This is nice since it allows the entire if statement to be 212 // optimized out. And for the other allocators, AllocatorMayHaveConcurrentGC is a constant since 213 // the allocator_type should be constant propagated. 214 if (AllocatorMayHaveConcurrentGC(allocator) && IsGcConcurrent()) { 215 // New_num_bytes_allocated is zero if we didn't update num_bytes_allocated_. 216 // That's fine. 217 CheckConcurrentGCForJava(self, new_num_bytes_allocated, &obj); 218 } 219 VerifyObject(obj); 220 self->VerifyStack(); 221 return obj.Ptr(); 222 } 223 224 // The size of a thread-local allocation stack in the number of references. 225 static constexpr size_t kThreadLocalAllocationStackSize = 128; 226 227 inline void Heap::PushOnAllocationStack(Thread* self, ObjPtr<mirror::Object>* obj) { 228 if (kUseThreadLocalAllocationStack) { 229 if (UNLIKELY(!self->PushOnThreadLocalAllocationStack(obj->Ptr()))) { 230 PushOnThreadLocalAllocationStackWithInternalGC(self, obj); 231 } 232 } else if (UNLIKELY(!allocation_stack_->AtomicPushBack(obj->Ptr()))) { 233 PushOnAllocationStackWithInternalGC(self, obj); 234 } 235 } 236 237 template <bool kInstrumented, typename PreFenceVisitor> 238 inline mirror::Object* Heap::AllocLargeObject(Thread* self, 239 ObjPtr<mirror::Class>* klass, 240 size_t byte_count, 241 const PreFenceVisitor& pre_fence_visitor) { 242 // Save and restore the class in case it moves. 243 StackHandleScope<1> hs(self); 244 auto klass_wrapper = hs.NewHandleWrapper(klass); 245 return AllocObjectWithAllocator<kInstrumented, false, PreFenceVisitor>(self, *klass, byte_count, 246 kAllocatorTypeLOS, 247 pre_fence_visitor); 248 } 249 250 template <const bool kInstrumented, const bool kGrow> 251 inline mirror::Object* Heap::TryToAllocate(Thread* self, 252 AllocatorType allocator_type, 253 size_t alloc_size, 254 size_t* bytes_allocated, 255 size_t* usable_size, 256 size_t* bytes_tl_bulk_allocated) { 257 if (allocator_type != kAllocatorTypeRegionTLAB && 258 allocator_type != kAllocatorTypeTLAB && 259 allocator_type != kAllocatorTypeRosAlloc && 260 UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, alloc_size, kGrow))) { 261 return nullptr; 262 } 263 mirror::Object* ret; 264 switch (allocator_type) { 265 case kAllocatorTypeBumpPointer: { 266 DCHECK(bump_pointer_space_ != nullptr); 267 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment); 268 ret = bump_pointer_space_->AllocNonvirtual(alloc_size); 269 if (LIKELY(ret != nullptr)) { 270 *bytes_allocated = alloc_size; 271 *usable_size = alloc_size; 272 *bytes_tl_bulk_allocated = alloc_size; 273 } 274 break; 275 } 276 case kAllocatorTypeRosAlloc: { 277 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) { 278 // If running on ASan, we should be using the instrumented path. 279 size_t max_bytes_tl_bulk_allocated = rosalloc_space_->MaxBytesBulkAllocatedFor(alloc_size); 280 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, 281 max_bytes_tl_bulk_allocated, 282 kGrow))) { 283 return nullptr; 284 } 285 ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size, 286 bytes_tl_bulk_allocated); 287 } else { 288 DCHECK(!is_running_on_memory_tool_); 289 size_t max_bytes_tl_bulk_allocated = 290 rosalloc_space_->MaxBytesBulkAllocatedForNonvirtual(alloc_size); 291 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, 292 max_bytes_tl_bulk_allocated, 293 kGrow))) { 294 return nullptr; 295 } 296 if (!kInstrumented) { 297 DCHECK(!rosalloc_space_->CanAllocThreadLocal(self, alloc_size)); 298 } 299 ret = rosalloc_space_->AllocNonvirtual(self, 300 alloc_size, 301 bytes_allocated, 302 usable_size, 303 bytes_tl_bulk_allocated); 304 } 305 break; 306 } 307 case kAllocatorTypeDlMalloc: { 308 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) { 309 // If running on ASan, we should be using the instrumented path. 310 ret = dlmalloc_space_->Alloc(self, 311 alloc_size, 312 bytes_allocated, 313 usable_size, 314 bytes_tl_bulk_allocated); 315 } else { 316 DCHECK(!is_running_on_memory_tool_); 317 ret = dlmalloc_space_->AllocNonvirtual(self, 318 alloc_size, 319 bytes_allocated, 320 usable_size, 321 bytes_tl_bulk_allocated); 322 } 323 break; 324 } 325 case kAllocatorTypeNonMoving: { 326 ret = non_moving_space_->Alloc(self, 327 alloc_size, 328 bytes_allocated, 329 usable_size, 330 bytes_tl_bulk_allocated); 331 break; 332 } 333 case kAllocatorTypeLOS: { 334 ret = large_object_space_->Alloc(self, 335 alloc_size, 336 bytes_allocated, 337 usable_size, 338 bytes_tl_bulk_allocated); 339 // Note that the bump pointer spaces aren't necessarily next to 340 // the other continuous spaces like the non-moving alloc space or 341 // the zygote space. 342 DCHECK(ret == nullptr || large_object_space_->Contains(ret)); 343 break; 344 } 345 case kAllocatorTypeRegion: { 346 DCHECK(region_space_ != nullptr); 347 alloc_size = RoundUp(alloc_size, space::RegionSpace::kAlignment); 348 ret = region_space_->AllocNonvirtual<false>(alloc_size, 349 bytes_allocated, 350 usable_size, 351 bytes_tl_bulk_allocated); 352 break; 353 } 354 case kAllocatorTypeTLAB: 355 FALLTHROUGH_INTENDED; 356 case kAllocatorTypeRegionTLAB: { 357 DCHECK_ALIGNED(alloc_size, kObjectAlignment); 358 static_assert(space::RegionSpace::kAlignment == space::BumpPointerSpace::kAlignment, 359 "mismatched alignments"); 360 static_assert(kObjectAlignment == space::BumpPointerSpace::kAlignment, 361 "mismatched alignments"); 362 if (UNLIKELY(self->TlabSize() < alloc_size)) { 363 // kAllocatorTypeTLAB may be the allocator for region space TLAB if the GC is not marking, 364 // that is why the allocator is not passed down. 365 return AllocWithNewTLAB(self, 366 alloc_size, 367 kGrow, 368 bytes_allocated, 369 usable_size, 370 bytes_tl_bulk_allocated); 371 } 372 // The allocation can't fail. 373 ret = self->AllocTlab(alloc_size); 374 DCHECK(ret != nullptr); 375 *bytes_allocated = alloc_size; 376 *bytes_tl_bulk_allocated = 0; // Allocated in an existing buffer. 377 *usable_size = alloc_size; 378 break; 379 } 380 default: { 381 LOG(FATAL) << "Invalid allocator type"; 382 ret = nullptr; 383 } 384 } 385 return ret; 386 } 387 388 inline bool Heap::ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_count) const { 389 // We need to have a zygote space or else our newly allocated large object can end up in the 390 // Zygote resulting in it being prematurely freed. 391 // We can only do this for primitive objects since large objects will not be within the card table 392 // range. This also means that we rely on SetClass not dirtying the object's card. 393 return byte_count >= large_object_threshold_ && (c->IsPrimitiveArray() || c->IsStringClass()); 394 } 395 396 inline bool Heap::IsOutOfMemoryOnAllocation(AllocatorType allocator_type, 397 size_t alloc_size, 398 bool grow) { 399 size_t old_target = target_footprint_.load(std::memory_order_relaxed); 400 while (true) { 401 size_t old_allocated = num_bytes_allocated_.load(std::memory_order_relaxed); 402 size_t new_footprint = old_allocated + alloc_size; 403 // Tests against heap limits are inherently approximate, since multiple allocations may 404 // race, and this is not atomic with the allocation. 405 if (UNLIKELY(new_footprint <= old_target)) { 406 return false; 407 } else if (UNLIKELY(new_footprint > growth_limit_)) { 408 return true; 409 } 410 // We are between target_footprint_ and growth_limit_ . 411 if (AllocatorMayHaveConcurrentGC(allocator_type) && IsGcConcurrent()) { 412 return false; 413 } else { 414 if (grow) { 415 if (target_footprint_.compare_exchange_weak(/*inout ref*/old_target, new_footprint, 416 std::memory_order_relaxed)) { 417 VlogHeapGrowth(old_target, new_footprint, alloc_size); 418 return false; 419 } // else try again. 420 } else { 421 return true; 422 } 423 } 424 } 425 } 426 427 inline bool Heap::ShouldConcurrentGCForJava(size_t new_num_bytes_allocated) { 428 // For a Java allocation, we only check whether the number of Java allocated bytes excceeds a 429 // threshold. By not considering native allocation here, we (a) ensure that Java heap bounds are 430 // maintained, and (b) reduce the cost of the check here. 431 return new_num_bytes_allocated >= concurrent_start_bytes_; 432 } 433 434 inline void Heap::CheckConcurrentGCForJava(Thread* self, 435 size_t new_num_bytes_allocated, 436 ObjPtr<mirror::Object>* obj) { 437 if (UNLIKELY(ShouldConcurrentGCForJava(new_num_bytes_allocated))) { 438 RequestConcurrentGCAndSaveObject(self, false /* force_full */, obj); 439 } 440 } 441 442 } // namespace gc 443 } // namespace art 444 445 #endif // ART_RUNTIME_GC_HEAP_INL_H_ 446
