1 2 /* 3 * Copyright (C) 2013 The Android Open Source Project 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 #include "rosalloc_space-inl.h" 19 20 #include "base/time_utils.h" 21 #include "gc/accounting/card_table.h" 22 #include "gc/accounting/space_bitmap-inl.h" 23 #include "gc/heap.h" 24 #include "mirror/class-inl.h" 25 #include "mirror/object-inl.h" 26 #include "runtime.h" 27 #include "scoped_thread_state_change-inl.h" 28 #include "thread.h" 29 #include "thread_list.h" 30 #include "utils.h" 31 #include "memory_tool_malloc_space-inl.h" 32 33 namespace art { 34 namespace gc { 35 namespace space { 36 37 static constexpr bool kPrefetchDuringRosAllocFreeList = false; 38 static constexpr size_t kPrefetchLookAhead = 8; 39 // Use this only for verification, it is not safe to use since the class of the object may have 40 // been freed. 41 static constexpr bool kVerifyFreedBytes = false; 42 43 // TODO: Fix 44 // template class MemoryToolMallocSpace<RosAllocSpace, allocator::RosAlloc*>; 45 46 RosAllocSpace::RosAllocSpace(MemMap* mem_map, size_t initial_size, const std::string& name, 47 art::gc::allocator::RosAlloc* rosalloc, uint8_t* begin, uint8_t* end, 48 uint8_t* limit, size_t growth_limit, bool can_move_objects, 49 size_t starting_size, bool low_memory_mode) 50 : MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects, 51 starting_size, initial_size), 52 rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) { 53 CHECK(rosalloc != nullptr); 54 } 55 56 RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap* mem_map, const std::string& name, 57 size_t starting_size, size_t initial_size, 58 size_t growth_limit, size_t capacity, 59 bool low_memory_mode, bool can_move_objects) { 60 DCHECK(mem_map != nullptr); 61 62 bool running_on_memory_tool = Runtime::Current()->IsRunningOnMemoryTool(); 63 64 allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map->Begin(), starting_size, initial_size, 65 capacity, low_memory_mode, running_on_memory_tool); 66 if (rosalloc == nullptr) { 67 LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")"; 68 return nullptr; 69 } 70 71 // Protect memory beyond the starting size. MoreCore will add r/w permissions when necessory 72 uint8_t* end = mem_map->Begin() + starting_size; 73 if (capacity - starting_size > 0) { 74 CHECK_MEMORY_CALL(mprotect, (end, capacity - starting_size, PROT_NONE), name); 75 } 76 77 // Everything is set so record in immutable structure and leave 78 uint8_t* begin = mem_map->Begin(); 79 // TODO: Fix RosAllocSpace to support Valgrind/ASan. There is currently some issues with 80 // AllocationSize caused by redzones. b/12944686 81 if (running_on_memory_tool) { 82 return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>( 83 mem_map, initial_size, name, rosalloc, begin, end, begin + capacity, growth_limit, 84 can_move_objects, starting_size, low_memory_mode); 85 } else { 86 return new RosAllocSpace(mem_map, initial_size, name, rosalloc, begin, end, begin + capacity, 87 growth_limit, can_move_objects, starting_size, low_memory_mode); 88 } 89 } 90 91 RosAllocSpace::~RosAllocSpace() { 92 delete rosalloc_; 93 } 94 95 RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_size, 96 size_t growth_limit, size_t capacity, uint8_t* requested_begin, 97 bool low_memory_mode, bool can_move_objects) { 98 uint64_t start_time = 0; 99 if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { 100 start_time = NanoTime(); 101 VLOG(startup) << "RosAllocSpace::Create entering " << name 102 << " initial_size=" << PrettySize(initial_size) 103 << " growth_limit=" << PrettySize(growth_limit) 104 << " capacity=" << PrettySize(capacity) 105 << " requested_begin=" << reinterpret_cast<void*>(requested_begin); 106 } 107 108 // Memory we promise to rosalloc before it asks for morecore. 109 // Note: making this value large means that large allocations are unlikely to succeed as rosalloc 110 // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the 111 // size of the large allocation) will be greater than the footprint limit. 112 size_t starting_size = Heap::kDefaultStartingSize; 113 MemMap* mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity, 114 requested_begin); 115 if (mem_map == nullptr) { 116 LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size " 117 << PrettySize(capacity); 118 return nullptr; 119 } 120 121 RosAllocSpace* space = CreateFromMemMap(mem_map, name, starting_size, initial_size, 122 growth_limit, capacity, low_memory_mode, 123 can_move_objects); 124 // We start out with only the initial size possibly containing objects. 125 if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { 126 LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time) 127 << " ) " << *space; 128 } 129 return space; 130 } 131 132 allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start, 133 size_t initial_size, 134 size_t maximum_size, bool low_memory_mode, 135 bool running_on_memory_tool) { 136 // clear errno to allow PLOG on error 137 errno = 0; 138 // create rosalloc using our backing storage starting at begin and 139 // with a footprint of morecore_start. When morecore_start bytes of 140 // memory is exhaused morecore will be called. 141 allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc( 142 begin, morecore_start, maximum_size, 143 low_memory_mode ? 144 art::gc::allocator::RosAlloc::kPageReleaseModeAll : 145 art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd, 146 running_on_memory_tool); 147 if (rosalloc != nullptr) { 148 rosalloc->SetFootprintLimit(initial_size); 149 } else { 150 PLOG(ERROR) << "RosAlloc::Create failed"; 151 } 152 return rosalloc; 153 } 154 155 mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, 156 size_t* bytes_allocated, size_t* usable_size, 157 size_t* bytes_tl_bulk_allocated) { 158 mirror::Object* result; 159 { 160 MutexLock mu(self, lock_); 161 // Grow as much as possible within the space. 162 size_t max_allowed = Capacity(); 163 rosalloc_->SetFootprintLimit(max_allowed); 164 // Try the allocation. 165 result = AllocCommon(self, num_bytes, bytes_allocated, usable_size, 166 bytes_tl_bulk_allocated); 167 // Shrink back down as small as possible. 168 size_t footprint = rosalloc_->Footprint(); 169 rosalloc_->SetFootprintLimit(footprint); 170 } 171 // Note RosAlloc zeroes memory internally. 172 // Return the new allocation or null. 173 CHECK(!kDebugSpaces || result == nullptr || Contains(result)); 174 return result; 175 } 176 177 MallocSpace* RosAllocSpace::CreateInstance(MemMap* mem_map, const std::string& name, 178 void* allocator, uint8_t* begin, uint8_t* end, 179 uint8_t* limit, size_t growth_limit, 180 bool can_move_objects) { 181 if (Runtime::Current()->IsRunningOnMemoryTool()) { 182 return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>( 183 mem_map, initial_size_, name, reinterpret_cast<allocator::RosAlloc*>(allocator), begin, end, 184 limit, growth_limit, can_move_objects, starting_size_, low_memory_mode_); 185 } else { 186 return new RosAllocSpace(mem_map, initial_size_, name, 187 reinterpret_cast<allocator::RosAlloc*>(allocator), begin, end, limit, 188 growth_limit, can_move_objects, starting_size_, low_memory_mode_); 189 } 190 } 191 192 size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) { 193 if (kDebugSpaces) { 194 CHECK(ptr != nullptr); 195 CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this; 196 } 197 if (kRecentFreeCount > 0) { 198 MutexLock mu(self, lock_); 199 RegisterRecentFree(ptr); 200 } 201 return rosalloc_->Free(self, ptr); 202 } 203 204 size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) { 205 DCHECK(ptrs != nullptr); 206 207 size_t verify_bytes = 0; 208 for (size_t i = 0; i < num_ptrs; i++) { 209 if (kPrefetchDuringRosAllocFreeList && i + kPrefetchLookAhead < num_ptrs) { 210 __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead])); 211 } 212 if (kVerifyFreedBytes) { 213 verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr); 214 } 215 } 216 217 if (kRecentFreeCount > 0) { 218 MutexLock mu(self, lock_); 219 for (size_t i = 0; i < num_ptrs; i++) { 220 RegisterRecentFree(ptrs[i]); 221 } 222 } 223 224 if (kDebugSpaces) { 225 size_t num_broken_ptrs = 0; 226 for (size_t i = 0; i < num_ptrs; i++) { 227 if (!Contains(ptrs[i])) { 228 num_broken_ptrs++; 229 LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this; 230 } else { 231 size_t size = rosalloc_->UsableSize(ptrs[i]); 232 memset(ptrs[i], 0xEF, size); 233 } 234 } 235 CHECK_EQ(num_broken_ptrs, 0u); 236 } 237 238 const size_t bytes_freed = rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs); 239 if (kVerifyFreedBytes) { 240 CHECK_EQ(verify_bytes, bytes_freed); 241 } 242 return bytes_freed; 243 } 244 245 size_t RosAllocSpace::Trim() { 246 VLOG(heap) << "RosAllocSpace::Trim() "; 247 { 248 Thread* const self = Thread::Current(); 249 // SOA required for Rosalloc::Trim() -> ArtRosAllocMoreCore() -> Heap::GetRosAllocSpace. 250 ScopedObjectAccess soa(self); 251 MutexLock mu(self, lock_); 252 // Trim to release memory at the end of the space. 253 rosalloc_->Trim(); 254 } 255 // Attempt to release pages if it does not release all empty pages. 256 if (!rosalloc_->DoesReleaseAllPages()) { 257 return rosalloc_->ReleasePages(); 258 } 259 return 0; 260 } 261 262 void RosAllocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg), 263 void* arg) { 264 InspectAllRosAlloc(callback, arg, true); 265 } 266 267 size_t RosAllocSpace::GetFootprint() { 268 MutexLock mu(Thread::Current(), lock_); 269 return rosalloc_->Footprint(); 270 } 271 272 size_t RosAllocSpace::GetFootprintLimit() { 273 MutexLock mu(Thread::Current(), lock_); 274 return rosalloc_->FootprintLimit(); 275 } 276 277 void RosAllocSpace::SetFootprintLimit(size_t new_size) { 278 MutexLock mu(Thread::Current(), lock_); 279 VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size); 280 // Compare against the actual footprint, rather than the Size(), because the heap may not have 281 // grown all the way to the allowed size yet. 282 size_t current_space_size = rosalloc_->Footprint(); 283 if (new_size < current_space_size) { 284 // Don't let the space grow any more. 285 new_size = current_space_size; 286 } 287 rosalloc_->SetFootprintLimit(new_size); 288 } 289 290 uint64_t RosAllocSpace::GetBytesAllocated() { 291 size_t bytes_allocated = 0; 292 InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated, false); 293 return bytes_allocated; 294 } 295 296 uint64_t RosAllocSpace::GetObjectsAllocated() { 297 size_t objects_allocated = 0; 298 InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated, false); 299 return objects_allocated; 300 } 301 302 void RosAllocSpace::InspectAllRosAllocWithSuspendAll( 303 void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg), 304 void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS { 305 // TODO: NO_THREAD_SAFETY_ANALYSIS. 306 Thread* self = Thread::Current(); 307 ScopedSuspendAll ssa(__FUNCTION__); 308 MutexLock mu(self, *Locks::runtime_shutdown_lock_); 309 MutexLock mu2(self, *Locks::thread_list_lock_); 310 rosalloc_->InspectAll(callback, arg); 311 if (do_null_callback_at_end) { 312 callback(nullptr, nullptr, 0, arg); // Indicate end of a space. 313 } 314 } 315 316 void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg), 317 void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS { 318 // TODO: NO_THREAD_SAFETY_ANALYSIS. 319 Thread* self = Thread::Current(); 320 if (Locks::mutator_lock_->IsExclusiveHeld(self)) { 321 // The mutators are already suspended. For example, a call path 322 // from SignalCatcher::HandleSigQuit(). 323 rosalloc_->InspectAll(callback, arg); 324 if (do_null_callback_at_end) { 325 callback(nullptr, nullptr, 0, arg); // Indicate end of a space. 326 } 327 } else if (Locks::mutator_lock_->IsSharedHeld(self)) { 328 // The mutators are not suspended yet and we have a shared access 329 // to the mutator lock. Temporarily release the shared access by 330 // transitioning to the suspend state, and suspend the mutators. 331 ScopedThreadSuspension sts(self, kSuspended); 332 InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end); 333 } else { 334 // The mutators are not suspended yet. Suspend the mutators. 335 InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end); 336 } 337 } 338 339 size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) { 340 return rosalloc_->RevokeThreadLocalRuns(thread); 341 } 342 343 size_t RosAllocSpace::RevokeAllThreadLocalBuffers() { 344 return rosalloc_->RevokeAllThreadLocalRuns(); 345 } 346 347 void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { 348 if (kIsDebugBuild) { 349 rosalloc_->AssertThreadLocalRunsAreRevoked(thread); 350 } 351 } 352 353 void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() { 354 if (kIsDebugBuild) { 355 rosalloc_->AssertAllThreadLocalRunsAreRevoked(); 356 } 357 } 358 359 void RosAllocSpace::Clear() { 360 size_t footprint_limit = GetFootprintLimit(); 361 madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED); 362 live_bitmap_->Clear(); 363 mark_bitmap_->Clear(); 364 SetEnd(begin_ + starting_size_); 365 delete rosalloc_; 366 rosalloc_ = CreateRosAlloc(mem_map_->Begin(), starting_size_, initial_size_, 367 NonGrowthLimitCapacity(), low_memory_mode_, 368 Runtime::Current()->IsRunningOnMemoryTool()); 369 SetFootprintLimit(footprint_limit); 370 } 371 372 void RosAllocSpace::DumpStats(std::ostream& os) { 373 ScopedSuspendAll ssa(__FUNCTION__); 374 rosalloc_->DumpStats(os); 375 } 376 377 template<bool kMaybeIsRunningOnMemoryTool> 378 size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { 379 // obj is a valid object. Use its class in the header to get the size. 380 // Don't use verification since the object may be dead if we are sweeping. 381 size_t size = obj->SizeOf<kVerifyNone>(); 382 bool add_redzones = false; 383 if (kMaybeIsRunningOnMemoryTool) { 384 add_redzones = RUNNING_ON_MEMORY_TOOL ? kMemoryToolAddsRedzones : 0; 385 if (add_redzones) { 386 size += 2 * kDefaultMemoryToolRedZoneBytes; 387 } 388 } else { 389 DCHECK_EQ(RUNNING_ON_MEMORY_TOOL, 0U); 390 } 391 size_t size_by_size = rosalloc_->UsableSize(size); 392 if (kIsDebugBuild) { 393 // On memory tool, the red zone has an impact... 394 const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj); 395 size_t size_by_ptr = rosalloc_->UsableSize( 396 obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0)); 397 if (size_by_size != size_by_ptr) { 398 LOG(INFO) << "Found a bad sized obj of size " << size 399 << " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec 400 << " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr; 401 } 402 DCHECK_EQ(size_by_size, size_by_ptr); 403 } 404 if (usable_size != nullptr) { 405 *usable_size = size_by_size; 406 } 407 return size_by_size; 408 } 409 410 } // namespace space 411 412 namespace allocator { 413 414 // Callback from rosalloc when it needs to increase the footprint. 415 void* ArtRosAllocMoreCore(allocator::RosAlloc* rosalloc, intptr_t increment) 416 REQUIRES_SHARED(Locks::mutator_lock_) { 417 Heap* heap = Runtime::Current()->GetHeap(); 418 art::gc::space::RosAllocSpace* rosalloc_space = heap->GetRosAllocSpace(rosalloc); 419 DCHECK(rosalloc_space != nullptr); 420 DCHECK_EQ(rosalloc_space->GetRosAlloc(), rosalloc); 421 return rosalloc_space->MoreCore(increment); 422 } 423 424 } // namespace allocator 425 426 } // namespace gc 427 } // namespace art 428