1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "cha.h" 18 19 #include "art_method-inl.h" 20 #include "jit/jit.h" 21 #include "jit/jit_code_cache.h" 22 #include "runtime.h" 23 #include "scoped_thread_state_change-inl.h" 24 #include "stack.h" 25 #include "thread.h" 26 #include "thread_list.h" 27 #include "thread_pool.h" 28 29 namespace art { 30 31 void ClassHierarchyAnalysis::AddDependency(ArtMethod* method, 32 ArtMethod* dependent_method, 33 OatQuickMethodHeader* dependent_header) { 34 auto it = cha_dependency_map_.find(method); 35 if (it == cha_dependency_map_.end()) { 36 cha_dependency_map_[method] = 37 new std::vector<std::pair<art::ArtMethod*, art::OatQuickMethodHeader*>>(); 38 it = cha_dependency_map_.find(method); 39 } else { 40 DCHECK(it->second != nullptr); 41 } 42 it->second->push_back(std::make_pair(dependent_method, dependent_header)); 43 } 44 45 std::vector<std::pair<ArtMethod*, OatQuickMethodHeader*>>* 46 ClassHierarchyAnalysis::GetDependents(ArtMethod* method) { 47 auto it = cha_dependency_map_.find(method); 48 if (it != cha_dependency_map_.end()) { 49 DCHECK(it->second != nullptr); 50 return it->second; 51 } 52 return nullptr; 53 } 54 55 void ClassHierarchyAnalysis::RemoveDependencyFor(ArtMethod* method) { 56 auto it = cha_dependency_map_.find(method); 57 if (it != cha_dependency_map_.end()) { 58 auto dependents = it->second; 59 cha_dependency_map_.erase(it); 60 delete dependents; 61 } 62 } 63 64 void ClassHierarchyAnalysis::RemoveDependentsWithMethodHeaders( 65 const std::unordered_set<OatQuickMethodHeader*>& method_headers) { 66 // Iterate through all entries in the dependency map and remove any entry that 67 // contains one of those in method_headers. 68 for (auto map_it = cha_dependency_map_.begin(); map_it != cha_dependency_map_.end(); ) { 69 auto dependents = map_it->second; 70 for (auto vec_it = dependents->begin(); vec_it != dependents->end(); ) { 71 OatQuickMethodHeader* method_header = vec_it->second; 72 auto it = std::find(method_headers.begin(), method_headers.end(), method_header); 73 if (it != method_headers.end()) { 74 vec_it = dependents->erase(vec_it); 75 } else { 76 vec_it++; 77 } 78 } 79 // Remove the map entry if there are no more dependents. 80 if (dependents->empty()) { 81 map_it = cha_dependency_map_.erase(map_it); 82 delete dependents; 83 } else { 84 map_it++; 85 } 86 } 87 } 88 89 // This stack visitor walks the stack and for compiled code with certain method 90 // headers, sets the should_deoptimize flag on stack to 1. 91 // TODO: also set the register value to 1 when should_deoptimize is allocated in 92 // a register. 93 class CHAStackVisitor FINAL : public StackVisitor { 94 public: 95 CHAStackVisitor(Thread* thread_in, 96 Context* context, 97 const std::unordered_set<OatQuickMethodHeader*>& method_headers) 98 : StackVisitor(thread_in, context, StackVisitor::StackWalkKind::kSkipInlinedFrames), 99 method_headers_(method_headers) { 100 } 101 102 bool VisitFrame() OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { 103 ArtMethod* method = GetMethod(); 104 // Avoid types of methods that do not have an oat quick method header. 105 if (method == nullptr || 106 method->IsRuntimeMethod() || 107 method->IsNative() || 108 method->IsProxyMethod()) { 109 return true; 110 } 111 if (GetCurrentQuickFrame() == nullptr) { 112 // Not compiled code. 113 return true; 114 } 115 // Method may have multiple versions of compiled code. Check 116 // the method header to see if it has should_deoptimize flag. 117 const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader(); 118 DCHECK(method_header != nullptr); 119 if (!method_header->HasShouldDeoptimizeFlag()) { 120 // This compiled version doesn't have should_deoptimize flag. Skip. 121 return true; 122 } 123 auto it = std::find(method_headers_.begin(), method_headers_.end(), method_header); 124 if (it == method_headers_.end()) { 125 // Not in the list of method headers that should be deoptimized. 126 return true; 127 } 128 129 // The compiled code on stack is not valid anymore. Need to deoptimize. 130 SetShouldDeoptimizeFlag(); 131 132 return true; 133 } 134 135 private: 136 void SetShouldDeoptimizeFlag() REQUIRES_SHARED(Locks::mutator_lock_) { 137 QuickMethodFrameInfo frame_info = GetCurrentQuickFrameInfo(); 138 size_t frame_size = frame_info.FrameSizeInBytes(); 139 uint8_t* sp = reinterpret_cast<uint8_t*>(GetCurrentQuickFrame()); 140 size_t core_spill_size = POPCOUNT(frame_info.CoreSpillMask()) * 141 GetBytesPerGprSpillLocation(kRuntimeISA); 142 size_t fpu_spill_size = POPCOUNT(frame_info.FpSpillMask()) * 143 GetBytesPerFprSpillLocation(kRuntimeISA); 144 size_t offset = frame_size - core_spill_size - fpu_spill_size - kShouldDeoptimizeFlagSize; 145 uint8_t* should_deoptimize_addr = sp + offset; 146 // Set deoptimization flag to 1. 147 DCHECK(*should_deoptimize_addr == 0 || *should_deoptimize_addr == 1); 148 *should_deoptimize_addr = 1; 149 } 150 151 // Set of method headers for compiled code that should be deoptimized. 152 const std::unordered_set<OatQuickMethodHeader*>& method_headers_; 153 154 DISALLOW_COPY_AND_ASSIGN(CHAStackVisitor); 155 }; 156 157 class CHACheckpoint FINAL : public Closure { 158 public: 159 explicit CHACheckpoint(const std::unordered_set<OatQuickMethodHeader*>& method_headers) 160 : barrier_(0), 161 method_headers_(method_headers) {} 162 163 void Run(Thread* thread) OVERRIDE { 164 // Note thread and self may not be equal if thread was already suspended at 165 // the point of the request. 166 Thread* self = Thread::Current(); 167 ScopedObjectAccess soa(self); 168 CHAStackVisitor visitor(thread, nullptr, method_headers_); 169 visitor.WalkStack(); 170 barrier_.Pass(self); 171 } 172 173 void WaitForThreadsToRunThroughCheckpoint(size_t threads_running_checkpoint) { 174 Thread* self = Thread::Current(); 175 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 176 barrier_.Increment(self, threads_running_checkpoint); 177 } 178 179 private: 180 // The barrier to be passed through and for the requestor to wait upon. 181 Barrier barrier_; 182 // List of method headers for invalidated compiled code. 183 const std::unordered_set<OatQuickMethodHeader*>& method_headers_; 184 185 DISALLOW_COPY_AND_ASSIGN(CHACheckpoint); 186 }; 187 188 void ClassHierarchyAnalysis::VerifyNonSingleImplementation(mirror::Class* verify_class, 189 uint16_t verify_index, 190 ArtMethod* excluded_method) { 191 // Grab cha_lock_ to make sure all single-implementation updates are seen. 192 PointerSize image_pointer_size = 193 Runtime::Current()->GetClassLinker()->GetImagePointerSize(); 194 MutexLock cha_mu(Thread::Current(), *Locks::cha_lock_); 195 while (verify_class != nullptr) { 196 if (verify_index >= verify_class->GetVTableLength()) { 197 return; 198 } 199 ArtMethod* verify_method = verify_class->GetVTableEntry(verify_index, image_pointer_size); 200 if (verify_method != excluded_method) { 201 DCHECK(!verify_method->HasSingleImplementation()) 202 << "class: " << verify_class->PrettyClass() 203 << " verify_method: " << verify_method->PrettyMethod(true) 204 << " excluded_method: " << excluded_method->PrettyMethod(true); 205 if (verify_method->IsAbstract()) { 206 DCHECK(verify_method->GetSingleImplementation(image_pointer_size) == nullptr); 207 } 208 } 209 verify_class = verify_class->GetSuperClass(); 210 } 211 } 212 213 void ClassHierarchyAnalysis::CheckVirtualMethodSingleImplementationInfo( 214 Handle<mirror::Class> klass, 215 ArtMethod* virtual_method, 216 ArtMethod* method_in_super, 217 std::unordered_set<ArtMethod*>& invalidated_single_impl_methods, 218 PointerSize pointer_size) { 219 // TODO: if klass is not instantiable, virtual_method isn't invocable yet so 220 // even if it overrides, it doesn't invalidate single-implementation 221 // assumption. 222 223 DCHECK((virtual_method != method_in_super) || virtual_method->IsAbstract()); 224 DCHECK(method_in_super->GetDeclaringClass()->IsResolved()) << "class isn't resolved"; 225 // If virtual_method doesn't come from a default interface method, it should 226 // be supplied by klass. 227 DCHECK(virtual_method == method_in_super || 228 virtual_method->IsCopied() || 229 virtual_method->GetDeclaringClass() == klass.Get()); 230 231 // To make updating single-implementation flags simple, we always maintain the following 232 // invariant: 233 // Say all virtual methods in the same vtable slot, starting from the bottom child class 234 // to super classes, is a sequence of unique methods m3, m2, m1, ... (after removing duplicate 235 // methods for inherited methods). 236 // For example for the following class hierarchy, 237 // class A { void m() { ... } } 238 // class B extends A { void m() { ... } } 239 // class C extends B {} 240 // class D extends C { void m() { ... } } 241 // the sequence is D.m(), B.m(), A.m(). 242 // The single-implementation status for that sequence of methods begin with one or two true's, 243 // then become all falses. The only case where two true's are possible is for one abstract 244 // method m and one non-abstract method mImpl that overrides method m. 245 // With the invariant, when linking in a new class, we only need to at most update one or 246 // two methods in the sequence for their single-implementation status, in order to maintain 247 // the invariant. 248 249 if (!method_in_super->HasSingleImplementation()) { 250 // method_in_super already has multiple implementations. All methods in the 251 // same vtable slots in its super classes should have 252 // non-single-implementation already. 253 if (kIsDebugBuild) { 254 VerifyNonSingleImplementation(klass->GetSuperClass()->GetSuperClass(), 255 method_in_super->GetMethodIndex(), 256 nullptr /* excluded_method */); 257 } 258 return; 259 } 260 261 uint16_t method_index = method_in_super->GetMethodIndex(); 262 if (method_in_super->IsAbstract()) { 263 if (kIsDebugBuild) { 264 // An abstract method should have made all methods in the same vtable 265 // slot above it in the class hierarchy having non-single-implementation. 266 mirror::Class* super_super = klass->GetSuperClass()->GetSuperClass(); 267 VerifyNonSingleImplementation(super_super, 268 method_index, 269 method_in_super); 270 } 271 272 if (virtual_method->IsAbstract()) { 273 // SUPER: abstract, VIRTUAL: abstract. 274 if (method_in_super == virtual_method) { 275 DCHECK(klass->IsInstantiable()); 276 // An instantiable subclass hasn't provided a concrete implementation of 277 // the abstract method. Invoking method_in_super may throw AbstractMethodError. 278 // This is an uncommon case, so we simply treat method_in_super as not 279 // having single-implementation. 280 invalidated_single_impl_methods.insert(method_in_super); 281 return; 282 } else { 283 // One abstract method overrides another abstract method. This is an uncommon 284 // case. We simply treat method_in_super as not having single-implementation. 285 invalidated_single_impl_methods.insert(method_in_super); 286 return; 287 } 288 } else { 289 // SUPER: abstract, VIRTUAL: non-abstract. 290 // A non-abstract method overrides an abstract method. 291 if (method_in_super->GetSingleImplementation(pointer_size) == nullptr) { 292 // Abstract method_in_super has no implementation yet. 293 // We need to grab cha_lock_ since there may be multiple class linking 294 // going on that can check/modify the single-implementation flag/method 295 // of method_in_super. 296 MutexLock cha_mu(Thread::Current(), *Locks::cha_lock_); 297 if (!method_in_super->HasSingleImplementation()) { 298 return; 299 } 300 if (method_in_super->GetSingleImplementation(pointer_size) == nullptr) { 301 // virtual_method becomes the first implementation for method_in_super. 302 method_in_super->SetSingleImplementation(virtual_method, pointer_size); 303 // Keep method_in_super's single-implementation status. 304 return; 305 } 306 // Fall through to invalidate method_in_super's single-implementation status. 307 } 308 // Abstract method_in_super already got one implementation. 309 // Invalidate method_in_super's single-implementation status. 310 invalidated_single_impl_methods.insert(method_in_super); 311 return; 312 } 313 } else { 314 if (virtual_method->IsAbstract()) { 315 // SUPER: non-abstract, VIRTUAL: abstract. 316 // An abstract method overrides a non-abstract method. This is an uncommon 317 // case, we simply treat both methods as not having single-implementation. 318 invalidated_single_impl_methods.insert(virtual_method); 319 // Fall-through to handle invalidating method_in_super of its 320 // single-implementation status. 321 } 322 323 // SUPER: non-abstract, VIRTUAL: non-abstract/abstract(fall-through from previous if). 324 // Invalidate method_in_super's single-implementation status. 325 invalidated_single_impl_methods.insert(method_in_super); 326 327 // method_in_super might be the single-implementation of another abstract method, 328 // which should be also invalidated of its single-implementation status. 329 mirror::Class* super_super = klass->GetSuperClass()->GetSuperClass(); 330 while (super_super != nullptr && 331 method_index < super_super->GetVTableLength()) { 332 ArtMethod* method_in_super_super = super_super->GetVTableEntry(method_index, pointer_size); 333 if (method_in_super_super != method_in_super) { 334 if (method_in_super_super->IsAbstract()) { 335 if (method_in_super_super->HasSingleImplementation()) { 336 // Invalidate method_in_super's single-implementation status. 337 invalidated_single_impl_methods.insert(method_in_super_super); 338 // No need to further traverse up the class hierarchy since if there 339 // are cases that one abstract method overrides another method, we 340 // should have made that method having non-single-implementation already. 341 } else { 342 // method_in_super_super is already non-single-implementation. 343 // No need to further traverse up the class hierarchy. 344 } 345 } else { 346 DCHECK(!method_in_super_super->HasSingleImplementation()); 347 // No need to further traverse up the class hierarchy since two non-abstract 348 // methods (method_in_super and method_in_super_super) should have set all 349 // other methods (abstract or not) in the vtable slot to be non-single-implementation. 350 } 351 352 if (kIsDebugBuild) { 353 VerifyNonSingleImplementation(super_super->GetSuperClass(), 354 method_index, 355 method_in_super_super); 356 } 357 // No need to go any further. 358 return; 359 } else { 360 super_super = super_super->GetSuperClass(); 361 } 362 } 363 } 364 } 365 366 void ClassHierarchyAnalysis::CheckInterfaceMethodSingleImplementationInfo( 367 Handle<mirror::Class> klass, 368 ArtMethod* interface_method, 369 ArtMethod* implementation_method, 370 std::unordered_set<ArtMethod*>& invalidated_single_impl_methods, 371 PointerSize pointer_size) { 372 DCHECK(klass->IsInstantiable()); 373 DCHECK(interface_method->IsAbstract() || interface_method->IsDefault()); 374 375 if (!interface_method->HasSingleImplementation()) { 376 return; 377 } 378 379 if (implementation_method->IsAbstract()) { 380 // An instantiable class doesn't supply an implementation for 381 // interface_method. Invoking the interface method on the class will throw 382 // AbstractMethodError. This is an uncommon case, so we simply treat 383 // interface_method as not having single-implementation. 384 invalidated_single_impl_methods.insert(interface_method); 385 return; 386 } 387 388 // We need to grab cha_lock_ since there may be multiple class linking going 389 // on that can check/modify the single-implementation flag/method of 390 // interface_method. 391 MutexLock cha_mu(Thread::Current(), *Locks::cha_lock_); 392 // Do this check again after we grab cha_lock_. 393 if (!interface_method->HasSingleImplementation()) { 394 return; 395 } 396 397 ArtMethod* single_impl = interface_method->GetSingleImplementation(pointer_size); 398 if (single_impl == nullptr) { 399 // implementation_method becomes the first implementation for 400 // interface_method. 401 interface_method->SetSingleImplementation(implementation_method, pointer_size); 402 // Keep interface_method's single-implementation status. 403 return; 404 } 405 DCHECK(!single_impl->IsAbstract()); 406 if (single_impl->GetDeclaringClass() == implementation_method->GetDeclaringClass()) { 407 // Same implementation. Since implementation_method may be a copy of a default 408 // method, we need to check the declaring class for equality. 409 return; 410 } 411 // Another implementation for interface_method. 412 invalidated_single_impl_methods.insert(interface_method); 413 } 414 415 void ClassHierarchyAnalysis::InitSingleImplementationFlag(Handle<mirror::Class> klass, 416 ArtMethod* method, 417 PointerSize pointer_size) { 418 DCHECK(method->IsCopied() || method->GetDeclaringClass() == klass.Get()); 419 if (klass->IsFinal() || method->IsFinal()) { 420 // Final classes or methods do not need CHA for devirtualization. 421 // This frees up modifier bits for intrinsics which currently are only 422 // used for static methods or methods of final classes. 423 return; 424 } 425 if (method->IsAbstract()) { 426 // single-implementation of abstract method shares the same field 427 // that's used for JNI function of native method. It's fine since a method 428 // cannot be both abstract and native. 429 DCHECK(!method->IsNative()) << "Abstract method cannot be native"; 430 431 if (method->GetDeclaringClass()->IsInstantiable()) { 432 // Rare case, but we do accept it (such as 800-smali/smali/b_26143249.smali). 433 // Do not attempt to devirtualize it. 434 method->SetHasSingleImplementation(false); 435 DCHECK(method->GetSingleImplementation(pointer_size) == nullptr); 436 } else { 437 // Abstract method starts with single-implementation flag set and null 438 // implementation method. 439 method->SetHasSingleImplementation(true); 440 DCHECK(method->GetSingleImplementation(pointer_size) == nullptr); 441 } 442 } else { 443 method->SetHasSingleImplementation(true); 444 // Single implementation of non-abstract method is itself. 445 DCHECK_EQ(method->GetSingleImplementation(pointer_size), method); 446 } 447 } 448 449 void ClassHierarchyAnalysis::UpdateAfterLoadingOf(Handle<mirror::Class> klass) { 450 PointerSize image_pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize(); 451 if (klass->IsInterface()) { 452 for (ArtMethod& method : klass->GetDeclaredVirtualMethods(image_pointer_size)) { 453 DCHECK(method.IsAbstract() || method.IsDefault()); 454 InitSingleImplementationFlag(klass, &method, image_pointer_size); 455 } 456 return; 457 } 458 459 mirror::Class* super_class = klass->GetSuperClass(); 460 if (super_class == nullptr) { 461 return; 462 } 463 464 // Keeps track of all methods whose single-implementation assumption 465 // is invalidated by linking `klass`. 466 std::unordered_set<ArtMethod*> invalidated_single_impl_methods; 467 468 // Do an entry-by-entry comparison of vtable contents with super's vtable. 469 for (int32_t i = 0; i < super_class->GetVTableLength(); ++i) { 470 ArtMethod* method = klass->GetVTableEntry(i, image_pointer_size); 471 ArtMethod* method_in_super = super_class->GetVTableEntry(i, image_pointer_size); 472 if (method == method_in_super) { 473 // vtable slot entry is inherited from super class. 474 if (method->IsAbstract() && klass->IsInstantiable()) { 475 // An instantiable class that inherits an abstract method is treated as 476 // supplying an implementation that throws AbstractMethodError. 477 CheckVirtualMethodSingleImplementationInfo(klass, 478 method, 479 method_in_super, 480 invalidated_single_impl_methods, 481 image_pointer_size); 482 } 483 continue; 484 } 485 InitSingleImplementationFlag(klass, method, image_pointer_size); 486 CheckVirtualMethodSingleImplementationInfo(klass, 487 method, 488 method_in_super, 489 invalidated_single_impl_methods, 490 image_pointer_size); 491 } 492 // For new virtual methods that don't override. 493 for (int32_t i = super_class->GetVTableLength(); i < klass->GetVTableLength(); ++i) { 494 ArtMethod* method = klass->GetVTableEntry(i, image_pointer_size); 495 InitSingleImplementationFlag(klass, method, image_pointer_size); 496 } 497 498 if (klass->IsInstantiable()) { 499 auto* iftable = klass->GetIfTable(); 500 const size_t ifcount = klass->GetIfTableCount(); 501 for (size_t i = 0; i < ifcount; ++i) { 502 mirror::Class* interface = iftable->GetInterface(i); 503 for (size_t j = 0, count = iftable->GetMethodArrayCount(i); j < count; ++j) { 504 ArtMethod* interface_method = interface->GetVirtualMethod(j, image_pointer_size); 505 mirror::PointerArray* method_array = iftable->GetMethodArray(i); 506 ArtMethod* implementation_method = 507 method_array->GetElementPtrSize<ArtMethod*>(j, image_pointer_size); 508 DCHECK(implementation_method != nullptr) << klass->PrettyClass(); 509 CheckInterfaceMethodSingleImplementationInfo(klass, 510 interface_method, 511 implementation_method, 512 invalidated_single_impl_methods, 513 image_pointer_size); 514 } 515 } 516 } 517 518 InvalidateSingleImplementationMethods(invalidated_single_impl_methods); 519 } 520 521 void ClassHierarchyAnalysis::InvalidateSingleImplementationMethods( 522 std::unordered_set<ArtMethod*>& invalidated_single_impl_methods) { 523 if (!invalidated_single_impl_methods.empty()) { 524 Runtime* const runtime = Runtime::Current(); 525 Thread *self = Thread::Current(); 526 // Method headers for compiled code to be invalidated. 527 std::unordered_set<OatQuickMethodHeader*> dependent_method_headers; 528 PointerSize image_pointer_size = 529 Runtime::Current()->GetClassLinker()->GetImagePointerSize(); 530 531 { 532 // We do this under cha_lock_. Committing code also grabs this lock to 533 // make sure the code is only committed when all single-implementation 534 // assumptions are still true. 535 MutexLock cha_mu(self, *Locks::cha_lock_); 536 // Invalidate compiled methods that assume some virtual calls have only 537 // single implementations. 538 for (ArtMethod* invalidated : invalidated_single_impl_methods) { 539 if (!invalidated->HasSingleImplementation()) { 540 // It might have been invalidated already when other class linking is 541 // going on. 542 continue; 543 } 544 invalidated->SetHasSingleImplementation(false); 545 if (invalidated->IsAbstract()) { 546 // Clear the single implementation method. 547 invalidated->SetSingleImplementation(nullptr, image_pointer_size); 548 } 549 550 if (runtime->IsAotCompiler()) { 551 // No need to invalidate any compiled code as the AotCompiler doesn't 552 // run any code. 553 continue; 554 } 555 556 // Invalidate all dependents. 557 auto dependents = GetDependents(invalidated); 558 if (dependents == nullptr) { 559 continue; 560 } 561 for (const auto& dependent : *dependents) { 562 ArtMethod* method = dependent.first;; 563 OatQuickMethodHeader* method_header = dependent.second; 564 VLOG(class_linker) << "CHA invalidated compiled code for " << method->PrettyMethod(); 565 DCHECK(runtime->UseJitCompilation()); 566 runtime->GetJit()->GetCodeCache()->InvalidateCompiledCodeFor( 567 method, method_header); 568 dependent_method_headers.insert(method_header); 569 } 570 RemoveDependencyFor(invalidated); 571 } 572 } 573 574 if (dependent_method_headers.empty()) { 575 return; 576 } 577 // Deoptimze compiled code on stack that should have been invalidated. 578 CHACheckpoint checkpoint(dependent_method_headers); 579 size_t threads_running_checkpoint = runtime->GetThreadList()->RunCheckpoint(&checkpoint); 580 if (threads_running_checkpoint != 0) { 581 checkpoint.WaitForThreadsToRunThroughCheckpoint(threads_running_checkpoint); 582 } 583 } 584 } 585 586 } // namespace art 587