1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "inliner.h" 18 19 #include "art_method-inl.h" 20 #include "base/enums.h" 21 #include "builder.h" 22 #include "class_linker.h" 23 #include "constant_folding.h" 24 #include "dead_code_elimination.h" 25 #include "dex/inline_method_analyser.h" 26 #include "dex/verified_method.h" 27 #include "dex/verification_results.h" 28 #include "driver/compiler_driver-inl.h" 29 #include "driver/compiler_options.h" 30 #include "driver/dex_compilation_unit.h" 31 #include "instruction_simplifier.h" 32 #include "intrinsics.h" 33 #include "jit/jit.h" 34 #include "jit/jit_code_cache.h" 35 #include "mirror/class_loader.h" 36 #include "mirror/dex_cache.h" 37 #include "nodes.h" 38 #include "optimizing_compiler.h" 39 #include "reference_type_propagation.h" 40 #include "register_allocator_linear_scan.h" 41 #include "sharpening.h" 42 #include "ssa_builder.h" 43 #include "ssa_phi_elimination.h" 44 #include "scoped_thread_state_change-inl.h" 45 #include "thread.h" 46 47 namespace art { 48 49 // Instruction limit to control memory. 50 static constexpr size_t kMaximumNumberOfTotalInstructions = 1024; 51 52 // Maximum number of instructions for considering a method small, 53 // which we will always try to inline if the other non-instruction limits 54 // are not reached. 55 static constexpr size_t kMaximumNumberOfInstructionsForSmallMethod = 3; 56 57 // Limit the number of dex registers that we accumulate while inlining 58 // to avoid creating large amount of nested environments. 59 static constexpr size_t kMaximumNumberOfCumulatedDexRegisters = 32; 60 61 // Limit recursive call inlining, which do not benefit from too 62 // much inlining compared to code locality. 63 static constexpr size_t kMaximumNumberOfRecursiveCalls = 4; 64 65 // Controls the use of inline caches in AOT mode. 66 static constexpr bool kUseAOTInlineCaches = true; 67 68 // We check for line numbers to make sure the DepthString implementation 69 // aligns the output nicely. 70 #define LOG_INTERNAL(msg) \ 71 static_assert(__LINE__ > 10, "Unhandled line number"); \ 72 static_assert(__LINE__ < 10000, "Unhandled line number"); \ 73 VLOG(compiler) << DepthString(__LINE__) << msg 74 75 #define LOG_TRY() LOG_INTERNAL("Try inlinining call: ") 76 #define LOG_NOTE() LOG_INTERNAL("Note: ") 77 #define LOG_SUCCESS() LOG_INTERNAL("Success: ") 78 #define LOG_FAIL(stat) MaybeRecordStat(stat); LOG_INTERNAL("Fail: ") 79 #define LOG_FAIL_NO_STAT() LOG_INTERNAL("Fail: ") 80 81 std::string HInliner::DepthString(int line) const { 82 std::string value; 83 // Indent according to the inlining depth. 84 size_t count = depth_; 85 // Line numbers get printed in the log, so add a space if the log's line is less 86 // than 1000, and two if less than 100. 10 cannot be reached as it's the copyright. 87 if (!kIsTargetBuild) { 88 if (line < 100) { 89 value += " "; 90 } 91 if (line < 1000) { 92 value += " "; 93 } 94 // Safeguard if this file reaches more than 10000 lines. 95 DCHECK_LT(line, 10000); 96 } 97 for (size_t i = 0; i < count; ++i) { 98 value += " "; 99 } 100 return value; 101 } 102 103 static size_t CountNumberOfInstructions(HGraph* graph) { 104 size_t number_of_instructions = 0; 105 for (HBasicBlock* block : graph->GetReversePostOrderSkipEntryBlock()) { 106 for (HInstructionIterator instr_it(block->GetInstructions()); 107 !instr_it.Done(); 108 instr_it.Advance()) { 109 ++number_of_instructions; 110 } 111 } 112 return number_of_instructions; 113 } 114 115 void HInliner::UpdateInliningBudget() { 116 if (total_number_of_instructions_ >= kMaximumNumberOfTotalInstructions) { 117 // Always try to inline small methods. 118 inlining_budget_ = kMaximumNumberOfInstructionsForSmallMethod; 119 } else { 120 inlining_budget_ = std::max( 121 kMaximumNumberOfInstructionsForSmallMethod, 122 kMaximumNumberOfTotalInstructions - total_number_of_instructions_); 123 } 124 } 125 126 void HInliner::Run() { 127 if (graph_->IsDebuggable()) { 128 // For simplicity, we currently never inline when the graph is debuggable. This avoids 129 // doing some logic in the runtime to discover if a method could have been inlined. 130 return; 131 } 132 133 // Initialize the number of instructions for the method being compiled. Recursive calls 134 // to HInliner::Run have already updated the instruction count. 135 if (outermost_graph_ == graph_) { 136 total_number_of_instructions_ = CountNumberOfInstructions(graph_); 137 } 138 139 UpdateInliningBudget(); 140 DCHECK_NE(total_number_of_instructions_, 0u); 141 DCHECK_NE(inlining_budget_, 0u); 142 143 // If we're compiling with a core image (which is only used for 144 // test purposes), honor inlining directives in method names: 145 // - if a method's name contains the substring "$inline$", ensure 146 // that this method is actually inlined; 147 // - if a method's name contains the substring "$noinline$", do not 148 // inline that method. 149 // We limit this to AOT compilation, as the JIT may or may not inline 150 // depending on the state of classes at runtime. 151 const bool honor_inlining_directives = 152 IsCompilingWithCoreImage() && Runtime::Current()->IsAotCompiler(); 153 154 // Keep a copy of all blocks when starting the visit. 155 ArenaVector<HBasicBlock*> blocks = graph_->GetReversePostOrder(); 156 DCHECK(!blocks.empty()); 157 // Because we are changing the graph when inlining, 158 // we just iterate over the blocks of the outer method. 159 // This avoids doing the inlining work again on the inlined blocks. 160 for (HBasicBlock* block : blocks) { 161 for (HInstruction* instruction = block->GetFirstInstruction(); instruction != nullptr;) { 162 HInstruction* next = instruction->GetNext(); 163 HInvoke* call = instruction->AsInvoke(); 164 // As long as the call is not intrinsified, it is worth trying to inline. 165 if (call != nullptr && call->GetIntrinsic() == Intrinsics::kNone) { 166 if (honor_inlining_directives) { 167 // Debugging case: directives in method names control or assert on inlining. 168 std::string callee_name = outer_compilation_unit_.GetDexFile()->PrettyMethod( 169 call->GetDexMethodIndex(), /* with_signature */ false); 170 // Tests prevent inlining by having $noinline$ in their method names. 171 if (callee_name.find("$noinline$") == std::string::npos) { 172 if (!TryInline(call)) { 173 bool should_have_inlined = (callee_name.find("$inline$") != std::string::npos); 174 CHECK(!should_have_inlined) << "Could not inline " << callee_name; 175 } 176 } 177 } else { 178 // Normal case: try to inline. 179 TryInline(call); 180 } 181 } 182 instruction = next; 183 } 184 } 185 } 186 187 static bool IsMethodOrDeclaringClassFinal(ArtMethod* method) 188 REQUIRES_SHARED(Locks::mutator_lock_) { 189 return method->IsFinal() || method->GetDeclaringClass()->IsFinal(); 190 } 191 192 /** 193 * Given the `resolved_method` looked up in the dex cache, try to find 194 * the actual runtime target of an interface or virtual call. 195 * Return nullptr if the runtime target cannot be proven. 196 */ 197 static ArtMethod* FindVirtualOrInterfaceTarget(HInvoke* invoke, ArtMethod* resolved_method) 198 REQUIRES_SHARED(Locks::mutator_lock_) { 199 if (IsMethodOrDeclaringClassFinal(resolved_method)) { 200 // No need to lookup further, the resolved method will be the target. 201 return resolved_method; 202 } 203 204 HInstruction* receiver = invoke->InputAt(0); 205 if (receiver->IsNullCheck()) { 206 // Due to multiple levels of inlining within the same pass, it might be that 207 // null check does not have the reference type of the actual receiver. 208 receiver = receiver->InputAt(0); 209 } 210 ReferenceTypeInfo info = receiver->GetReferenceTypeInfo(); 211 DCHECK(info.IsValid()) << "Invalid RTI for " << receiver->DebugName(); 212 if (!info.IsExact()) { 213 // We currently only support inlining with known receivers. 214 // TODO: Remove this check, we should be able to inline final methods 215 // on unknown receivers. 216 return nullptr; 217 } else if (info.GetTypeHandle()->IsInterface()) { 218 // Statically knowing that the receiver has an interface type cannot 219 // help us find what is the target method. 220 return nullptr; 221 } else if (!resolved_method->GetDeclaringClass()->IsAssignableFrom(info.GetTypeHandle().Get())) { 222 // The method that we're trying to call is not in the receiver's class or super classes. 223 return nullptr; 224 } else if (info.GetTypeHandle()->IsErroneous()) { 225 // If the type is erroneous, do not go further, as we are going to query the vtable or 226 // imt table, that we can only safely do on non-erroneous classes. 227 return nullptr; 228 } 229 230 ClassLinker* cl = Runtime::Current()->GetClassLinker(); 231 PointerSize pointer_size = cl->GetImagePointerSize(); 232 if (invoke->IsInvokeInterface()) { 233 resolved_method = info.GetTypeHandle()->FindVirtualMethodForInterface( 234 resolved_method, pointer_size); 235 } else { 236 DCHECK(invoke->IsInvokeVirtual()); 237 resolved_method = info.GetTypeHandle()->FindVirtualMethodForVirtual( 238 resolved_method, pointer_size); 239 } 240 241 if (resolved_method == nullptr) { 242 // The information we had on the receiver was not enough to find 243 // the target method. Since we check above the exact type of the receiver, 244 // the only reason this can happen is an IncompatibleClassChangeError. 245 return nullptr; 246 } else if (!resolved_method->IsInvokable()) { 247 // The information we had on the receiver was not enough to find 248 // the target method. Since we check above the exact type of the receiver, 249 // the only reason this can happen is an IncompatibleClassChangeError. 250 return nullptr; 251 } else if (IsMethodOrDeclaringClassFinal(resolved_method)) { 252 // A final method has to be the target method. 253 return resolved_method; 254 } else if (info.IsExact()) { 255 // If we found a method and the receiver's concrete type is statically 256 // known, we know for sure the target. 257 return resolved_method; 258 } else { 259 // Even if we did find a method, the receiver type was not enough to 260 // statically find the runtime target. 261 return nullptr; 262 } 263 } 264 265 static uint32_t FindMethodIndexIn(ArtMethod* method, 266 const DexFile& dex_file, 267 uint32_t name_and_signature_index) 268 REQUIRES_SHARED(Locks::mutator_lock_) { 269 if (IsSameDexFile(*method->GetDexFile(), dex_file)) { 270 return method->GetDexMethodIndex(); 271 } else { 272 return method->FindDexMethodIndexInOtherDexFile(dex_file, name_and_signature_index); 273 } 274 } 275 276 static dex::TypeIndex FindClassIndexIn(mirror::Class* cls, 277 const DexCompilationUnit& compilation_unit) 278 REQUIRES_SHARED(Locks::mutator_lock_) { 279 const DexFile& dex_file = *compilation_unit.GetDexFile(); 280 dex::TypeIndex index; 281 if (cls->GetDexCache() == nullptr) { 282 DCHECK(cls->IsArrayClass()) << cls->PrettyClass(); 283 index = cls->FindTypeIndexInOtherDexFile(dex_file); 284 } else if (!cls->GetDexTypeIndex().IsValid()) { 285 DCHECK(cls->IsProxyClass()) << cls->PrettyClass(); 286 // TODO: deal with proxy classes. 287 } else if (IsSameDexFile(cls->GetDexFile(), dex_file)) { 288 DCHECK_EQ(cls->GetDexCache(), compilation_unit.GetDexCache().Get()); 289 index = cls->GetDexTypeIndex(); 290 } else { 291 index = cls->FindTypeIndexInOtherDexFile(dex_file); 292 // We cannot guarantee the entry will resolve to the same class, 293 // as there may be different class loaders. So only return the index if it's 294 // the right class already resolved with the class loader. 295 if (index.IsValid()) { 296 ObjPtr<mirror::Class> resolved = ClassLinker::LookupResolvedType( 297 index, compilation_unit.GetDexCache().Get(), compilation_unit.GetClassLoader().Get()); 298 if (resolved != cls) { 299 index = dex::TypeIndex::Invalid(); 300 } 301 } 302 } 303 304 return index; 305 } 306 307 class ScopedProfilingInfoInlineUse { 308 public: 309 explicit ScopedProfilingInfoInlineUse(ArtMethod* method, Thread* self) 310 : method_(method), 311 self_(self), 312 // Fetch the profiling info ahead of using it. If it's null when fetching, 313 // we should not call JitCodeCache::DoneInlining. 314 profiling_info_( 315 Runtime::Current()->GetJit()->GetCodeCache()->NotifyCompilerUse(method, self)) { 316 } 317 318 ~ScopedProfilingInfoInlineUse() { 319 if (profiling_info_ != nullptr) { 320 PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize(); 321 DCHECK_EQ(profiling_info_, method_->GetProfilingInfo(pointer_size)); 322 Runtime::Current()->GetJit()->GetCodeCache()->DoneCompilerUse(method_, self_); 323 } 324 } 325 326 ProfilingInfo* GetProfilingInfo() const { return profiling_info_; } 327 328 private: 329 ArtMethod* const method_; 330 Thread* const self_; 331 ProfilingInfo* const profiling_info_; 332 }; 333 334 HInliner::InlineCacheType HInliner::GetInlineCacheType( 335 const Handle<mirror::ObjectArray<mirror::Class>>& classes) 336 REQUIRES_SHARED(Locks::mutator_lock_) { 337 uint8_t number_of_types = 0; 338 for (; number_of_types < InlineCache::kIndividualCacheSize; ++number_of_types) { 339 if (classes->Get(number_of_types) == nullptr) { 340 break; 341 } 342 } 343 344 if (number_of_types == 0) { 345 return kInlineCacheUninitialized; 346 } else if (number_of_types == 1) { 347 return kInlineCacheMonomorphic; 348 } else if (number_of_types == InlineCache::kIndividualCacheSize) { 349 return kInlineCacheMegamorphic; 350 } else { 351 return kInlineCachePolymorphic; 352 } 353 } 354 355 static mirror::Class* GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Class>> classes) 356 REQUIRES_SHARED(Locks::mutator_lock_) { 357 DCHECK(classes->Get(0) != nullptr); 358 return classes->Get(0); 359 } 360 361 ArtMethod* HInliner::TryCHADevirtualization(ArtMethod* resolved_method) { 362 if (!resolved_method->HasSingleImplementation()) { 363 return nullptr; 364 } 365 if (Runtime::Current()->IsAotCompiler()) { 366 // No CHA-based devirtulization for AOT compiler (yet). 367 return nullptr; 368 } 369 if (outermost_graph_->IsCompilingOsr()) { 370 // We do not support HDeoptimize in OSR methods. 371 return nullptr; 372 } 373 PointerSize pointer_size = caller_compilation_unit_.GetClassLinker()->GetImagePointerSize(); 374 ArtMethod* single_impl = resolved_method->GetSingleImplementation(pointer_size); 375 if (single_impl == nullptr) { 376 return nullptr; 377 } 378 if (single_impl->IsProxyMethod()) { 379 // Proxy method is a generic invoker that's not worth 380 // devirtualizing/inlining. It also causes issues when the proxy 381 // method is in another dex file if we try to rewrite invoke-interface to 382 // invoke-virtual because a proxy method doesn't have a real dex file. 383 return nullptr; 384 } 385 if (!single_impl->GetDeclaringClass()->IsResolved()) { 386 // There's a race with the class loading, which updates the CHA info 387 // before setting the class to resolved. So we just bail for this 388 // rare occurence. 389 return nullptr; 390 } 391 return single_impl; 392 } 393 394 bool HInliner::TryInline(HInvoke* invoke_instruction) { 395 if (invoke_instruction->IsInvokeUnresolved() || 396 invoke_instruction->IsInvokePolymorphic()) { 397 return false; // Don't bother to move further if we know the method is unresolved or an 398 // invoke-polymorphic. 399 } 400 401 ScopedObjectAccess soa(Thread::Current()); 402 uint32_t method_index = invoke_instruction->GetDexMethodIndex(); 403 const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); 404 LOG_TRY() << caller_dex_file.PrettyMethod(method_index); 405 406 ArtMethod* resolved_method = invoke_instruction->GetResolvedMethod(); 407 if (resolved_method == nullptr) { 408 DCHECK(invoke_instruction->IsInvokeStaticOrDirect()); 409 DCHECK(invoke_instruction->AsInvokeStaticOrDirect()->IsStringInit()); 410 LOG_FAIL_NO_STAT() << "Not inlining a String.<init> method"; 411 return false; 412 } 413 ArtMethod* actual_method = nullptr; 414 415 if (invoke_instruction->IsInvokeStaticOrDirect()) { 416 actual_method = resolved_method; 417 } else { 418 // Check if we can statically find the method. 419 actual_method = FindVirtualOrInterfaceTarget(invoke_instruction, resolved_method); 420 } 421 422 bool cha_devirtualize = false; 423 if (actual_method == nullptr) { 424 ArtMethod* method = TryCHADevirtualization(resolved_method); 425 if (method != nullptr) { 426 cha_devirtualize = true; 427 actual_method = method; 428 LOG_NOTE() << "Try CHA-based inlining of " << actual_method->PrettyMethod(); 429 } 430 } 431 432 if (actual_method != nullptr) { 433 bool result = TryInlineAndReplace(invoke_instruction, 434 actual_method, 435 ReferenceTypeInfo::CreateInvalid(), 436 /* do_rtp */ true, 437 cha_devirtualize); 438 if (result && !invoke_instruction->IsInvokeStaticOrDirect()) { 439 if (cha_devirtualize) { 440 // Add dependency due to devirtulization. We've assumed resolved_method 441 // has single implementation. 442 outermost_graph_->AddCHASingleImplementationDependency(resolved_method); 443 MaybeRecordStat(kCHAInline); 444 } else { 445 MaybeRecordStat(kInlinedInvokeVirtualOrInterface); 446 } 447 } 448 return result; 449 } 450 DCHECK(!invoke_instruction->IsInvokeStaticOrDirect()); 451 452 // Try using inline caches. 453 return TryInlineFromInlineCache(caller_dex_file, invoke_instruction, resolved_method); 454 } 455 456 static Handle<mirror::ObjectArray<mirror::Class>> AllocateInlineCacheHolder( 457 const DexCompilationUnit& compilation_unit, 458 StackHandleScope<1>* hs) 459 REQUIRES_SHARED(Locks::mutator_lock_) { 460 Thread* self = Thread::Current(); 461 ClassLinker* class_linker = compilation_unit.GetClassLinker(); 462 Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs->NewHandle( 463 mirror::ObjectArray<mirror::Class>::Alloc( 464 self, 465 class_linker->GetClassRoot(ClassLinker::kClassArrayClass), 466 InlineCache::kIndividualCacheSize)); 467 if (inline_cache == nullptr) { 468 // We got an OOME. Just clear the exception, and don't inline. 469 DCHECK(self->IsExceptionPending()); 470 self->ClearException(); 471 VLOG(compiler) << "Out of memory in the compiler when trying to inline"; 472 } 473 return inline_cache; 474 } 475 476 bool HInliner::UseOnlyPolymorphicInliningWithNoDeopt() { 477 // If we are compiling AOT or OSR, pretend the call using inline caches is polymorphic and 478 // do not generate a deopt. 479 // 480 // For AOT: 481 // Generating a deopt does not ensure that we will actually capture the new types; 482 // and the danger is that we could be stuck in a loop with "forever" deoptimizations. 483 // Take for example the following scenario: 484 // - we capture the inline cache in one run 485 // - the next run, we deoptimize because we miss a type check, but the method 486 // never becomes hot again 487 // In this case, the inline cache will not be updated in the profile and the AOT code 488 // will keep deoptimizing. 489 // Another scenario is if we use profile compilation for a process which is not allowed 490 // to JIT (e.g. system server). If we deoptimize we will run interpreted code for the 491 // rest of the lifetime. 492 // TODO(calin): 493 // This is a compromise because we will most likely never update the inline cache 494 // in the profile (unless there's another reason to deopt). So we might be stuck with 495 // a sub-optimal inline cache. 496 // We could be smarter when capturing inline caches to mitigate this. 497 // (e.g. by having different thresholds for new and old methods). 498 // 499 // For OSR: 500 // We may come from the interpreter and it may have seen different receiver types. 501 return Runtime::Current()->IsAotCompiler() || outermost_graph_->IsCompilingOsr(); 502 } 503 bool HInliner::TryInlineFromInlineCache(const DexFile& caller_dex_file, 504 HInvoke* invoke_instruction, 505 ArtMethod* resolved_method) 506 REQUIRES_SHARED(Locks::mutator_lock_) { 507 if (Runtime::Current()->IsAotCompiler() && !kUseAOTInlineCaches) { 508 return false; 509 } 510 511 StackHandleScope<1> hs(Thread::Current()); 512 Handle<mirror::ObjectArray<mirror::Class>> inline_cache; 513 InlineCacheType inline_cache_type = Runtime::Current()->IsAotCompiler() 514 ? GetInlineCacheAOT(caller_dex_file, invoke_instruction, &hs, &inline_cache) 515 : GetInlineCacheJIT(invoke_instruction, &hs, &inline_cache); 516 517 switch (inline_cache_type) { 518 case kInlineCacheNoData: { 519 LOG_FAIL_NO_STAT() 520 << "Interface or virtual call to " 521 << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) 522 << " could not be statically determined"; 523 return false; 524 } 525 526 case kInlineCacheUninitialized: { 527 LOG_FAIL_NO_STAT() 528 << "Interface or virtual call to " 529 << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) 530 << " is not hit and not inlined"; 531 return false; 532 } 533 534 case kInlineCacheMonomorphic: { 535 MaybeRecordStat(kMonomorphicCall); 536 if (UseOnlyPolymorphicInliningWithNoDeopt()) { 537 return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache); 538 } else { 539 return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache); 540 } 541 } 542 543 case kInlineCachePolymorphic: { 544 MaybeRecordStat(kPolymorphicCall); 545 return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache); 546 } 547 548 case kInlineCacheMegamorphic: { 549 LOG_FAIL_NO_STAT() 550 << "Interface or virtual call to " 551 << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) 552 << " is megamorphic and not inlined"; 553 MaybeRecordStat(kMegamorphicCall); 554 return false; 555 } 556 557 case kInlineCacheMissingTypes: { 558 LOG_FAIL_NO_STAT() 559 << "Interface or virtual call to " 560 << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) 561 << " is missing types and not inlined"; 562 return false; 563 } 564 } 565 UNREACHABLE(); 566 } 567 568 HInliner::InlineCacheType HInliner::GetInlineCacheJIT( 569 HInvoke* invoke_instruction, 570 StackHandleScope<1>* hs, 571 /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache) 572 REQUIRES_SHARED(Locks::mutator_lock_) { 573 DCHECK(Runtime::Current()->UseJitCompilation()); 574 575 ArtMethod* caller = graph_->GetArtMethod(); 576 // Under JIT, we should always know the caller. 577 DCHECK(caller != nullptr); 578 ScopedProfilingInfoInlineUse spiis(caller, Thread::Current()); 579 ProfilingInfo* profiling_info = spiis.GetProfilingInfo(); 580 581 if (profiling_info == nullptr) { 582 return kInlineCacheNoData; 583 } 584 585 *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs); 586 if (inline_cache->Get() == nullptr) { 587 // We can't extract any data if we failed to allocate; 588 return kInlineCacheNoData; 589 } else { 590 Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto( 591 *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()), 592 *inline_cache); 593 return GetInlineCacheType(*inline_cache); 594 } 595 } 596 597 HInliner::InlineCacheType HInliner::GetInlineCacheAOT( 598 const DexFile& caller_dex_file, 599 HInvoke* invoke_instruction, 600 StackHandleScope<1>* hs, 601 /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache) 602 REQUIRES_SHARED(Locks::mutator_lock_) { 603 DCHECK(Runtime::Current()->IsAotCompiler()); 604 const ProfileCompilationInfo* pci = compiler_driver_->GetProfileCompilationInfo(); 605 if (pci == nullptr) { 606 return kInlineCacheNoData; 607 } 608 609 std::unique_ptr<ProfileCompilationInfo::OfflineProfileMethodInfo> offline_profile = 610 pci->GetMethod(caller_dex_file.GetLocation(), 611 caller_dex_file.GetLocationChecksum(), 612 caller_compilation_unit_.GetDexMethodIndex()); 613 if (offline_profile == nullptr) { 614 return kInlineCacheNoData; // no profile information for this invocation. 615 } 616 617 *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs); 618 if (inline_cache == nullptr) { 619 // We can't extract any data if we failed to allocate; 620 return kInlineCacheNoData; 621 } else { 622 return ExtractClassesFromOfflineProfile(invoke_instruction, 623 *(offline_profile.get()), 624 *inline_cache); 625 } 626 } 627 628 HInliner::InlineCacheType HInliner::ExtractClassesFromOfflineProfile( 629 const HInvoke* invoke_instruction, 630 const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile, 631 /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache) 632 REQUIRES_SHARED(Locks::mutator_lock_) { 633 const auto it = offline_profile.inline_caches->find(invoke_instruction->GetDexPc()); 634 if (it == offline_profile.inline_caches->end()) { 635 return kInlineCacheUninitialized; 636 } 637 638 const ProfileCompilationInfo::DexPcData& dex_pc_data = it->second; 639 640 if (dex_pc_data.is_missing_types) { 641 return kInlineCacheMissingTypes; 642 } 643 if (dex_pc_data.is_megamorphic) { 644 return kInlineCacheMegamorphic; 645 } 646 647 DCHECK_LE(dex_pc_data.classes.size(), InlineCache::kIndividualCacheSize); 648 Thread* self = Thread::Current(); 649 // We need to resolve the class relative to the containing dex file. 650 // So first, build a mapping from the index of dex file in the profile to 651 // its dex cache. This will avoid repeating the lookup when walking over 652 // the inline cache types. 653 std::vector<ObjPtr<mirror::DexCache>> dex_profile_index_to_dex_cache( 654 offline_profile.dex_references.size()); 655 for (size_t i = 0; i < offline_profile.dex_references.size(); i++) { 656 bool found = false; 657 for (const DexFile* dex_file : compiler_driver_->GetDexFilesForOatFile()) { 658 if (offline_profile.dex_references[i].MatchesDex(dex_file)) { 659 dex_profile_index_to_dex_cache[i] = 660 caller_compilation_unit_.GetClassLinker()->FindDexCache(self, *dex_file); 661 found = true; 662 } 663 } 664 if (!found) { 665 VLOG(compiler) << "Could not find profiled dex file: " 666 << offline_profile.dex_references[i].dex_location; 667 return kInlineCacheMissingTypes; 668 } 669 } 670 671 // Walk over the classes and resolve them. If we cannot find a type we return 672 // kInlineCacheMissingTypes. 673 int ic_index = 0; 674 for (const ProfileCompilationInfo::ClassReference& class_ref : dex_pc_data.classes) { 675 ObjPtr<mirror::DexCache> dex_cache = 676 dex_profile_index_to_dex_cache[class_ref.dex_profile_index]; 677 DCHECK(dex_cache != nullptr); 678 679 if (!dex_cache->GetDexFile()->IsTypeIndexValid(class_ref.type_index)) { 680 VLOG(compiler) << "Profile data corrupt: type index " << class_ref.type_index 681 << "is invalid in location" << dex_cache->GetDexFile()->GetLocation(); 682 return kInlineCacheNoData; 683 } 684 ObjPtr<mirror::Class> clazz = ClassLinker::LookupResolvedType( 685 class_ref.type_index, 686 dex_cache, 687 caller_compilation_unit_.GetClassLoader().Get()); 688 if (clazz != nullptr) { 689 inline_cache->Set(ic_index++, clazz); 690 } else { 691 VLOG(compiler) << "Could not resolve class from inline cache in AOT mode " 692 << caller_compilation_unit_.GetDexFile()->PrettyMethod( 693 invoke_instruction->GetDexMethodIndex()) << " : " 694 << caller_compilation_unit_ 695 .GetDexFile()->StringByTypeIdx(class_ref.type_index); 696 return kInlineCacheMissingTypes; 697 } 698 } 699 return GetInlineCacheType(inline_cache); 700 } 701 702 HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker, 703 HInstruction* receiver, 704 uint32_t dex_pc) const { 705 ArtField* field = class_linker->GetClassRoot(ClassLinker::kJavaLangObject)->GetInstanceField(0); 706 DCHECK_EQ(std::string(field->GetName()), "shadow$_klass_"); 707 HInstanceFieldGet* result = new (graph_->GetArena()) HInstanceFieldGet( 708 receiver, 709 field, 710 Primitive::kPrimNot, 711 field->GetOffset(), 712 field->IsVolatile(), 713 field->GetDexFieldIndex(), 714 field->GetDeclaringClass()->GetDexClassDefIndex(), 715 *field->GetDexFile(), 716 dex_pc); 717 // The class of a field is effectively final, and does not have any memory dependencies. 718 result->SetSideEffects(SideEffects::None()); 719 return result; 720 } 721 722 static ArtMethod* ResolveMethodFromInlineCache(Handle<mirror::Class> klass, 723 ArtMethod* resolved_method, 724 HInstruction* invoke_instruction, 725 PointerSize pointer_size) 726 REQUIRES_SHARED(Locks::mutator_lock_) { 727 if (Runtime::Current()->IsAotCompiler()) { 728 // We can get unrelated types when working with profiles (corruption, 729 // systme updates, or anyone can write to it). So first check if the class 730 // actually implements the declaring class of the method that is being 731 // called in bytecode. 732 // Note: the lookup methods used below require to have assignable types. 733 if (!resolved_method->GetDeclaringClass()->IsAssignableFrom(klass.Get())) { 734 return nullptr; 735 } 736 } 737 738 if (invoke_instruction->IsInvokeInterface()) { 739 resolved_method = klass->FindVirtualMethodForInterface(resolved_method, pointer_size); 740 } else { 741 DCHECK(invoke_instruction->IsInvokeVirtual()); 742 resolved_method = klass->FindVirtualMethodForVirtual(resolved_method, pointer_size); 743 } 744 DCHECK(resolved_method != nullptr); 745 return resolved_method; 746 } 747 748 bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, 749 ArtMethod* resolved_method, 750 Handle<mirror::ObjectArray<mirror::Class>> classes) { 751 DCHECK(invoke_instruction->IsInvokeVirtual() || invoke_instruction->IsInvokeInterface()) 752 << invoke_instruction->DebugName(); 753 754 dex::TypeIndex class_index = FindClassIndexIn( 755 GetMonomorphicType(classes), caller_compilation_unit_); 756 if (!class_index.IsValid()) { 757 LOG_FAIL(kNotInlinedDexCache) 758 << "Call to " << ArtMethod::PrettyMethod(resolved_method) 759 << " from inline cache is not inlined because its class is not" 760 << " accessible to the caller"; 761 return false; 762 } 763 764 ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); 765 PointerSize pointer_size = class_linker->GetImagePointerSize(); 766 Handle<mirror::Class> monomorphic_type = handles_->NewHandle(GetMonomorphicType(classes)); 767 resolved_method = ResolveMethodFromInlineCache( 768 monomorphic_type, resolved_method, invoke_instruction, pointer_size); 769 770 LOG_NOTE() << "Try inline monomorphic call to " << resolved_method->PrettyMethod(); 771 if (resolved_method == nullptr) { 772 // Bogus AOT profile, bail. 773 DCHECK(Runtime::Current()->IsAotCompiler()); 774 return false; 775 } 776 777 HInstruction* receiver = invoke_instruction->InputAt(0); 778 HInstruction* cursor = invoke_instruction->GetPrevious(); 779 HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); 780 if (!TryInlineAndReplace(invoke_instruction, 781 resolved_method, 782 ReferenceTypeInfo::Create(monomorphic_type, /* is_exact */ true), 783 /* do_rtp */ false, 784 /* cha_devirtualize */ false)) { 785 return false; 786 } 787 788 // We successfully inlined, now add a guard. 789 AddTypeGuard(receiver, 790 cursor, 791 bb_cursor, 792 class_index, 793 monomorphic_type, 794 invoke_instruction, 795 /* with_deoptimization */ true); 796 797 // Run type propagation to get the guard typed, and eventually propagate the 798 // type of the receiver. 799 ReferenceTypePropagation rtp_fixup(graph_, 800 outer_compilation_unit_.GetClassLoader(), 801 outer_compilation_unit_.GetDexCache(), 802 handles_, 803 /* is_first_run */ false); 804 rtp_fixup.Run(); 805 806 MaybeRecordStat(kInlinedMonomorphicCall); 807 return true; 808 } 809 810 void HInliner::AddCHAGuard(HInstruction* invoke_instruction, 811 uint32_t dex_pc, 812 HInstruction* cursor, 813 HBasicBlock* bb_cursor) { 814 HShouldDeoptimizeFlag* deopt_flag = new (graph_->GetArena()) 815 HShouldDeoptimizeFlag(graph_->GetArena(), dex_pc); 816 HInstruction* compare = new (graph_->GetArena()) HNotEqual( 817 deopt_flag, graph_->GetIntConstant(0, dex_pc)); 818 HInstruction* deopt = new (graph_->GetArena()) HDeoptimize( 819 graph_->GetArena(), compare, DeoptimizationKind::kCHA, dex_pc); 820 821 if (cursor != nullptr) { 822 bb_cursor->InsertInstructionAfter(deopt_flag, cursor); 823 } else { 824 bb_cursor->InsertInstructionBefore(deopt_flag, bb_cursor->GetFirstInstruction()); 825 } 826 bb_cursor->InsertInstructionAfter(compare, deopt_flag); 827 bb_cursor->InsertInstructionAfter(deopt, compare); 828 829 // Add receiver as input to aid CHA guard optimization later. 830 deopt_flag->AddInput(invoke_instruction->InputAt(0)); 831 DCHECK_EQ(deopt_flag->InputCount(), 1u); 832 deopt->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); 833 outermost_graph_->IncrementNumberOfCHAGuards(); 834 } 835 836 HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, 837 HInstruction* cursor, 838 HBasicBlock* bb_cursor, 839 dex::TypeIndex class_index, 840 Handle<mirror::Class> klass, 841 HInstruction* invoke_instruction, 842 bool with_deoptimization) { 843 ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); 844 HInstanceFieldGet* receiver_class = BuildGetReceiverClass( 845 class_linker, receiver, invoke_instruction->GetDexPc()); 846 if (cursor != nullptr) { 847 bb_cursor->InsertInstructionAfter(receiver_class, cursor); 848 } else { 849 bb_cursor->InsertInstructionBefore(receiver_class, bb_cursor->GetFirstInstruction()); 850 } 851 852 const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); 853 bool is_referrer; 854 ArtMethod* outermost_art_method = outermost_graph_->GetArtMethod(); 855 if (outermost_art_method == nullptr) { 856 DCHECK(Runtime::Current()->IsAotCompiler()); 857 // We are in AOT mode and we don't have an ART method to determine 858 // if the inlined method belongs to the referrer. Assume it doesn't. 859 is_referrer = false; 860 } else { 861 is_referrer = klass.Get() == outermost_art_method->GetDeclaringClass(); 862 } 863 864 // Note that we will just compare the classes, so we don't need Java semantics access checks. 865 // Note that the type index and the dex file are relative to the method this type guard is 866 // inlined into. 867 HLoadClass* load_class = new (graph_->GetArena()) HLoadClass(graph_->GetCurrentMethod(), 868 class_index, 869 caller_dex_file, 870 klass, 871 is_referrer, 872 invoke_instruction->GetDexPc(), 873 /* needs_access_check */ false); 874 HLoadClass::LoadKind kind = HSharpening::ComputeLoadClassKind( 875 load_class, codegen_, compiler_driver_, caller_compilation_unit_); 876 DCHECK(kind != HLoadClass::LoadKind::kInvalid) 877 << "We should always be able to reference a class for inline caches"; 878 // Insert before setting the kind, as setting the kind affects the inputs. 879 bb_cursor->InsertInstructionAfter(load_class, receiver_class); 880 load_class->SetLoadKind(kind); 881 // In AOT mode, we will most likely load the class from BSS, which will involve a call 882 // to the runtime. In this case, the load instruction will need an environment so copy 883 // it from the invoke instruction. 884 if (load_class->NeedsEnvironment()) { 885 DCHECK(Runtime::Current()->IsAotCompiler()); 886 load_class->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); 887 } 888 889 HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class); 890 bb_cursor->InsertInstructionAfter(compare, load_class); 891 if (with_deoptimization) { 892 HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( 893 graph_->GetArena(), 894 compare, 895 receiver, 896 Runtime::Current()->IsAotCompiler() 897 ? DeoptimizationKind::kAotInlineCache 898 : DeoptimizationKind::kJitInlineCache, 899 invoke_instruction->GetDexPc()); 900 bb_cursor->InsertInstructionAfter(deoptimize, compare); 901 deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); 902 DCHECK_EQ(invoke_instruction->InputAt(0), receiver); 903 receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize); 904 deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo()); 905 } 906 return compare; 907 } 908 909 bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, 910 ArtMethod* resolved_method, 911 Handle<mirror::ObjectArray<mirror::Class>> classes) { 912 DCHECK(invoke_instruction->IsInvokeVirtual() || invoke_instruction->IsInvokeInterface()) 913 << invoke_instruction->DebugName(); 914 915 if (TryInlinePolymorphicCallToSameTarget(invoke_instruction, resolved_method, classes)) { 916 return true; 917 } 918 919 ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); 920 PointerSize pointer_size = class_linker->GetImagePointerSize(); 921 922 bool all_targets_inlined = true; 923 bool one_target_inlined = false; 924 for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) { 925 if (classes->Get(i) == nullptr) { 926 break; 927 } 928 ArtMethod* method = nullptr; 929 930 Handle<mirror::Class> handle = handles_->NewHandle(classes->Get(i)); 931 method = ResolveMethodFromInlineCache( 932 handle, resolved_method, invoke_instruction, pointer_size); 933 if (method == nullptr) { 934 DCHECK(Runtime::Current()->IsAotCompiler()); 935 // AOT profile is bogus. This loop expects to iterate over all entries, 936 // so just just continue. 937 all_targets_inlined = false; 938 continue; 939 } 940 941 HInstruction* receiver = invoke_instruction->InputAt(0); 942 HInstruction* cursor = invoke_instruction->GetPrevious(); 943 HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); 944 945 dex::TypeIndex class_index = FindClassIndexIn(handle.Get(), caller_compilation_unit_); 946 HInstruction* return_replacement = nullptr; 947 LOG_NOTE() << "Try inline polymorphic call to " << method->PrettyMethod(); 948 if (!class_index.IsValid() || 949 !TryBuildAndInline(invoke_instruction, 950 method, 951 ReferenceTypeInfo::Create(handle, /* is_exact */ true), 952 &return_replacement)) { 953 all_targets_inlined = false; 954 } else { 955 one_target_inlined = true; 956 957 LOG_SUCCESS() << "Polymorphic call to " << ArtMethod::PrettyMethod(resolved_method) 958 << " has inlined " << ArtMethod::PrettyMethod(method); 959 960 // If we have inlined all targets before, and this receiver is the last seen, 961 // we deoptimize instead of keeping the original invoke instruction. 962 bool deoptimize = !UseOnlyPolymorphicInliningWithNoDeopt() && 963 all_targets_inlined && 964 (i != InlineCache::kIndividualCacheSize - 1) && 965 (classes->Get(i + 1) == nullptr); 966 967 HInstruction* compare = AddTypeGuard(receiver, 968 cursor, 969 bb_cursor, 970 class_index, 971 handle, 972 invoke_instruction, 973 deoptimize); 974 if (deoptimize) { 975 if (return_replacement != nullptr) { 976 invoke_instruction->ReplaceWith(return_replacement); 977 } 978 invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); 979 // Because the inline cache data can be populated concurrently, we force the end of the 980 // iteration. Otherwise, we could see a new receiver type. 981 break; 982 } else { 983 CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction); 984 } 985 } 986 } 987 988 if (!one_target_inlined) { 989 LOG_FAIL_NO_STAT() 990 << "Call to " << ArtMethod::PrettyMethod(resolved_method) 991 << " from inline cache is not inlined because none" 992 << " of its targets could be inlined"; 993 return false; 994 } 995 996 MaybeRecordStat(kInlinedPolymorphicCall); 997 998 // Run type propagation to get the guards typed. 999 ReferenceTypePropagation rtp_fixup(graph_, 1000 outer_compilation_unit_.GetClassLoader(), 1001 outer_compilation_unit_.GetDexCache(), 1002 handles_, 1003 /* is_first_run */ false); 1004 rtp_fixup.Run(); 1005 return true; 1006 } 1007 1008 void HInliner::CreateDiamondPatternForPolymorphicInline(HInstruction* compare, 1009 HInstruction* return_replacement, 1010 HInstruction* invoke_instruction) { 1011 uint32_t dex_pc = invoke_instruction->GetDexPc(); 1012 HBasicBlock* cursor_block = compare->GetBlock(); 1013 HBasicBlock* original_invoke_block = invoke_instruction->GetBlock(); 1014 ArenaAllocator* allocator = graph_->GetArena(); 1015 1016 // Spit the block after the compare: `cursor_block` will now be the start of the diamond, 1017 // and the returned block is the start of the then branch (that could contain multiple blocks). 1018 HBasicBlock* then = cursor_block->SplitAfterForInlining(compare); 1019 1020 // Split the block containing the invoke before and after the invoke. The returned block 1021 // of the split before will contain the invoke and will be the otherwise branch of 1022 // the diamond. The returned block of the split after will be the merge block 1023 // of the diamond. 1024 HBasicBlock* end_then = invoke_instruction->GetBlock(); 1025 HBasicBlock* otherwise = end_then->SplitBeforeForInlining(invoke_instruction); 1026 HBasicBlock* merge = otherwise->SplitAfterForInlining(invoke_instruction); 1027 1028 // If the methods we are inlining return a value, we create a phi in the merge block 1029 // that will have the `invoke_instruction and the `return_replacement` as inputs. 1030 if (return_replacement != nullptr) { 1031 HPhi* phi = new (allocator) HPhi( 1032 allocator, kNoRegNumber, 0, HPhi::ToPhiType(invoke_instruction->GetType()), dex_pc); 1033 merge->AddPhi(phi); 1034 invoke_instruction->ReplaceWith(phi); 1035 phi->AddInput(return_replacement); 1036 phi->AddInput(invoke_instruction); 1037 } 1038 1039 // Add the control flow instructions. 1040 otherwise->AddInstruction(new (allocator) HGoto(dex_pc)); 1041 end_then->AddInstruction(new (allocator) HGoto(dex_pc)); 1042 cursor_block->AddInstruction(new (allocator) HIf(compare, dex_pc)); 1043 1044 // Add the newly created blocks to the graph. 1045 graph_->AddBlock(then); 1046 graph_->AddBlock(otherwise); 1047 graph_->AddBlock(merge); 1048 1049 // Set up successor (and implictly predecessor) relations. 1050 cursor_block->AddSuccessor(otherwise); 1051 cursor_block->AddSuccessor(then); 1052 end_then->AddSuccessor(merge); 1053 otherwise->AddSuccessor(merge); 1054 1055 // Set up dominance information. 1056 then->SetDominator(cursor_block); 1057 cursor_block->AddDominatedBlock(then); 1058 otherwise->SetDominator(cursor_block); 1059 cursor_block->AddDominatedBlock(otherwise); 1060 merge->SetDominator(cursor_block); 1061 cursor_block->AddDominatedBlock(merge); 1062 1063 // Update the revert post order. 1064 size_t index = IndexOfElement(graph_->reverse_post_order_, cursor_block); 1065 MakeRoomFor(&graph_->reverse_post_order_, 1, index); 1066 graph_->reverse_post_order_[++index] = then; 1067 index = IndexOfElement(graph_->reverse_post_order_, end_then); 1068 MakeRoomFor(&graph_->reverse_post_order_, 2, index); 1069 graph_->reverse_post_order_[++index] = otherwise; 1070 graph_->reverse_post_order_[++index] = merge; 1071 1072 1073 graph_->UpdateLoopAndTryInformationOfNewBlock( 1074 then, original_invoke_block, /* replace_if_back_edge */ false); 1075 graph_->UpdateLoopAndTryInformationOfNewBlock( 1076 otherwise, original_invoke_block, /* replace_if_back_edge */ false); 1077 1078 // In case the original invoke location was a back edge, we need to update 1079 // the loop to now have the merge block as a back edge. 1080 graph_->UpdateLoopAndTryInformationOfNewBlock( 1081 merge, original_invoke_block, /* replace_if_back_edge */ true); 1082 } 1083 1084 bool HInliner::TryInlinePolymorphicCallToSameTarget( 1085 HInvoke* invoke_instruction, 1086 ArtMethod* resolved_method, 1087 Handle<mirror::ObjectArray<mirror::Class>> classes) { 1088 // This optimization only works under JIT for now. 1089 if (!Runtime::Current()->UseJitCompilation()) { 1090 return false; 1091 } 1092 1093 ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); 1094 PointerSize pointer_size = class_linker->GetImagePointerSize(); 1095 1096 DCHECK(resolved_method != nullptr); 1097 ArtMethod* actual_method = nullptr; 1098 size_t method_index = invoke_instruction->IsInvokeVirtual() 1099 ? invoke_instruction->AsInvokeVirtual()->GetVTableIndex() 1100 : invoke_instruction->AsInvokeInterface()->GetImtIndex(); 1101 1102 // Check whether we are actually calling the same method among 1103 // the different types seen. 1104 for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) { 1105 if (classes->Get(i) == nullptr) { 1106 break; 1107 } 1108 ArtMethod* new_method = nullptr; 1109 if (invoke_instruction->IsInvokeInterface()) { 1110 new_method = classes->Get(i)->GetImt(pointer_size)->Get( 1111 method_index, pointer_size); 1112 if (new_method->IsRuntimeMethod()) { 1113 // Bail out as soon as we see a conflict trampoline in one of the target's 1114 // interface table. 1115 return false; 1116 } 1117 } else { 1118 DCHECK(invoke_instruction->IsInvokeVirtual()); 1119 new_method = classes->Get(i)->GetEmbeddedVTableEntry(method_index, pointer_size); 1120 } 1121 DCHECK(new_method != nullptr); 1122 if (actual_method == nullptr) { 1123 actual_method = new_method; 1124 } else if (actual_method != new_method) { 1125 // Different methods, bailout. 1126 return false; 1127 } 1128 } 1129 1130 HInstruction* receiver = invoke_instruction->InputAt(0); 1131 HInstruction* cursor = invoke_instruction->GetPrevious(); 1132 HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); 1133 1134 HInstruction* return_replacement = nullptr; 1135 if (!TryBuildAndInline(invoke_instruction, 1136 actual_method, 1137 ReferenceTypeInfo::CreateInvalid(), 1138 &return_replacement)) { 1139 return false; 1140 } 1141 1142 // We successfully inlined, now add a guard. 1143 HInstanceFieldGet* receiver_class = BuildGetReceiverClass( 1144 class_linker, receiver, invoke_instruction->GetDexPc()); 1145 1146 Primitive::Type type = Is64BitInstructionSet(graph_->GetInstructionSet()) 1147 ? Primitive::kPrimLong 1148 : Primitive::kPrimInt; 1149 HClassTableGet* class_table_get = new (graph_->GetArena()) HClassTableGet( 1150 receiver_class, 1151 type, 1152 invoke_instruction->IsInvokeVirtual() ? HClassTableGet::TableKind::kVTable 1153 : HClassTableGet::TableKind::kIMTable, 1154 method_index, 1155 invoke_instruction->GetDexPc()); 1156 1157 HConstant* constant; 1158 if (type == Primitive::kPrimLong) { 1159 constant = graph_->GetLongConstant( 1160 reinterpret_cast<intptr_t>(actual_method), invoke_instruction->GetDexPc()); 1161 } else { 1162 constant = graph_->GetIntConstant( 1163 reinterpret_cast<intptr_t>(actual_method), invoke_instruction->GetDexPc()); 1164 } 1165 1166 HNotEqual* compare = new (graph_->GetArena()) HNotEqual(class_table_get, constant); 1167 if (cursor != nullptr) { 1168 bb_cursor->InsertInstructionAfter(receiver_class, cursor); 1169 } else { 1170 bb_cursor->InsertInstructionBefore(receiver_class, bb_cursor->GetFirstInstruction()); 1171 } 1172 bb_cursor->InsertInstructionAfter(class_table_get, receiver_class); 1173 bb_cursor->InsertInstructionAfter(compare, class_table_get); 1174 1175 if (outermost_graph_->IsCompilingOsr()) { 1176 CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction); 1177 } else { 1178 HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( 1179 graph_->GetArena(), 1180 compare, 1181 receiver, 1182 DeoptimizationKind::kJitSameTarget, 1183 invoke_instruction->GetDexPc()); 1184 bb_cursor->InsertInstructionAfter(deoptimize, compare); 1185 deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); 1186 if (return_replacement != nullptr) { 1187 invoke_instruction->ReplaceWith(return_replacement); 1188 } 1189 receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize); 1190 invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); 1191 deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo()); 1192 } 1193 1194 // Run type propagation to get the guard typed. 1195 ReferenceTypePropagation rtp_fixup(graph_, 1196 outer_compilation_unit_.GetClassLoader(), 1197 outer_compilation_unit_.GetDexCache(), 1198 handles_, 1199 /* is_first_run */ false); 1200 rtp_fixup.Run(); 1201 1202 MaybeRecordStat(kInlinedPolymorphicCall); 1203 1204 LOG_SUCCESS() << "Inlined same polymorphic target " << actual_method->PrettyMethod(); 1205 return true; 1206 } 1207 1208 bool HInliner::TryInlineAndReplace(HInvoke* invoke_instruction, 1209 ArtMethod* method, 1210 ReferenceTypeInfo receiver_type, 1211 bool do_rtp, 1212 bool cha_devirtualize) { 1213 HInstruction* return_replacement = nullptr; 1214 uint32_t dex_pc = invoke_instruction->GetDexPc(); 1215 HInstruction* cursor = invoke_instruction->GetPrevious(); 1216 HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); 1217 if (!TryBuildAndInline(invoke_instruction, method, receiver_type, &return_replacement)) { 1218 if (invoke_instruction->IsInvokeInterface()) { 1219 DCHECK(!method->IsProxyMethod()); 1220 // Turn an invoke-interface into an invoke-virtual. An invoke-virtual is always 1221 // better than an invoke-interface because: 1222 // 1) In the best case, the interface call has one more indirection (to fetch the IMT). 1223 // 2) We will not go to the conflict trampoline with an invoke-virtual. 1224 // TODO: Consider sharpening once it is not dependent on the compiler driver. 1225 1226 if (method->IsDefault() && !method->IsCopied()) { 1227 // Changing to invoke-virtual cannot be done on an original default method 1228 // since it's not in any vtable. Devirtualization by exact type/inline-cache 1229 // always uses a method in the iftable which is never an original default 1230 // method. 1231 // On the other hand, inlining an original default method by CHA is fine. 1232 DCHECK(cha_devirtualize); 1233 return false; 1234 } 1235 1236 const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); 1237 uint32_t dex_method_index = FindMethodIndexIn( 1238 method, caller_dex_file, invoke_instruction->GetDexMethodIndex()); 1239 if (dex_method_index == DexFile::kDexNoIndex) { 1240 return false; 1241 } 1242 HInvokeVirtual* new_invoke = new (graph_->GetArena()) HInvokeVirtual( 1243 graph_->GetArena(), 1244 invoke_instruction->GetNumberOfArguments(), 1245 invoke_instruction->GetType(), 1246 invoke_instruction->GetDexPc(), 1247 dex_method_index, 1248 method, 1249 method->GetMethodIndex()); 1250 HInputsRef inputs = invoke_instruction->GetInputs(); 1251 for (size_t index = 0; index != inputs.size(); ++index) { 1252 new_invoke->SetArgumentAt(index, inputs[index]); 1253 } 1254 invoke_instruction->GetBlock()->InsertInstructionBefore(new_invoke, invoke_instruction); 1255 new_invoke->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); 1256 if (invoke_instruction->GetType() == Primitive::kPrimNot) { 1257 new_invoke->SetReferenceTypeInfo(invoke_instruction->GetReferenceTypeInfo()); 1258 } 1259 return_replacement = new_invoke; 1260 } else { 1261 // TODO: Consider sharpening an invoke virtual once it is not dependent on the 1262 // compiler driver. 1263 return false; 1264 } 1265 } 1266 if (cha_devirtualize) { 1267 AddCHAGuard(invoke_instruction, dex_pc, cursor, bb_cursor); 1268 } 1269 if (return_replacement != nullptr) { 1270 invoke_instruction->ReplaceWith(return_replacement); 1271 } 1272 invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); 1273 FixUpReturnReferenceType(method, return_replacement); 1274 if (do_rtp && ReturnTypeMoreSpecific(invoke_instruction, return_replacement)) { 1275 // Actual return value has a more specific type than the method's declared 1276 // return type. Run RTP again on the outer graph to propagate it. 1277 ReferenceTypePropagation(graph_, 1278 outer_compilation_unit_.GetClassLoader(), 1279 outer_compilation_unit_.GetDexCache(), 1280 handles_, 1281 /* is_first_run */ false).Run(); 1282 } 1283 return true; 1284 } 1285 1286 size_t HInliner::CountRecursiveCallsOf(ArtMethod* method) const { 1287 const HInliner* current = this; 1288 size_t count = 0; 1289 do { 1290 if (current->graph_->GetArtMethod() == method) { 1291 ++count; 1292 } 1293 current = current->parent_; 1294 } while (current != nullptr); 1295 return count; 1296 } 1297 1298 bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, 1299 ArtMethod* method, 1300 ReferenceTypeInfo receiver_type, 1301 HInstruction** return_replacement) { 1302 if (method->IsProxyMethod()) { 1303 LOG_FAIL(kNotInlinedProxy) 1304 << "Method " << method->PrettyMethod() 1305 << " is not inlined because of unimplemented inline support for proxy methods."; 1306 return false; 1307 } 1308 1309 if (CountRecursiveCallsOf(method) > kMaximumNumberOfRecursiveCalls) { 1310 LOG_FAIL(kNotInlinedRecursiveBudget) 1311 << "Method " 1312 << method->PrettyMethod() 1313 << " is not inlined because it has reached its recursive call budget."; 1314 return false; 1315 } 1316 1317 // Check whether we're allowed to inline. The outermost compilation unit is the relevant 1318 // dex file here (though the transitivity of an inline chain would allow checking the calller). 1319 if (!compiler_driver_->MayInline(method->GetDexFile(), 1320 outer_compilation_unit_.GetDexFile())) { 1321 if (TryPatternSubstitution(invoke_instruction, method, return_replacement)) { 1322 LOG_SUCCESS() << "Successfully replaced pattern of invoke " 1323 << method->PrettyMethod(); 1324 MaybeRecordStat(kReplacedInvokeWithSimplePattern); 1325 return true; 1326 } 1327 LOG_FAIL(kNotInlinedWont) 1328 << "Won't inline " << method->PrettyMethod() << " in " 1329 << outer_compilation_unit_.GetDexFile()->GetLocation() << " (" 1330 << caller_compilation_unit_.GetDexFile()->GetLocation() << ") from " 1331 << method->GetDexFile()->GetLocation(); 1332 return false; 1333 } 1334 1335 bool same_dex_file = IsSameDexFile(*outer_compilation_unit_.GetDexFile(), *method->GetDexFile()); 1336 1337 const DexFile::CodeItem* code_item = method->GetCodeItem(); 1338 1339 if (code_item == nullptr) { 1340 LOG_FAIL_NO_STAT() 1341 << "Method " << method->PrettyMethod() << " is not inlined because it is native"; 1342 return false; 1343 } 1344 1345 size_t inline_max_code_units = compiler_driver_->GetCompilerOptions().GetInlineMaxCodeUnits(); 1346 if (code_item->insns_size_in_code_units_ > inline_max_code_units) { 1347 LOG_FAIL(kNotInlinedCodeItem) 1348 << "Method " << method->PrettyMethod() 1349 << " is not inlined because its code item is too big: " 1350 << code_item->insns_size_in_code_units_ 1351 << " > " 1352 << inline_max_code_units; 1353 return false; 1354 } 1355 1356 if (code_item->tries_size_ != 0) { 1357 LOG_FAIL(kNotInlinedTryCatch) 1358 << "Method " << method->PrettyMethod() << " is not inlined because of try block"; 1359 return false; 1360 } 1361 1362 if (!method->IsCompilable()) { 1363 LOG_FAIL(kNotInlinedNotVerified) 1364 << "Method " << method->PrettyMethod() 1365 << " has soft failures un-handled by the compiler, so it cannot be inlined"; 1366 } 1367 1368 if (!method->GetDeclaringClass()->IsVerified()) { 1369 uint16_t class_def_idx = method->GetDeclaringClass()->GetDexClassDefIndex(); 1370 if (Runtime::Current()->UseJitCompilation() || 1371 !compiler_driver_->IsMethodVerifiedWithoutFailures( 1372 method->GetDexMethodIndex(), class_def_idx, *method->GetDexFile())) { 1373 LOG_FAIL(kNotInlinedNotVerified) 1374 << "Method " << method->PrettyMethod() 1375 << " couldn't be verified, so it cannot be inlined"; 1376 return false; 1377 } 1378 } 1379 1380 if (invoke_instruction->IsInvokeStaticOrDirect() && 1381 invoke_instruction->AsInvokeStaticOrDirect()->IsStaticWithImplicitClinitCheck()) { 1382 // Case of a static method that cannot be inlined because it implicitly 1383 // requires an initialization check of its declaring class. 1384 LOG_FAIL(kNotInlinedDexCache) << "Method " << method->PrettyMethod() 1385 << " is not inlined because it is static and requires a clinit" 1386 << " check that cannot be emitted due to Dex cache limitations"; 1387 return false; 1388 } 1389 1390 if (!TryBuildAndInlineHelper( 1391 invoke_instruction, method, receiver_type, same_dex_file, return_replacement)) { 1392 return false; 1393 } 1394 1395 LOG_SUCCESS() << method->PrettyMethod(); 1396 MaybeRecordStat(kInlinedInvoke); 1397 return true; 1398 } 1399 1400 static HInstruction* GetInvokeInputForArgVRegIndex(HInvoke* invoke_instruction, 1401 size_t arg_vreg_index) 1402 REQUIRES_SHARED(Locks::mutator_lock_) { 1403 size_t input_index = 0; 1404 for (size_t i = 0; i < arg_vreg_index; ++i, ++input_index) { 1405 DCHECK_LT(input_index, invoke_instruction->GetNumberOfArguments()); 1406 if (Primitive::Is64BitType(invoke_instruction->InputAt(input_index)->GetType())) { 1407 ++i; 1408 DCHECK_NE(i, arg_vreg_index); 1409 } 1410 } 1411 DCHECK_LT(input_index, invoke_instruction->GetNumberOfArguments()); 1412 return invoke_instruction->InputAt(input_index); 1413 } 1414 1415 // Try to recognize known simple patterns and replace invoke call with appropriate instructions. 1416 bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, 1417 ArtMethod* resolved_method, 1418 HInstruction** return_replacement) { 1419 InlineMethod inline_method; 1420 if (!InlineMethodAnalyser::AnalyseMethodCode(resolved_method, &inline_method)) { 1421 return false; 1422 } 1423 1424 switch (inline_method.opcode) { 1425 case kInlineOpNop: 1426 DCHECK_EQ(invoke_instruction->GetType(), Primitive::kPrimVoid); 1427 *return_replacement = nullptr; 1428 break; 1429 case kInlineOpReturnArg: 1430 *return_replacement = GetInvokeInputForArgVRegIndex(invoke_instruction, 1431 inline_method.d.return_data.arg); 1432 break; 1433 case kInlineOpNonWideConst: 1434 if (resolved_method->GetShorty()[0] == 'L') { 1435 DCHECK_EQ(inline_method.d.data, 0u); 1436 *return_replacement = graph_->GetNullConstant(); 1437 } else { 1438 *return_replacement = graph_->GetIntConstant(static_cast<int32_t>(inline_method.d.data)); 1439 } 1440 break; 1441 case kInlineOpIGet: { 1442 const InlineIGetIPutData& data = inline_method.d.ifield_data; 1443 if (data.method_is_static || data.object_arg != 0u) { 1444 // TODO: Needs null check. 1445 return false; 1446 } 1447 HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg); 1448 HInstanceFieldGet* iget = CreateInstanceFieldGet(data.field_idx, resolved_method, obj); 1449 DCHECK_EQ(iget->GetFieldOffset().Uint32Value(), data.field_offset); 1450 DCHECK_EQ(iget->IsVolatile() ? 1u : 0u, data.is_volatile); 1451 invoke_instruction->GetBlock()->InsertInstructionBefore(iget, invoke_instruction); 1452 *return_replacement = iget; 1453 break; 1454 } 1455 case kInlineOpIPut: { 1456 const InlineIGetIPutData& data = inline_method.d.ifield_data; 1457 if (data.method_is_static || data.object_arg != 0u) { 1458 // TODO: Needs null check. 1459 return false; 1460 } 1461 HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg); 1462 HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, data.src_arg); 1463 HInstanceFieldSet* iput = CreateInstanceFieldSet(data.field_idx, resolved_method, obj, value); 1464 DCHECK_EQ(iput->GetFieldOffset().Uint32Value(), data.field_offset); 1465 DCHECK_EQ(iput->IsVolatile() ? 1u : 0u, data.is_volatile); 1466 invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction); 1467 if (data.return_arg_plus1 != 0u) { 1468 size_t return_arg = data.return_arg_plus1 - 1u; 1469 *return_replacement = GetInvokeInputForArgVRegIndex(invoke_instruction, return_arg); 1470 } 1471 break; 1472 } 1473 case kInlineOpConstructor: { 1474 const InlineConstructorData& data = inline_method.d.constructor_data; 1475 // Get the indexes to arrays for easier processing. 1476 uint16_t iput_field_indexes[] = { 1477 data.iput0_field_index, data.iput1_field_index, data.iput2_field_index 1478 }; 1479 uint16_t iput_args[] = { data.iput0_arg, data.iput1_arg, data.iput2_arg }; 1480 static_assert(arraysize(iput_args) == arraysize(iput_field_indexes), "Size mismatch"); 1481 // Count valid field indexes. 1482 size_t number_of_iputs = 0u; 1483 while (number_of_iputs != arraysize(iput_field_indexes) && 1484 iput_field_indexes[number_of_iputs] != DexFile::kDexNoIndex16) { 1485 // Check that there are no duplicate valid field indexes. 1486 DCHECK_EQ(0, std::count(iput_field_indexes + number_of_iputs + 1, 1487 iput_field_indexes + arraysize(iput_field_indexes), 1488 iput_field_indexes[number_of_iputs])); 1489 ++number_of_iputs; 1490 } 1491 // Check that there are no valid field indexes in the rest of the array. 1492 DCHECK_EQ(0, std::count_if(iput_field_indexes + number_of_iputs, 1493 iput_field_indexes + arraysize(iput_field_indexes), 1494 [](uint16_t index) { return index != DexFile::kDexNoIndex16; })); 1495 1496 // Create HInstanceFieldSet for each IPUT that stores non-zero data. 1497 HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, /* this */ 0u); 1498 bool needs_constructor_barrier = false; 1499 for (size_t i = 0; i != number_of_iputs; ++i) { 1500 HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]); 1501 if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) { 1502 uint16_t field_index = iput_field_indexes[i]; 1503 bool is_final; 1504 HInstanceFieldSet* iput = 1505 CreateInstanceFieldSet(field_index, resolved_method, obj, value, &is_final); 1506 invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction); 1507 1508 // Check whether the field is final. If it is, we need to add a barrier. 1509 if (is_final) { 1510 needs_constructor_barrier = true; 1511 } 1512 } 1513 } 1514 if (needs_constructor_barrier) { 1515 // See CompilerDriver::RequiresConstructorBarrier for more details. 1516 DCHECK(obj != nullptr) << "only non-static methods can have a constructor fence"; 1517 1518 HConstructorFence* constructor_fence = 1519 new (graph_->GetArena()) HConstructorFence(obj, kNoDexPc, graph_->GetArena()); 1520 invoke_instruction->GetBlock()->InsertInstructionBefore(constructor_fence, 1521 invoke_instruction); 1522 } 1523 *return_replacement = nullptr; 1524 break; 1525 } 1526 default: 1527 LOG(FATAL) << "UNREACHABLE"; 1528 UNREACHABLE(); 1529 } 1530 return true; 1531 } 1532 1533 HInstanceFieldGet* HInliner::CreateInstanceFieldGet(uint32_t field_index, 1534 ArtMethod* referrer, 1535 HInstruction* obj) 1536 REQUIRES_SHARED(Locks::mutator_lock_) { 1537 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 1538 ArtField* resolved_field = 1539 class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false); 1540 DCHECK(resolved_field != nullptr); 1541 HInstanceFieldGet* iget = new (graph_->GetArena()) HInstanceFieldGet( 1542 obj, 1543 resolved_field, 1544 resolved_field->GetTypeAsPrimitiveType(), 1545 resolved_field->GetOffset(), 1546 resolved_field->IsVolatile(), 1547 field_index, 1548 resolved_field->GetDeclaringClass()->GetDexClassDefIndex(), 1549 *referrer->GetDexFile(), 1550 // Read barrier generates a runtime call in slow path and we need a valid 1551 // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537. 1552 /* dex_pc */ 0); 1553 if (iget->GetType() == Primitive::kPrimNot) { 1554 // Use the same dex_cache that we used for field lookup as the hint_dex_cache. 1555 Handle<mirror::DexCache> dex_cache = handles_->NewHandle(referrer->GetDexCache()); 1556 ReferenceTypePropagation rtp(graph_, 1557 outer_compilation_unit_.GetClassLoader(), 1558 dex_cache, 1559 handles_, 1560 /* is_first_run */ false); 1561 rtp.Visit(iget); 1562 } 1563 return iget; 1564 } 1565 1566 HInstanceFieldSet* HInliner::CreateInstanceFieldSet(uint32_t field_index, 1567 ArtMethod* referrer, 1568 HInstruction* obj, 1569 HInstruction* value, 1570 bool* is_final) 1571 REQUIRES_SHARED(Locks::mutator_lock_) { 1572 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 1573 ArtField* resolved_field = 1574 class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false); 1575 DCHECK(resolved_field != nullptr); 1576 if (is_final != nullptr) { 1577 // This information is needed only for constructors. 1578 DCHECK(referrer->IsConstructor()); 1579 *is_final = resolved_field->IsFinal(); 1580 } 1581 HInstanceFieldSet* iput = new (graph_->GetArena()) HInstanceFieldSet( 1582 obj, 1583 value, 1584 resolved_field, 1585 resolved_field->GetTypeAsPrimitiveType(), 1586 resolved_field->GetOffset(), 1587 resolved_field->IsVolatile(), 1588 field_index, 1589 resolved_field->GetDeclaringClass()->GetDexClassDefIndex(), 1590 *referrer->GetDexFile(), 1591 // Read barrier generates a runtime call in slow path and we need a valid 1592 // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537. 1593 /* dex_pc */ 0); 1594 return iput; 1595 } 1596 1597 template <typename T> 1598 static inline Handle<T> NewHandleIfDifferent(T* object, 1599 Handle<T> hint, 1600 VariableSizedHandleScope* handles) 1601 REQUIRES_SHARED(Locks::mutator_lock_) { 1602 return (object != hint.Get()) ? handles->NewHandle(object) : hint; 1603 } 1604 1605 bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, 1606 ArtMethod* resolved_method, 1607 ReferenceTypeInfo receiver_type, 1608 bool same_dex_file, 1609 HInstruction** return_replacement) { 1610 DCHECK(!(resolved_method->IsStatic() && receiver_type.IsValid())); 1611 ScopedObjectAccess soa(Thread::Current()); 1612 const DexFile::CodeItem* code_item = resolved_method->GetCodeItem(); 1613 const DexFile& callee_dex_file = *resolved_method->GetDexFile(); 1614 uint32_t method_index = resolved_method->GetDexMethodIndex(); 1615 ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); 1616 Handle<mirror::DexCache> dex_cache = NewHandleIfDifferent(resolved_method->GetDexCache(), 1617 caller_compilation_unit_.GetDexCache(), 1618 handles_); 1619 Handle<mirror::ClassLoader> class_loader = 1620 NewHandleIfDifferent(resolved_method->GetDeclaringClass()->GetClassLoader(), 1621 caller_compilation_unit_.GetClassLoader(), 1622 handles_); 1623 1624 DexCompilationUnit dex_compilation_unit( 1625 class_loader, 1626 class_linker, 1627 callee_dex_file, 1628 code_item, 1629 resolved_method->GetDeclaringClass()->GetDexClassDefIndex(), 1630 method_index, 1631 resolved_method->GetAccessFlags(), 1632 /* verified_method */ nullptr, 1633 dex_cache); 1634 1635 InvokeType invoke_type = invoke_instruction->GetInvokeType(); 1636 if (invoke_type == kInterface) { 1637 // We have statically resolved the dispatch. To please the class linker 1638 // at runtime, we change this call as if it was a virtual call. 1639 invoke_type = kVirtual; 1640 } 1641 1642 const int32_t caller_instruction_counter = graph_->GetCurrentInstructionId(); 1643 HGraph* callee_graph = new (graph_->GetArena()) HGraph( 1644 graph_->GetArena(), 1645 callee_dex_file, 1646 method_index, 1647 compiler_driver_->GetInstructionSet(), 1648 invoke_type, 1649 graph_->IsDebuggable(), 1650 /* osr */ false, 1651 caller_instruction_counter); 1652 callee_graph->SetArtMethod(resolved_method); 1653 1654 // When they are needed, allocate `inline_stats_` on the Arena instead 1655 // of on the stack, as Clang might produce a stack frame too large 1656 // for this function, that would not fit the requirements of the 1657 // `-Wframe-larger-than` option. 1658 if (stats_ != nullptr) { 1659 // Reuse one object for all inline attempts from this caller to keep Arena memory usage low. 1660 if (inline_stats_ == nullptr) { 1661 void* storage = graph_->GetArena()->Alloc<OptimizingCompilerStats>(kArenaAllocMisc); 1662 inline_stats_ = new (storage) OptimizingCompilerStats; 1663 } else { 1664 inline_stats_->Reset(); 1665 } 1666 } 1667 HGraphBuilder builder(callee_graph, 1668 &dex_compilation_unit, 1669 &outer_compilation_unit_, 1670 resolved_method->GetDexFile(), 1671 *code_item, 1672 compiler_driver_, 1673 codegen_, 1674 inline_stats_, 1675 resolved_method->GetQuickenedInfo(class_linker->GetImagePointerSize()), 1676 dex_cache, 1677 handles_); 1678 1679 if (builder.BuildGraph() != kAnalysisSuccess) { 1680 LOG_FAIL(kNotInlinedCannotBuild) 1681 << "Method " << callee_dex_file.PrettyMethod(method_index) 1682 << " could not be built, so cannot be inlined"; 1683 return false; 1684 } 1685 1686 if (!RegisterAllocator::CanAllocateRegistersFor(*callee_graph, 1687 compiler_driver_->GetInstructionSet())) { 1688 LOG_FAIL(kNotInlinedRegisterAllocator) 1689 << "Method " << callee_dex_file.PrettyMethod(method_index) 1690 << " cannot be inlined because of the register allocator"; 1691 return false; 1692 } 1693 1694 size_t parameter_index = 0; 1695 bool run_rtp = false; 1696 for (HInstructionIterator instructions(callee_graph->GetEntryBlock()->GetInstructions()); 1697 !instructions.Done(); 1698 instructions.Advance()) { 1699 HInstruction* current = instructions.Current(); 1700 if (current->IsParameterValue()) { 1701 HInstruction* argument = invoke_instruction->InputAt(parameter_index); 1702 if (argument->IsNullConstant()) { 1703 current->ReplaceWith(callee_graph->GetNullConstant()); 1704 } else if (argument->IsIntConstant()) { 1705 current->ReplaceWith(callee_graph->GetIntConstant(argument->AsIntConstant()->GetValue())); 1706 } else if (argument->IsLongConstant()) { 1707 current->ReplaceWith(callee_graph->GetLongConstant(argument->AsLongConstant()->GetValue())); 1708 } else if (argument->IsFloatConstant()) { 1709 current->ReplaceWith( 1710 callee_graph->GetFloatConstant(argument->AsFloatConstant()->GetValue())); 1711 } else if (argument->IsDoubleConstant()) { 1712 current->ReplaceWith( 1713 callee_graph->GetDoubleConstant(argument->AsDoubleConstant()->GetValue())); 1714 } else if (argument->GetType() == Primitive::kPrimNot) { 1715 if (!resolved_method->IsStatic() && parameter_index == 0 && receiver_type.IsValid()) { 1716 run_rtp = true; 1717 current->SetReferenceTypeInfo(receiver_type); 1718 } else { 1719 current->SetReferenceTypeInfo(argument->GetReferenceTypeInfo()); 1720 } 1721 current->AsParameterValue()->SetCanBeNull(argument->CanBeNull()); 1722 } 1723 ++parameter_index; 1724 } 1725 } 1726 1727 // We have replaced formal arguments with actual arguments. If actual types 1728 // are more specific than the declared ones, run RTP again on the inner graph. 1729 if (run_rtp || ArgumentTypesMoreSpecific(invoke_instruction, resolved_method)) { 1730 ReferenceTypePropagation(callee_graph, 1731 outer_compilation_unit_.GetClassLoader(), 1732 dex_compilation_unit.GetDexCache(), 1733 handles_, 1734 /* is_first_run */ false).Run(); 1735 } 1736 1737 RunOptimizations(callee_graph, code_item, dex_compilation_unit); 1738 1739 HBasicBlock* exit_block = callee_graph->GetExitBlock(); 1740 if (exit_block == nullptr) { 1741 LOG_FAIL(kNotInlinedInfiniteLoop) 1742 << "Method " << callee_dex_file.PrettyMethod(method_index) 1743 << " could not be inlined because it has an infinite loop"; 1744 return false; 1745 } 1746 1747 bool has_one_return = false; 1748 for (HBasicBlock* predecessor : exit_block->GetPredecessors()) { 1749 if (predecessor->GetLastInstruction()->IsThrow()) { 1750 if (invoke_instruction->GetBlock()->IsTryBlock()) { 1751 // TODO(ngeoffray): Support adding HTryBoundary in Hgraph::InlineInto. 1752 LOG_FAIL(kNotInlinedTryCatch) 1753 << "Method " << callee_dex_file.PrettyMethod(method_index) 1754 << " could not be inlined because one branch always throws and" 1755 << " caller is in a try/catch block"; 1756 return false; 1757 } else if (graph_->GetExitBlock() == nullptr) { 1758 // TODO(ngeoffray): Support adding HExit in the caller graph. 1759 LOG_FAIL(kNotInlinedInfiniteLoop) 1760 << "Method " << callee_dex_file.PrettyMethod(method_index) 1761 << " could not be inlined because one branch always throws and" 1762 << " caller does not have an exit block"; 1763 return false; 1764 } else if (graph_->HasIrreducibleLoops()) { 1765 // TODO(ngeoffray): Support re-computing loop information to graphs with 1766 // irreducible loops? 1767 VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) 1768 << " could not be inlined because one branch always throws and" 1769 << " caller has irreducible loops"; 1770 return false; 1771 } 1772 } else { 1773 has_one_return = true; 1774 } 1775 } 1776 1777 if (!has_one_return) { 1778 LOG_FAIL(kNotInlinedAlwaysThrows) 1779 << "Method " << callee_dex_file.PrettyMethod(method_index) 1780 << " could not be inlined because it always throws"; 1781 return false; 1782 } 1783 1784 size_t number_of_instructions = 0; 1785 // Skip the entry block, it does not contain instructions that prevent inlining. 1786 for (HBasicBlock* block : callee_graph->GetReversePostOrderSkipEntryBlock()) { 1787 if (block->IsLoopHeader()) { 1788 if (block->GetLoopInformation()->IsIrreducible()) { 1789 // Don't inline methods with irreducible loops, they could prevent some 1790 // optimizations to run. 1791 LOG_FAIL(kNotInlinedIrreducibleLoop) 1792 << "Method " << callee_dex_file.PrettyMethod(method_index) 1793 << " could not be inlined because it contains an irreducible loop"; 1794 return false; 1795 } 1796 if (!block->GetLoopInformation()->HasExitEdge()) { 1797 // Don't inline methods with loops without exit, since they cause the 1798 // loop information to be computed incorrectly when updating after 1799 // inlining. 1800 LOG_FAIL(kNotInlinedLoopWithoutExit) 1801 << "Method " << callee_dex_file.PrettyMethod(method_index) 1802 << " could not be inlined because it contains a loop with no exit"; 1803 return false; 1804 } 1805 } 1806 1807 for (HInstructionIterator instr_it(block->GetInstructions()); 1808 !instr_it.Done(); 1809 instr_it.Advance()) { 1810 if (++number_of_instructions >= inlining_budget_) { 1811 LOG_FAIL(kNotInlinedInstructionBudget) 1812 << "Method " << callee_dex_file.PrettyMethod(method_index) 1813 << " is not inlined because the outer method has reached" 1814 << " its instruction budget limit."; 1815 return false; 1816 } 1817 HInstruction* current = instr_it.Current(); 1818 if (current->NeedsEnvironment() && 1819 (total_number_of_dex_registers_ >= kMaximumNumberOfCumulatedDexRegisters)) { 1820 LOG_FAIL(kNotInlinedEnvironmentBudget) 1821 << "Method " << callee_dex_file.PrettyMethod(method_index) 1822 << " is not inlined because its caller has reached" 1823 << " its environment budget limit."; 1824 return false; 1825 } 1826 1827 if (current->NeedsEnvironment() && 1828 !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(), 1829 resolved_method)) { 1830 LOG_FAIL(kNotInlinedStackMaps) 1831 << "Method " << callee_dex_file.PrettyMethod(method_index) 1832 << " could not be inlined because " << current->DebugName() 1833 << " needs an environment, is in a different dex file" 1834 << ", and cannot be encoded in the stack maps."; 1835 return false; 1836 } 1837 1838 if (!same_dex_file && current->NeedsDexCacheOfDeclaringClass()) { 1839 LOG_FAIL(kNotInlinedDexCache) 1840 << "Method " << callee_dex_file.PrettyMethod(method_index) 1841 << " could not be inlined because " << current->DebugName() 1842 << " it is in a different dex file and requires access to the dex cache"; 1843 return false; 1844 } 1845 1846 if (current->IsUnresolvedStaticFieldGet() || 1847 current->IsUnresolvedInstanceFieldGet() || 1848 current->IsUnresolvedStaticFieldSet() || 1849 current->IsUnresolvedInstanceFieldSet()) { 1850 // Entrypoint for unresolved fields does not handle inlined frames. 1851 LOG_FAIL(kNotInlinedUnresolvedEntrypoint) 1852 << "Method " << callee_dex_file.PrettyMethod(method_index) 1853 << " could not be inlined because it is using an unresolved" 1854 << " entrypoint"; 1855 return false; 1856 } 1857 } 1858 } 1859 DCHECK_EQ(caller_instruction_counter, graph_->GetCurrentInstructionId()) 1860 << "No instructions can be added to the outer graph while inner graph is being built"; 1861 1862 // Inline the callee graph inside the caller graph. 1863 const int32_t callee_instruction_counter = callee_graph->GetCurrentInstructionId(); 1864 graph_->SetCurrentInstructionId(callee_instruction_counter); 1865 *return_replacement = callee_graph->InlineInto(graph_, invoke_instruction); 1866 // Update our budget for other inlining attempts in `caller_graph`. 1867 total_number_of_instructions_ += number_of_instructions; 1868 UpdateInliningBudget(); 1869 1870 DCHECK_EQ(callee_instruction_counter, callee_graph->GetCurrentInstructionId()) 1871 << "No instructions can be added to the inner graph during inlining into the outer graph"; 1872 1873 if (stats_ != nullptr) { 1874 DCHECK(inline_stats_ != nullptr); 1875 inline_stats_->AddTo(stats_); 1876 } 1877 1878 return true; 1879 } 1880 1881 void HInliner::RunOptimizations(HGraph* callee_graph, 1882 const DexFile::CodeItem* code_item, 1883 const DexCompilationUnit& dex_compilation_unit) { 1884 // Note: if the outermost_graph_ is being compiled OSR, we should not run any 1885 // optimization that could lead to a HDeoptimize. The following optimizations do not. 1886 HDeadCodeElimination dce(callee_graph, inline_stats_, "dead_code_elimination$inliner"); 1887 HConstantFolding fold(callee_graph, "constant_folding$inliner"); 1888 HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_, handles_); 1889 InstructionSimplifier simplify(callee_graph, codegen_, compiler_driver_, inline_stats_); 1890 IntrinsicsRecognizer intrinsics(callee_graph, inline_stats_); 1891 1892 HOptimization* optimizations[] = { 1893 &intrinsics, 1894 &sharpening, 1895 &simplify, 1896 &fold, 1897 &dce, 1898 }; 1899 1900 for (size_t i = 0; i < arraysize(optimizations); ++i) { 1901 HOptimization* optimization = optimizations[i]; 1902 optimization->Run(); 1903 } 1904 1905 // Bail early for pathological cases on the environment (for example recursive calls, 1906 // or too large environment). 1907 if (total_number_of_dex_registers_ >= kMaximumNumberOfCumulatedDexRegisters) { 1908 LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod() 1909 << " will not be inlined because the outer method has reached" 1910 << " its environment budget limit."; 1911 return; 1912 } 1913 1914 // Bail early if we know we already are over the limit. 1915 size_t number_of_instructions = CountNumberOfInstructions(callee_graph); 1916 if (number_of_instructions > inlining_budget_) { 1917 LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod() 1918 << " will not be inlined because the outer method has reached" 1919 << " its instruction budget limit. " << number_of_instructions; 1920 return; 1921 } 1922 1923 HInliner inliner(callee_graph, 1924 outermost_graph_, 1925 codegen_, 1926 outer_compilation_unit_, 1927 dex_compilation_unit, 1928 compiler_driver_, 1929 handles_, 1930 inline_stats_, 1931 total_number_of_dex_registers_ + code_item->registers_size_, 1932 total_number_of_instructions_ + number_of_instructions, 1933 this, 1934 depth_ + 1); 1935 inliner.Run(); 1936 } 1937 1938 static bool IsReferenceTypeRefinement(ReferenceTypeInfo declared_rti, 1939 bool declared_can_be_null, 1940 HInstruction* actual_obj) 1941 REQUIRES_SHARED(Locks::mutator_lock_) { 1942 if (declared_can_be_null && !actual_obj->CanBeNull()) { 1943 return true; 1944 } 1945 1946 ReferenceTypeInfo actual_rti = actual_obj->GetReferenceTypeInfo(); 1947 return (actual_rti.IsExact() && !declared_rti.IsExact()) || 1948 declared_rti.IsStrictSupertypeOf(actual_rti); 1949 } 1950 1951 ReferenceTypeInfo HInliner::GetClassRTI(mirror::Class* klass) { 1952 return ReferenceTypePropagation::IsAdmissible(klass) 1953 ? ReferenceTypeInfo::Create(handles_->NewHandle(klass)) 1954 : graph_->GetInexactObjectRti(); 1955 } 1956 1957 bool HInliner::ArgumentTypesMoreSpecific(HInvoke* invoke_instruction, ArtMethod* resolved_method) { 1958 // If this is an instance call, test whether the type of the `this` argument 1959 // is more specific than the class which declares the method. 1960 if (!resolved_method->IsStatic()) { 1961 if (IsReferenceTypeRefinement(GetClassRTI(resolved_method->GetDeclaringClass()), 1962 /* declared_can_be_null */ false, 1963 invoke_instruction->InputAt(0u))) { 1964 return true; 1965 } 1966 } 1967 1968 // Iterate over the list of parameter types and test whether any of the 1969 // actual inputs has a more specific reference type than the type declared in 1970 // the signature. 1971 const DexFile::TypeList* param_list = resolved_method->GetParameterTypeList(); 1972 for (size_t param_idx = 0, 1973 input_idx = resolved_method->IsStatic() ? 0 : 1, 1974 e = (param_list == nullptr ? 0 : param_list->Size()); 1975 param_idx < e; 1976 ++param_idx, ++input_idx) { 1977 HInstruction* input = invoke_instruction->InputAt(input_idx); 1978 if (input->GetType() == Primitive::kPrimNot) { 1979 mirror::Class* param_cls = resolved_method->GetClassFromTypeIndex( 1980 param_list->GetTypeItem(param_idx).type_idx_, 1981 /* resolve */ false); 1982 if (IsReferenceTypeRefinement(GetClassRTI(param_cls), 1983 /* declared_can_be_null */ true, 1984 input)) { 1985 return true; 1986 } 1987 } 1988 } 1989 1990 return false; 1991 } 1992 1993 bool HInliner::ReturnTypeMoreSpecific(HInvoke* invoke_instruction, 1994 HInstruction* return_replacement) { 1995 // Check the integrity of reference types and run another type propagation if needed. 1996 if (return_replacement != nullptr) { 1997 if (return_replacement->GetType() == Primitive::kPrimNot) { 1998 // Test if the return type is a refinement of the declared return type. 1999 if (IsReferenceTypeRefinement(invoke_instruction->GetReferenceTypeInfo(), 2000 /* declared_can_be_null */ true, 2001 return_replacement)) { 2002 return true; 2003 } else if (return_replacement->IsInstanceFieldGet()) { 2004 HInstanceFieldGet* field_get = return_replacement->AsInstanceFieldGet(); 2005 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 2006 if (field_get->GetFieldInfo().GetField() == 2007 class_linker->GetClassRoot(ClassLinker::kJavaLangObject)->GetInstanceField(0)) { 2008 return true; 2009 } 2010 } 2011 } else if (return_replacement->IsInstanceOf()) { 2012 // Inlining InstanceOf into an If may put a tighter bound on reference types. 2013 return true; 2014 } 2015 } 2016 2017 return false; 2018 } 2019 2020 void HInliner::FixUpReturnReferenceType(ArtMethod* resolved_method, 2021 HInstruction* return_replacement) { 2022 if (return_replacement != nullptr) { 2023 if (return_replacement->GetType() == Primitive::kPrimNot) { 2024 if (!return_replacement->GetReferenceTypeInfo().IsValid()) { 2025 // Make sure that we have a valid type for the return. We may get an invalid one when 2026 // we inline invokes with multiple branches and create a Phi for the result. 2027 // TODO: we could be more precise by merging the phi inputs but that requires 2028 // some functionality from the reference type propagation. 2029 DCHECK(return_replacement->IsPhi()); 2030 mirror::Class* cls = resolved_method->GetReturnType(false /* resolve */); 2031 return_replacement->SetReferenceTypeInfo(GetClassRTI(cls)); 2032 } 2033 } 2034 } 2035 } 2036 2037 } // namespace art 2038