1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "code_generator.h" 18 19 #ifdef ART_ENABLE_CODEGEN_arm 20 #include "code_generator_arm.h" 21 #include "code_generator_arm_vixl.h" 22 #endif 23 24 #ifdef ART_ENABLE_CODEGEN_arm64 25 #include "code_generator_arm64.h" 26 #endif 27 28 #ifdef ART_ENABLE_CODEGEN_x86 29 #include "code_generator_x86.h" 30 #endif 31 32 #ifdef ART_ENABLE_CODEGEN_x86_64 33 #include "code_generator_x86_64.h" 34 #endif 35 36 #ifdef ART_ENABLE_CODEGEN_mips 37 #include "code_generator_mips.h" 38 #endif 39 40 #ifdef ART_ENABLE_CODEGEN_mips64 41 #include "code_generator_mips64.h" 42 #endif 43 44 #include "bytecode_utils.h" 45 #include "class_linker.h" 46 #include "compiled_method.h" 47 #include "dex/verified_method.h" 48 #include "driver/compiler_driver.h" 49 #include "graph_visualizer.h" 50 #include "intern_table.h" 51 #include "intrinsics.h" 52 #include "leb128.h" 53 #include "mirror/array-inl.h" 54 #include "mirror/object_array-inl.h" 55 #include "mirror/object_reference.h" 56 #include "mirror/reference.h" 57 #include "mirror/string.h" 58 #include "parallel_move_resolver.h" 59 #include "ssa_liveness_analysis.h" 60 #include "scoped_thread_state_change-inl.h" 61 #include "thread-inl.h" 62 #include "utils/assembler.h" 63 64 namespace art { 65 66 // If true, we record the static and direct invokes in the invoke infos. 67 static constexpr bool kEnableDexLayoutOptimizations = false; 68 69 // Return whether a location is consistent with a type. 70 static bool CheckType(Primitive::Type type, Location location) { 71 if (location.IsFpuRegister() 72 || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) { 73 return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble); 74 } else if (location.IsRegister() || 75 (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) { 76 return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot); 77 } else if (location.IsRegisterPair()) { 78 return type == Primitive::kPrimLong; 79 } else if (location.IsFpuRegisterPair()) { 80 return type == Primitive::kPrimDouble; 81 } else if (location.IsStackSlot()) { 82 return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong) 83 || (type == Primitive::kPrimFloat) 84 || (type == Primitive::kPrimNot); 85 } else if (location.IsDoubleStackSlot()) { 86 return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble); 87 } else if (location.IsConstant()) { 88 if (location.GetConstant()->IsIntConstant()) { 89 return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong); 90 } else if (location.GetConstant()->IsNullConstant()) { 91 return type == Primitive::kPrimNot; 92 } else if (location.GetConstant()->IsLongConstant()) { 93 return type == Primitive::kPrimLong; 94 } else if (location.GetConstant()->IsFloatConstant()) { 95 return type == Primitive::kPrimFloat; 96 } else { 97 return location.GetConstant()->IsDoubleConstant() 98 && (type == Primitive::kPrimDouble); 99 } 100 } else { 101 return location.IsInvalid() || (location.GetPolicy() == Location::kAny); 102 } 103 } 104 105 // Check that a location summary is consistent with an instruction. 106 static bool CheckTypeConsistency(HInstruction* instruction) { 107 LocationSummary* locations = instruction->GetLocations(); 108 if (locations == nullptr) { 109 return true; 110 } 111 112 if (locations->Out().IsUnallocated() 113 && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) { 114 DCHECK(CheckType(instruction->GetType(), locations->InAt(0))) 115 << instruction->GetType() 116 << " " << locations->InAt(0); 117 } else { 118 DCHECK(CheckType(instruction->GetType(), locations->Out())) 119 << instruction->GetType() 120 << " " << locations->Out(); 121 } 122 123 HConstInputsRef inputs = instruction->GetInputs(); 124 for (size_t i = 0; i < inputs.size(); ++i) { 125 DCHECK(CheckType(inputs[i]->GetType(), locations->InAt(i))) 126 << inputs[i]->GetType() << " " << locations->InAt(i); 127 } 128 129 HEnvironment* environment = instruction->GetEnvironment(); 130 for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) { 131 if (environment->GetInstructionAt(i) != nullptr) { 132 Primitive::Type type = environment->GetInstructionAt(i)->GetType(); 133 DCHECK(CheckType(type, environment->GetLocationAt(i))) 134 << type << " " << environment->GetLocationAt(i); 135 } else { 136 DCHECK(environment->GetLocationAt(i).IsInvalid()) 137 << environment->GetLocationAt(i); 138 } 139 } 140 return true; 141 } 142 143 size_t CodeGenerator::GetCacheOffset(uint32_t index) { 144 return sizeof(GcRoot<mirror::Object>) * index; 145 } 146 147 size_t CodeGenerator::GetCachePointerOffset(uint32_t index) { 148 auto pointer_size = InstructionSetPointerSize(GetInstructionSet()); 149 return static_cast<size_t>(pointer_size) * index; 150 } 151 152 uint32_t CodeGenerator::GetArrayLengthOffset(HArrayLength* array_length) { 153 return array_length->IsStringLength() 154 ? mirror::String::CountOffset().Uint32Value() 155 : mirror::Array::LengthOffset().Uint32Value(); 156 } 157 158 uint32_t CodeGenerator::GetArrayDataOffset(HArrayGet* array_get) { 159 DCHECK(array_get->GetType() == Primitive::kPrimChar || !array_get->IsStringCharAt()); 160 return array_get->IsStringCharAt() 161 ? mirror::String::ValueOffset().Uint32Value() 162 : mirror::Array::DataOffset(Primitive::ComponentSize(array_get->GetType())).Uint32Value(); 163 } 164 165 bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const { 166 DCHECK_EQ((*block_order_)[current_block_index_], current); 167 return GetNextBlockToEmit() == FirstNonEmptyBlock(next); 168 } 169 170 HBasicBlock* CodeGenerator::GetNextBlockToEmit() const { 171 for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) { 172 HBasicBlock* block = (*block_order_)[i]; 173 if (!block->IsSingleJump()) { 174 return block; 175 } 176 } 177 return nullptr; 178 } 179 180 HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const { 181 while (block->IsSingleJump()) { 182 block = block->GetSuccessors()[0]; 183 } 184 return block; 185 } 186 187 class DisassemblyScope { 188 public: 189 DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen) 190 : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) { 191 if (codegen_.GetDisassemblyInformation() != nullptr) { 192 start_offset_ = codegen_.GetAssembler().CodeSize(); 193 } 194 } 195 196 ~DisassemblyScope() { 197 // We avoid building this data when we know it will not be used. 198 if (codegen_.GetDisassemblyInformation() != nullptr) { 199 codegen_.GetDisassemblyInformation()->AddInstructionInterval( 200 instruction_, start_offset_, codegen_.GetAssembler().CodeSize()); 201 } 202 } 203 204 private: 205 const CodeGenerator& codegen_; 206 HInstruction* instruction_; 207 size_t start_offset_; 208 }; 209 210 211 void CodeGenerator::GenerateSlowPaths() { 212 size_t code_start = 0; 213 for (const std::unique_ptr<SlowPathCode>& slow_path_unique_ptr : slow_paths_) { 214 SlowPathCode* slow_path = slow_path_unique_ptr.get(); 215 current_slow_path_ = slow_path; 216 if (disasm_info_ != nullptr) { 217 code_start = GetAssembler()->CodeSize(); 218 } 219 // Record the dex pc at start of slow path (required for java line number mapping). 220 MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path); 221 slow_path->EmitNativeCode(this); 222 if (disasm_info_ != nullptr) { 223 disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize()); 224 } 225 } 226 current_slow_path_ = nullptr; 227 } 228 229 void CodeGenerator::Compile(CodeAllocator* allocator) { 230 // The register allocator already called `InitializeCodeGeneration`, 231 // where the frame size has been computed. 232 DCHECK(block_order_ != nullptr); 233 Initialize(); 234 235 HGraphVisitor* instruction_visitor = GetInstructionVisitor(); 236 DCHECK_EQ(current_block_index_, 0u); 237 238 size_t frame_start = GetAssembler()->CodeSize(); 239 GenerateFrameEntry(); 240 DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_)); 241 if (disasm_info_ != nullptr) { 242 disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize()); 243 } 244 245 for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) { 246 HBasicBlock* block = (*block_order_)[current_block_index_]; 247 // Don't generate code for an empty block. Its predecessors will branch to its successor 248 // directly. Also, the label of that block will not be emitted, so this helps catch 249 // errors where we reference that label. 250 if (block->IsSingleJump()) continue; 251 Bind(block); 252 // This ensures that we have correct native line mapping for all native instructions. 253 // It is necessary to make stepping over a statement work. Otherwise, any initial 254 // instructions (e.g. moves) would be assumed to be the start of next statement. 255 MaybeRecordNativeDebugInfo(nullptr /* instruction */, block->GetDexPc()); 256 for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { 257 HInstruction* current = it.Current(); 258 if (current->HasEnvironment()) { 259 // Create stackmap for HNativeDebugInfo or any instruction which calls native code. 260 // Note that we need correct mapping for the native PC of the call instruction, 261 // so the runtime's stackmap is not sufficient since it is at PC after the call. 262 MaybeRecordNativeDebugInfo(current, block->GetDexPc()); 263 } 264 DisassemblyScope disassembly_scope(current, *this); 265 DCHECK(CheckTypeConsistency(current)); 266 current->Accept(instruction_visitor); 267 } 268 } 269 270 GenerateSlowPaths(); 271 272 // Emit catch stack maps at the end of the stack map stream as expected by the 273 // runtime exception handler. 274 if (graph_->HasTryCatch()) { 275 RecordCatchBlockInfo(); 276 } 277 278 // Finalize instructions in assember; 279 Finalize(allocator); 280 } 281 282 void CodeGenerator::Finalize(CodeAllocator* allocator) { 283 size_t code_size = GetAssembler()->CodeSize(); 284 uint8_t* buffer = allocator->Allocate(code_size); 285 286 MemoryRegion code(buffer, code_size); 287 GetAssembler()->FinalizeInstructions(code); 288 } 289 290 void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) { 291 // No linker patches by default. 292 } 293 294 void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots, 295 size_t maximum_safepoint_spill_size, 296 size_t number_of_out_slots, 297 const ArenaVector<HBasicBlock*>& block_order) { 298 block_order_ = &block_order; 299 DCHECK(!block_order.empty()); 300 DCHECK(block_order[0] == GetGraph()->GetEntryBlock()); 301 ComputeSpillMask(); 302 first_register_slot_in_slow_path_ = RoundUp( 303 (number_of_out_slots + number_of_spill_slots) * kVRegSize, GetPreferredSlotsAlignment()); 304 305 if (number_of_spill_slots == 0 306 && !HasAllocatedCalleeSaveRegisters() 307 && IsLeafMethod() 308 && !RequiresCurrentMethod()) { 309 DCHECK_EQ(maximum_safepoint_spill_size, 0u); 310 SetFrameSize(CallPushesPC() ? GetWordSize() : 0); 311 } else { 312 SetFrameSize(RoundUp( 313 first_register_slot_in_slow_path_ 314 + maximum_safepoint_spill_size 315 + (GetGraph()->HasShouldDeoptimizeFlag() ? kShouldDeoptimizeFlagSize : 0) 316 + FrameEntrySpillSize(), 317 kStackAlignment)); 318 } 319 } 320 321 void CodeGenerator::CreateCommonInvokeLocationSummary( 322 HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) { 323 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena(); 324 LocationSummary* locations = new (allocator) LocationSummary(invoke, 325 LocationSummary::kCallOnMainOnly); 326 327 for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) { 328 HInstruction* input = invoke->InputAt(i); 329 locations->SetInAt(i, visitor->GetNextLocation(input->GetType())); 330 } 331 332 locations->SetOut(visitor->GetReturnLocation(invoke->GetType())); 333 334 if (invoke->IsInvokeStaticOrDirect()) { 335 HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect(); 336 switch (call->GetMethodLoadKind()) { 337 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 338 locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation()); 339 break; 340 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: 341 locations->AddTemp(visitor->GetMethodLocation()); 342 locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister()); 343 break; 344 default: 345 locations->AddTemp(visitor->GetMethodLocation()); 346 break; 347 } 348 } else { 349 locations->AddTemp(visitor->GetMethodLocation()); 350 } 351 } 352 353 void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) { 354 MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex()); 355 356 // Initialize to anything to silent compiler warnings. 357 QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck; 358 switch (invoke->GetInvokeType()) { 359 case kStatic: 360 entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck; 361 break; 362 case kDirect: 363 entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck; 364 break; 365 case kVirtual: 366 entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck; 367 break; 368 case kSuper: 369 entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck; 370 break; 371 case kInterface: 372 entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck; 373 break; 374 } 375 InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr); 376 } 377 378 void CodeGenerator::GenerateInvokePolymorphicCall(HInvokePolymorphic* invoke) { 379 MoveConstant(invoke->GetLocations()->GetTemp(0), static_cast<int32_t>(invoke->GetType())); 380 QuickEntrypointEnum entrypoint = kQuickInvokePolymorphic; 381 InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr); 382 } 383 384 void CodeGenerator::CreateUnresolvedFieldLocationSummary( 385 HInstruction* field_access, 386 Primitive::Type field_type, 387 const FieldAccessCallingConvention& calling_convention) { 388 bool is_instance = field_access->IsUnresolvedInstanceFieldGet() 389 || field_access->IsUnresolvedInstanceFieldSet(); 390 bool is_get = field_access->IsUnresolvedInstanceFieldGet() 391 || field_access->IsUnresolvedStaticFieldGet(); 392 393 ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena(); 394 LocationSummary* locations = 395 new (allocator) LocationSummary(field_access, LocationSummary::kCallOnMainOnly); 396 397 locations->AddTemp(calling_convention.GetFieldIndexLocation()); 398 399 if (is_instance) { 400 // Add the `this` object for instance field accesses. 401 locations->SetInAt(0, calling_convention.GetObjectLocation()); 402 } 403 404 // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64 405 // regardless of the the type. Because of that we forced to special case 406 // the access to floating point values. 407 if (is_get) { 408 if (Primitive::IsFloatingPointType(field_type)) { 409 // The return value will be stored in regular registers while register 410 // allocator expects it in a floating point register. 411 // Note We don't need to request additional temps because the return 412 // register(s) are already blocked due the call and they may overlap with 413 // the input or field index. 414 // The transfer between the two will be done at codegen level. 415 locations->SetOut(calling_convention.GetFpuLocation(field_type)); 416 } else { 417 locations->SetOut(calling_convention.GetReturnLocation(field_type)); 418 } 419 } else { 420 size_t set_index = is_instance ? 1 : 0; 421 if (Primitive::IsFloatingPointType(field_type)) { 422 // The set value comes from a float location while the calling convention 423 // expects it in a regular register location. Allocate a temp for it and 424 // make the transfer at codegen. 425 AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations); 426 locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type)); 427 } else { 428 locations->SetInAt(set_index, 429 calling_convention.GetSetValueLocation(field_type, is_instance)); 430 } 431 } 432 } 433 434 void CodeGenerator::GenerateUnresolvedFieldAccess( 435 HInstruction* field_access, 436 Primitive::Type field_type, 437 uint32_t field_index, 438 uint32_t dex_pc, 439 const FieldAccessCallingConvention& calling_convention) { 440 LocationSummary* locations = field_access->GetLocations(); 441 442 MoveConstant(locations->GetTemp(0), field_index); 443 444 bool is_instance = field_access->IsUnresolvedInstanceFieldGet() 445 || field_access->IsUnresolvedInstanceFieldSet(); 446 bool is_get = field_access->IsUnresolvedInstanceFieldGet() 447 || field_access->IsUnresolvedStaticFieldGet(); 448 449 if (!is_get && Primitive::IsFloatingPointType(field_type)) { 450 // Copy the float value to be set into the calling convention register. 451 // Note that using directly the temp location is problematic as we don't 452 // support temp register pairs. To avoid boilerplate conversion code, use 453 // the location from the calling convention. 454 MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance), 455 locations->InAt(is_instance ? 1 : 0), 456 (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt)); 457 } 458 459 QuickEntrypointEnum entrypoint = kQuickSet8Static; // Initialize to anything to avoid warnings. 460 switch (field_type) { 461 case Primitive::kPrimBoolean: 462 entrypoint = is_instance 463 ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance) 464 : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static); 465 break; 466 case Primitive::kPrimByte: 467 entrypoint = is_instance 468 ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance) 469 : (is_get ? kQuickGetByteStatic : kQuickSet8Static); 470 break; 471 case Primitive::kPrimShort: 472 entrypoint = is_instance 473 ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance) 474 : (is_get ? kQuickGetShortStatic : kQuickSet16Static); 475 break; 476 case Primitive::kPrimChar: 477 entrypoint = is_instance 478 ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance) 479 : (is_get ? kQuickGetCharStatic : kQuickSet16Static); 480 break; 481 case Primitive::kPrimInt: 482 case Primitive::kPrimFloat: 483 entrypoint = is_instance 484 ? (is_get ? kQuickGet32Instance : kQuickSet32Instance) 485 : (is_get ? kQuickGet32Static : kQuickSet32Static); 486 break; 487 case Primitive::kPrimNot: 488 entrypoint = is_instance 489 ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance) 490 : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic); 491 break; 492 case Primitive::kPrimLong: 493 case Primitive::kPrimDouble: 494 entrypoint = is_instance 495 ? (is_get ? kQuickGet64Instance : kQuickSet64Instance) 496 : (is_get ? kQuickGet64Static : kQuickSet64Static); 497 break; 498 default: 499 LOG(FATAL) << "Invalid type " << field_type; 500 } 501 InvokeRuntime(entrypoint, field_access, dex_pc, nullptr); 502 503 if (is_get && Primitive::IsFloatingPointType(field_type)) { 504 MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type); 505 } 506 } 507 508 void CodeGenerator::CreateLoadClassRuntimeCallLocationSummary(HLoadClass* cls, 509 Location runtime_type_index_location, 510 Location runtime_return_location) { 511 DCHECK_EQ(cls->GetLoadKind(), HLoadClass::LoadKind::kDexCacheViaMethod); 512 DCHECK_EQ(cls->InputCount(), 1u); 513 LocationSummary* locations = new (cls->GetBlock()->GetGraph()->GetArena()) LocationSummary( 514 cls, LocationSummary::kCallOnMainOnly); 515 locations->SetInAt(0, Location::NoLocation()); 516 locations->AddTemp(runtime_type_index_location); 517 locations->SetOut(runtime_return_location); 518 } 519 520 void CodeGenerator::GenerateLoadClassRuntimeCall(HLoadClass* cls) { 521 DCHECK_EQ(cls->GetLoadKind(), HLoadClass::LoadKind::kDexCacheViaMethod); 522 LocationSummary* locations = cls->GetLocations(); 523 MoveConstant(locations->GetTemp(0), cls->GetTypeIndex().index_); 524 if (cls->NeedsAccessCheck()) { 525 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 526 InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 527 } else if (cls->MustGenerateClinitCheck()) { 528 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 529 InvokeRuntime(kQuickInitializeStaticStorage, cls, cls->GetDexPc()); 530 } else { 531 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 532 InvokeRuntime(kQuickInitializeType, cls, cls->GetDexPc()); 533 } 534 } 535 536 void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const { 537 // The DCHECKS below check that a register is not specified twice in 538 // the summary. The out location can overlap with an input, so we need 539 // to special case it. 540 if (location.IsRegister()) { 541 DCHECK(is_out || !blocked_core_registers_[location.reg()]); 542 blocked_core_registers_[location.reg()] = true; 543 } else if (location.IsFpuRegister()) { 544 DCHECK(is_out || !blocked_fpu_registers_[location.reg()]); 545 blocked_fpu_registers_[location.reg()] = true; 546 } else if (location.IsFpuRegisterPair()) { 547 DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]); 548 blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true; 549 DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]); 550 blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true; 551 } else if (location.IsRegisterPair()) { 552 DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]); 553 blocked_core_registers_[location.AsRegisterPairLow<int>()] = true; 554 DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]); 555 blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true; 556 } 557 } 558 559 void CodeGenerator::AllocateLocations(HInstruction* instruction) { 560 instruction->Accept(GetLocationBuilder()); 561 DCHECK(CheckTypeConsistency(instruction)); 562 LocationSummary* locations = instruction->GetLocations(); 563 if (!instruction->IsSuspendCheckEntry()) { 564 if (locations != nullptr) { 565 if (locations->CanCall()) { 566 MarkNotLeaf(); 567 } else if (locations->Intrinsified() && 568 instruction->IsInvokeStaticOrDirect() && 569 !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) { 570 // A static method call that has been fully intrinsified, and cannot call on the slow 571 // path or refer to the current method directly, no longer needs current method. 572 return; 573 } 574 } 575 if (instruction->NeedsCurrentMethod()) { 576 SetRequiresCurrentMethod(); 577 } 578 } 579 } 580 581 void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const { 582 if (stats_ != nullptr) { 583 stats_->RecordStat(compilation_stat, count); 584 } 585 } 586 587 std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph, 588 InstructionSet instruction_set, 589 const InstructionSetFeatures& isa_features, 590 const CompilerOptions& compiler_options, 591 OptimizingCompilerStats* stats) { 592 ArenaAllocator* arena = graph->GetArena(); 593 switch (instruction_set) { 594 #ifdef ART_ENABLE_CODEGEN_arm 595 case kArm: 596 case kThumb2: { 597 if (kArmUseVIXL32) { 598 return std::unique_ptr<CodeGenerator>( 599 new (arena) arm::CodeGeneratorARMVIXL(graph, 600 *isa_features.AsArmInstructionSetFeatures(), 601 compiler_options, 602 stats)); 603 } else { 604 return std::unique_ptr<CodeGenerator>( 605 new (arena) arm::CodeGeneratorARM(graph, 606 *isa_features.AsArmInstructionSetFeatures(), 607 compiler_options, 608 stats)); 609 } 610 } 611 #endif 612 #ifdef ART_ENABLE_CODEGEN_arm64 613 case kArm64: { 614 return std::unique_ptr<CodeGenerator>( 615 new (arena) arm64::CodeGeneratorARM64(graph, 616 *isa_features.AsArm64InstructionSetFeatures(), 617 compiler_options, 618 stats)); 619 } 620 #endif 621 #ifdef ART_ENABLE_CODEGEN_mips 622 case kMips: { 623 return std::unique_ptr<CodeGenerator>( 624 new (arena) mips::CodeGeneratorMIPS(graph, 625 *isa_features.AsMipsInstructionSetFeatures(), 626 compiler_options, 627 stats)); 628 } 629 #endif 630 #ifdef ART_ENABLE_CODEGEN_mips64 631 case kMips64: { 632 return std::unique_ptr<CodeGenerator>( 633 new (arena) mips64::CodeGeneratorMIPS64(graph, 634 *isa_features.AsMips64InstructionSetFeatures(), 635 compiler_options, 636 stats)); 637 } 638 #endif 639 #ifdef ART_ENABLE_CODEGEN_x86 640 case kX86: { 641 return std::unique_ptr<CodeGenerator>( 642 new (arena) x86::CodeGeneratorX86(graph, 643 *isa_features.AsX86InstructionSetFeatures(), 644 compiler_options, 645 stats)); 646 } 647 #endif 648 #ifdef ART_ENABLE_CODEGEN_x86_64 649 case kX86_64: { 650 return std::unique_ptr<CodeGenerator>( 651 new (arena) x86_64::CodeGeneratorX86_64(graph, 652 *isa_features.AsX86_64InstructionSetFeatures(), 653 compiler_options, 654 stats)); 655 } 656 #endif 657 default: 658 return nullptr; 659 } 660 } 661 662 void CodeGenerator::ComputeStackMapAndMethodInfoSize(size_t* stack_map_size, 663 size_t* method_info_size) { 664 DCHECK(stack_map_size != nullptr); 665 DCHECK(method_info_size != nullptr); 666 *stack_map_size = stack_map_stream_.PrepareForFillIn(); 667 *method_info_size = stack_map_stream_.ComputeMethodInfoSize(); 668 } 669 670 static void CheckCovers(uint32_t dex_pc, 671 const HGraph& graph, 672 const CodeInfo& code_info, 673 const ArenaVector<HSuspendCheck*>& loop_headers, 674 ArenaVector<size_t>* covered) { 675 CodeInfoEncoding encoding = code_info.ExtractEncoding(); 676 for (size_t i = 0; i < loop_headers.size(); ++i) { 677 if (loop_headers[i]->GetDexPc() == dex_pc) { 678 if (graph.IsCompilingOsr()) { 679 DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc, encoding).IsValid()); 680 } 681 ++(*covered)[i]; 682 } 683 } 684 } 685 686 // Debug helper to ensure loop entries in compiled code are matched by 687 // dex branch instructions. 688 static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph, 689 const CodeInfo& code_info, 690 const DexFile::CodeItem& code_item) { 691 if (graph.HasTryCatch()) { 692 // One can write loops through try/catch, which we do not support for OSR anyway. 693 return; 694 } 695 ArenaVector<HSuspendCheck*> loop_headers(graph.GetArena()->Adapter(kArenaAllocMisc)); 696 for (HBasicBlock* block : graph.GetReversePostOrder()) { 697 if (block->IsLoopHeader()) { 698 HSuspendCheck* suspend_check = block->GetLoopInformation()->GetSuspendCheck(); 699 if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) { 700 loop_headers.push_back(suspend_check); 701 } 702 } 703 } 704 ArenaVector<size_t> covered(loop_headers.size(), 0, graph.GetArena()->Adapter(kArenaAllocMisc)); 705 const uint16_t* code_ptr = code_item.insns_; 706 const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_; 707 708 size_t dex_pc = 0; 709 while (code_ptr < code_end) { 710 const Instruction& instruction = *Instruction::At(code_ptr); 711 if (instruction.IsBranch()) { 712 uint32_t target = dex_pc + instruction.GetTargetOffset(); 713 CheckCovers(target, graph, code_info, loop_headers, &covered); 714 } else if (instruction.IsSwitch()) { 715 DexSwitchTable table(instruction, dex_pc); 716 uint16_t num_entries = table.GetNumEntries(); 717 size_t offset = table.GetFirstValueIndex(); 718 719 // Use a larger loop counter type to avoid overflow issues. 720 for (size_t i = 0; i < num_entries; ++i) { 721 // The target of the case. 722 uint32_t target = dex_pc + table.GetEntryAt(i + offset); 723 CheckCovers(target, graph, code_info, loop_headers, &covered); 724 } 725 } 726 dex_pc += instruction.SizeInCodeUnits(); 727 code_ptr += instruction.SizeInCodeUnits(); 728 } 729 730 for (size_t i = 0; i < covered.size(); ++i) { 731 DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent"; 732 } 733 } 734 735 void CodeGenerator::BuildStackMaps(MemoryRegion stack_map_region, 736 MemoryRegion method_info_region, 737 const DexFile::CodeItem& code_item) { 738 stack_map_stream_.FillInCodeInfo(stack_map_region); 739 stack_map_stream_.FillInMethodInfo(method_info_region); 740 if (kIsDebugBuild) { 741 CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(stack_map_region), code_item); 742 } 743 } 744 745 void CodeGenerator::RecordPcInfo(HInstruction* instruction, 746 uint32_t dex_pc, 747 SlowPathCode* slow_path) { 748 if (instruction != nullptr) { 749 // The code generated for some type conversions 750 // may call the runtime, thus normally requiring a subsequent 751 // call to this method. However, the method verifier does not 752 // produce PC information for certain instructions, which are 753 // considered "atomic" (they cannot join a GC). 754 // Therefore we do not currently record PC information for such 755 // instructions. As this may change later, we added this special 756 // case so that code generators may nevertheless call 757 // CodeGenerator::RecordPcInfo without triggering an error in 758 // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x") 759 // thereafter. 760 if (instruction->IsTypeConversion()) { 761 return; 762 } 763 if (instruction->IsRem()) { 764 Primitive::Type type = instruction->AsRem()->GetResultType(); 765 if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) { 766 return; 767 } 768 } 769 } 770 771 uint32_t outer_dex_pc = dex_pc; 772 uint32_t outer_environment_size = 0; 773 uint32_t inlining_depth = 0; 774 if (instruction != nullptr) { 775 for (HEnvironment* environment = instruction->GetEnvironment(); 776 environment != nullptr; 777 environment = environment->GetParent()) { 778 outer_dex_pc = environment->GetDexPc(); 779 outer_environment_size = environment->Size(); 780 if (environment != instruction->GetEnvironment()) { 781 inlining_depth++; 782 } 783 } 784 } 785 786 // Collect PC infos for the mapping table. 787 uint32_t native_pc = GetAssembler()->CodePosition(); 788 789 if (instruction == nullptr) { 790 // For stack overflow checks and native-debug-info entries without dex register 791 // mapping (i.e. start of basic block or start of slow path). 792 stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0); 793 stack_map_stream_.EndStackMapEntry(); 794 return; 795 } 796 LocationSummary* locations = instruction->GetLocations(); 797 798 uint32_t register_mask = locations->GetRegisterMask(); 799 DCHECK_EQ(register_mask & ~locations->GetLiveRegisters()->GetCoreRegisters(), 0u); 800 if (locations->OnlyCallsOnSlowPath()) { 801 // In case of slow path, we currently set the location of caller-save registers 802 // to register (instead of their stack location when pushed before the slow-path 803 // call). Therefore register_mask contains both callee-save and caller-save 804 // registers that hold objects. We must remove the spilled caller-save from the 805 // mask, since they will be overwritten by the callee. 806 uint32_t spills = GetSlowPathSpills(locations, /* core_registers */ true); 807 register_mask &= ~spills; 808 } else { 809 // The register mask must be a subset of callee-save registers. 810 DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask); 811 } 812 stack_map_stream_.BeginStackMapEntry(outer_dex_pc, 813 native_pc, 814 register_mask, 815 locations->GetStackMask(), 816 outer_environment_size, 817 inlining_depth); 818 819 HEnvironment* const environment = instruction->GetEnvironment(); 820 EmitEnvironment(environment, slow_path); 821 // Record invoke info, the common case for the trampoline is super and static invokes. Only 822 // record these to reduce oat file size. 823 if (kEnableDexLayoutOptimizations) { 824 if (environment != nullptr && 825 instruction->IsInvoke() && 826 instruction->IsInvokeStaticOrDirect()) { 827 HInvoke* const invoke = instruction->AsInvoke(); 828 stack_map_stream_.AddInvoke(invoke->GetInvokeType(), invoke->GetDexMethodIndex()); 829 } 830 } 831 stack_map_stream_.EndStackMapEntry(); 832 833 HLoopInformation* info = instruction->GetBlock()->GetLoopInformation(); 834 if (instruction->IsSuspendCheck() && 835 (info != nullptr) && 836 graph_->IsCompilingOsr() && 837 (inlining_depth == 0)) { 838 DCHECK_EQ(info->GetSuspendCheck(), instruction); 839 // We duplicate the stack map as a marker that this stack map can be an OSR entry. 840 // Duplicating it avoids having the runtime recognize and skip an OSR stack map. 841 DCHECK(info->IsIrreducible()); 842 stack_map_stream_.BeginStackMapEntry( 843 dex_pc, native_pc, register_mask, locations->GetStackMask(), outer_environment_size, 0); 844 EmitEnvironment(instruction->GetEnvironment(), slow_path); 845 stack_map_stream_.EndStackMapEntry(); 846 if (kIsDebugBuild) { 847 for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) { 848 HInstruction* in_environment = environment->GetInstructionAt(i); 849 if (in_environment != nullptr) { 850 DCHECK(in_environment->IsPhi() || in_environment->IsConstant()); 851 Location location = environment->GetLocationAt(i); 852 DCHECK(location.IsStackSlot() || 853 location.IsDoubleStackSlot() || 854 location.IsConstant() || 855 location.IsInvalid()); 856 if (location.IsStackSlot() || location.IsDoubleStackSlot()) { 857 DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize())); 858 } 859 } 860 } 861 } 862 } else if (kIsDebugBuild) { 863 // Ensure stack maps are unique, by checking that the native pc in the stack map 864 // last emitted is different than the native pc of the stack map just emitted. 865 size_t number_of_stack_maps = stack_map_stream_.GetNumberOfStackMaps(); 866 if (number_of_stack_maps > 1) { 867 DCHECK_NE(stack_map_stream_.GetStackMap(number_of_stack_maps - 1).native_pc_code_offset, 868 stack_map_stream_.GetStackMap(number_of_stack_maps - 2).native_pc_code_offset); 869 } 870 } 871 } 872 873 bool CodeGenerator::HasStackMapAtCurrentPc() { 874 uint32_t pc = GetAssembler()->CodeSize(); 875 size_t count = stack_map_stream_.GetNumberOfStackMaps(); 876 if (count == 0) { 877 return false; 878 } 879 CodeOffset native_pc_offset = stack_map_stream_.GetStackMap(count - 1).native_pc_code_offset; 880 return (native_pc_offset.Uint32Value(GetInstructionSet()) == pc); 881 } 882 883 void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction, 884 uint32_t dex_pc, 885 SlowPathCode* slow_path) { 886 if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) { 887 if (HasStackMapAtCurrentPc()) { 888 // Ensure that we do not collide with the stack map of the previous instruction. 889 GenerateNop(); 890 } 891 RecordPcInfo(instruction, dex_pc, slow_path); 892 } 893 } 894 895 void CodeGenerator::RecordCatchBlockInfo() { 896 ArenaAllocator* arena = graph_->GetArena(); 897 898 for (HBasicBlock* block : *block_order_) { 899 if (!block->IsCatchBlock()) { 900 continue; 901 } 902 903 uint32_t dex_pc = block->GetDexPc(); 904 uint32_t num_vregs = graph_->GetNumberOfVRegs(); 905 uint32_t inlining_depth = 0; // Inlining of catch blocks is not supported at the moment. 906 uint32_t native_pc = GetAddressOf(block); 907 uint32_t register_mask = 0; // Not used. 908 909 // The stack mask is not used, so we leave it empty. 910 ArenaBitVector* stack_mask = 911 ArenaBitVector::Create(arena, 0, /* expandable */ true, kArenaAllocCodeGenerator); 912 913 stack_map_stream_.BeginStackMapEntry(dex_pc, 914 native_pc, 915 register_mask, 916 stack_mask, 917 num_vregs, 918 inlining_depth); 919 920 HInstruction* current_phi = block->GetFirstPhi(); 921 for (size_t vreg = 0; vreg < num_vregs; ++vreg) { 922 while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) { 923 HInstruction* next_phi = current_phi->GetNext(); 924 DCHECK(next_phi == nullptr || 925 current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber()) 926 << "Phis need to be sorted by vreg number to keep this a linear-time loop."; 927 current_phi = next_phi; 928 } 929 930 if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) { 931 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0); 932 } else { 933 Location location = current_phi->GetLiveInterval()->ToLocation(); 934 switch (location.GetKind()) { 935 case Location::kStackSlot: { 936 stack_map_stream_.AddDexRegisterEntry( 937 DexRegisterLocation::Kind::kInStack, location.GetStackIndex()); 938 break; 939 } 940 case Location::kDoubleStackSlot: { 941 stack_map_stream_.AddDexRegisterEntry( 942 DexRegisterLocation::Kind::kInStack, location.GetStackIndex()); 943 stack_map_stream_.AddDexRegisterEntry( 944 DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize)); 945 ++vreg; 946 DCHECK_LT(vreg, num_vregs); 947 break; 948 } 949 default: { 950 // All catch phis must be allocated to a stack slot. 951 LOG(FATAL) << "Unexpected kind " << location.GetKind(); 952 UNREACHABLE(); 953 } 954 } 955 } 956 } 957 958 stack_map_stream_.EndStackMapEntry(); 959 } 960 } 961 962 void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) { 963 if (environment == nullptr) return; 964 965 if (environment->GetParent() != nullptr) { 966 // We emit the parent environment first. 967 EmitEnvironment(environment->GetParent(), slow_path); 968 stack_map_stream_.BeginInlineInfoEntry(environment->GetMethod(), 969 environment->GetDexPc(), 970 environment->Size(), 971 &graph_->GetDexFile()); 972 } 973 974 // Walk over the environment, and record the location of dex registers. 975 for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) { 976 HInstruction* current = environment->GetInstructionAt(i); 977 if (current == nullptr) { 978 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0); 979 continue; 980 } 981 982 Location location = environment->GetLocationAt(i); 983 switch (location.GetKind()) { 984 case Location::kConstant: { 985 DCHECK_EQ(current, location.GetConstant()); 986 if (current->IsLongConstant()) { 987 int64_t value = current->AsLongConstant()->GetValue(); 988 stack_map_stream_.AddDexRegisterEntry( 989 DexRegisterLocation::Kind::kConstant, Low32Bits(value)); 990 stack_map_stream_.AddDexRegisterEntry( 991 DexRegisterLocation::Kind::kConstant, High32Bits(value)); 992 ++i; 993 DCHECK_LT(i, environment_size); 994 } else if (current->IsDoubleConstant()) { 995 int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue()); 996 stack_map_stream_.AddDexRegisterEntry( 997 DexRegisterLocation::Kind::kConstant, Low32Bits(value)); 998 stack_map_stream_.AddDexRegisterEntry( 999 DexRegisterLocation::Kind::kConstant, High32Bits(value)); 1000 ++i; 1001 DCHECK_LT(i, environment_size); 1002 } else if (current->IsIntConstant()) { 1003 int32_t value = current->AsIntConstant()->GetValue(); 1004 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value); 1005 } else if (current->IsNullConstant()) { 1006 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0); 1007 } else { 1008 DCHECK(current->IsFloatConstant()) << current->DebugName(); 1009 int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue()); 1010 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value); 1011 } 1012 break; 1013 } 1014 1015 case Location::kStackSlot: { 1016 stack_map_stream_.AddDexRegisterEntry( 1017 DexRegisterLocation::Kind::kInStack, location.GetStackIndex()); 1018 break; 1019 } 1020 1021 case Location::kDoubleStackSlot: { 1022 stack_map_stream_.AddDexRegisterEntry( 1023 DexRegisterLocation::Kind::kInStack, location.GetStackIndex()); 1024 stack_map_stream_.AddDexRegisterEntry( 1025 DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize)); 1026 ++i; 1027 DCHECK_LT(i, environment_size); 1028 break; 1029 } 1030 1031 case Location::kRegister : { 1032 int id = location.reg(); 1033 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) { 1034 uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id); 1035 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset); 1036 if (current->GetType() == Primitive::kPrimLong) { 1037 stack_map_stream_.AddDexRegisterEntry( 1038 DexRegisterLocation::Kind::kInStack, offset + kVRegSize); 1039 ++i; 1040 DCHECK_LT(i, environment_size); 1041 } 1042 } else { 1043 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id); 1044 if (current->GetType() == Primitive::kPrimLong) { 1045 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id); 1046 ++i; 1047 DCHECK_LT(i, environment_size); 1048 } 1049 } 1050 break; 1051 } 1052 1053 case Location::kFpuRegister : { 1054 int id = location.reg(); 1055 if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) { 1056 uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id); 1057 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset); 1058 if (current->GetType() == Primitive::kPrimDouble) { 1059 stack_map_stream_.AddDexRegisterEntry( 1060 DexRegisterLocation::Kind::kInStack, offset + kVRegSize); 1061 ++i; 1062 DCHECK_LT(i, environment_size); 1063 } 1064 } else { 1065 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id); 1066 if (current->GetType() == Primitive::kPrimDouble) { 1067 stack_map_stream_.AddDexRegisterEntry( 1068 DexRegisterLocation::Kind::kInFpuRegisterHigh, id); 1069 ++i; 1070 DCHECK_LT(i, environment_size); 1071 } 1072 } 1073 break; 1074 } 1075 1076 case Location::kFpuRegisterPair : { 1077 int low = location.low(); 1078 int high = location.high(); 1079 if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) { 1080 uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low); 1081 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset); 1082 } else { 1083 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low); 1084 } 1085 if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) { 1086 uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high); 1087 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset); 1088 ++i; 1089 } else { 1090 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high); 1091 ++i; 1092 } 1093 DCHECK_LT(i, environment_size); 1094 break; 1095 } 1096 1097 case Location::kRegisterPair : { 1098 int low = location.low(); 1099 int high = location.high(); 1100 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) { 1101 uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low); 1102 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset); 1103 } else { 1104 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low); 1105 } 1106 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) { 1107 uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high); 1108 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset); 1109 } else { 1110 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high); 1111 } 1112 ++i; 1113 DCHECK_LT(i, environment_size); 1114 break; 1115 } 1116 1117 case Location::kInvalid: { 1118 stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0); 1119 break; 1120 } 1121 1122 default: 1123 LOG(FATAL) << "Unexpected kind " << location.GetKind(); 1124 } 1125 } 1126 1127 if (environment->GetParent() != nullptr) { 1128 stack_map_stream_.EndInlineInfoEntry(); 1129 } 1130 } 1131 1132 bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) { 1133 HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves(); 1134 1135 return (first_next_not_move != nullptr) 1136 && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0)); 1137 } 1138 1139 void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) { 1140 if (!compiler_options_.GetImplicitNullChecks()) { 1141 return; 1142 } 1143 1144 // If we are from a static path don't record the pc as we can't throw NPE. 1145 // NB: having the checks here makes the code much less verbose in the arch 1146 // specific code generators. 1147 if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) { 1148 return; 1149 } 1150 1151 if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) { 1152 return; 1153 } 1154 1155 // Find the first previous instruction which is not a move. 1156 HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves(); 1157 1158 // If the instruction is a null check it means that `instr` is the first user 1159 // and needs to record the pc. 1160 if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) { 1161 HNullCheck* null_check = first_prev_not_move->AsNullCheck(); 1162 // TODO: The parallel moves modify the environment. Their changes need to be 1163 // reverted otherwise the stack maps at the throw point will not be correct. 1164 RecordPcInfo(null_check, null_check->GetDexPc()); 1165 } 1166 } 1167 1168 LocationSummary* CodeGenerator::CreateThrowingSlowPathLocations(HInstruction* instruction, 1169 RegisterSet caller_saves) { 1170 // Note: Using kNoCall allows the method to be treated as leaf (and eliminate the 1171 // HSuspendCheck from entry block). However, it will still get a valid stack frame 1172 // because the HNullCheck needs an environment. 1173 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 1174 // When throwing from a try block, we may need to retrieve dalvik registers from 1175 // physical registers and we also need to set up stack mask for GC. This is 1176 // implicitly achieved by passing kCallOnSlowPath to the LocationSummary. 1177 bool can_throw_into_catch_block = instruction->CanThrowIntoCatchBlock(); 1178 if (can_throw_into_catch_block) { 1179 call_kind = LocationSummary::kCallOnSlowPath; 1180 } 1181 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 1182 if (can_throw_into_catch_block && compiler_options_.GetImplicitNullChecks()) { 1183 locations->SetCustomSlowPathCallerSaves(caller_saves); // Default: no caller-save registers. 1184 } 1185 DCHECK(!instruction->HasUses()); 1186 return locations; 1187 } 1188 1189 void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) { 1190 if (compiler_options_.GetImplicitNullChecks()) { 1191 MaybeRecordStat(kImplicitNullCheckGenerated); 1192 GenerateImplicitNullCheck(instruction); 1193 } else { 1194 MaybeRecordStat(kExplicitNullCheckGenerated); 1195 GenerateExplicitNullCheck(instruction); 1196 } 1197 } 1198 1199 void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const { 1200 LocationSummary* locations = suspend_check->GetLocations(); 1201 HBasicBlock* block = suspend_check->GetBlock(); 1202 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check); 1203 DCHECK(block->IsLoopHeader()); 1204 1205 for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { 1206 HInstruction* current = it.Current(); 1207 LiveInterval* interval = current->GetLiveInterval(); 1208 // We only need to clear bits of loop phis containing objects and allocated in register. 1209 // Loop phis allocated on stack already have the object in the stack. 1210 if (current->GetType() == Primitive::kPrimNot 1211 && interval->HasRegister() 1212 && interval->HasSpillSlot()) { 1213 locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize); 1214 } 1215 } 1216 } 1217 1218 void CodeGenerator::EmitParallelMoves(Location from1, 1219 Location to1, 1220 Primitive::Type type1, 1221 Location from2, 1222 Location to2, 1223 Primitive::Type type2) { 1224 HParallelMove parallel_move(GetGraph()->GetArena()); 1225 parallel_move.AddMove(from1, to1, type1, nullptr); 1226 parallel_move.AddMove(from2, to2, type2, nullptr); 1227 GetMoveResolver()->EmitNativeCode(¶llel_move); 1228 } 1229 1230 void CodeGenerator::ValidateInvokeRuntime(QuickEntrypointEnum entrypoint, 1231 HInstruction* instruction, 1232 SlowPathCode* slow_path) { 1233 // Ensure that the call kind indication given to the register allocator is 1234 // coherent with the runtime call generated. 1235 if (slow_path == nullptr) { 1236 DCHECK(instruction->GetLocations()->WillCall()) 1237 << "instruction->DebugName()=" << instruction->DebugName(); 1238 } else { 1239 DCHECK(instruction->GetLocations()->CallsOnSlowPath() || slow_path->IsFatal()) 1240 << "instruction->DebugName()=" << instruction->DebugName() 1241 << " slow_path->GetDescription()=" << slow_path->GetDescription(); 1242 } 1243 1244 // Check that the GC side effect is set when required. 1245 // TODO: Reverse EntrypointCanTriggerGC 1246 if (EntrypointCanTriggerGC(entrypoint)) { 1247 if (slow_path == nullptr) { 1248 DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC())) 1249 << "instruction->DebugName()=" << instruction->DebugName() 1250 << " instruction->GetSideEffects().ToString()=" 1251 << instruction->GetSideEffects().ToString(); 1252 } else { 1253 DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) || 1254 // When (non-Baker) read barriers are enabled, some instructions 1255 // use a slow path to emit a read barrier, which does not trigger 1256 // GC. 1257 (kEmitCompilerReadBarrier && 1258 !kUseBakerReadBarrier && 1259 (instruction->IsInstanceFieldGet() || 1260 instruction->IsStaticFieldGet() || 1261 instruction->IsArrayGet() || 1262 instruction->IsLoadClass() || 1263 instruction->IsLoadString() || 1264 instruction->IsInstanceOf() || 1265 instruction->IsCheckCast() || 1266 (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified())))) 1267 << "instruction->DebugName()=" << instruction->DebugName() 1268 << " instruction->GetSideEffects().ToString()=" 1269 << instruction->GetSideEffects().ToString() 1270 << " slow_path->GetDescription()=" << slow_path->GetDescription(); 1271 } 1272 } else { 1273 // The GC side effect is not required for the instruction. But the instruction might still have 1274 // it, for example if it calls other entrypoints requiring it. 1275 } 1276 1277 // Check the coherency of leaf information. 1278 DCHECK(instruction->IsSuspendCheck() 1279 || ((slow_path != nullptr) && slow_path->IsFatal()) 1280 || instruction->GetLocations()->CanCall() 1281 || !IsLeafMethod()) 1282 << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : ""); 1283 } 1284 1285 void CodeGenerator::ValidateInvokeRuntimeWithoutRecordingPcInfo(HInstruction* instruction, 1286 SlowPathCode* slow_path) { 1287 DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath()) 1288 << "instruction->DebugName()=" << instruction->DebugName() 1289 << " slow_path->GetDescription()=" << slow_path->GetDescription(); 1290 // Only the Baker read barrier marking slow path used by certains 1291 // instructions is expected to invoke the runtime without recording 1292 // PC-related information. 1293 DCHECK(kUseBakerReadBarrier); 1294 DCHECK(instruction->IsInstanceFieldGet() || 1295 instruction->IsStaticFieldGet() || 1296 instruction->IsArrayGet() || 1297 instruction->IsArraySet() || 1298 instruction->IsLoadClass() || 1299 instruction->IsLoadString() || 1300 instruction->IsInstanceOf() || 1301 instruction->IsCheckCast() || 1302 (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()) || 1303 (instruction->IsInvokeStaticOrDirect() && instruction->GetLocations()->Intrinsified())) 1304 << "instruction->DebugName()=" << instruction->DebugName() 1305 << " slow_path->GetDescription()=" << slow_path->GetDescription(); 1306 } 1307 1308 void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 1309 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 1310 1311 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 1312 for (uint32_t i : LowToHighBits(core_spills)) { 1313 // If the register holds an object, update the stack mask. 1314 if (locations->RegisterContainsObject(i)) { 1315 locations->SetStackBit(stack_offset / kVRegSize); 1316 } 1317 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 1318 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 1319 saved_core_stack_offsets_[i] = stack_offset; 1320 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 1321 } 1322 1323 const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 1324 for (uint32_t i : LowToHighBits(fp_spills)) { 1325 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 1326 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 1327 saved_fpu_stack_offsets_[i] = stack_offset; 1328 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i); 1329 } 1330 } 1331 1332 void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 1333 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 1334 1335 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 1336 for (uint32_t i : LowToHighBits(core_spills)) { 1337 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 1338 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 1339 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 1340 } 1341 1342 const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 1343 for (uint32_t i : LowToHighBits(fp_spills)) { 1344 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 1345 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 1346 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i); 1347 } 1348 } 1349 1350 void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) { 1351 // Check to see if we have known failures that will cause us to have to bail out 1352 // to the runtime, and just generate the runtime call directly. 1353 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); 1354 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); 1355 1356 // The positions must be non-negative. 1357 if ((src_pos != nullptr && src_pos->GetValue() < 0) || 1358 (dest_pos != nullptr && dest_pos->GetValue() < 0)) { 1359 // We will have to fail anyways. 1360 return; 1361 } 1362 1363 // The length must be >= 0. 1364 HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); 1365 if (length != nullptr) { 1366 int32_t len = length->GetValue(); 1367 if (len < 0) { 1368 // Just call as normal. 1369 return; 1370 } 1371 } 1372 1373 SystemArrayCopyOptimizations optimizations(invoke); 1374 1375 if (optimizations.GetDestinationIsSource()) { 1376 if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) { 1377 // We only support backward copying if source and destination are the same. 1378 return; 1379 } 1380 } 1381 1382 if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) { 1383 // We currently don't intrinsify primitive copying. 1384 return; 1385 } 1386 1387 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena(); 1388 LocationSummary* locations = new (allocator) LocationSummary(invoke, 1389 LocationSummary::kCallOnSlowPath, 1390 kIntrinsified); 1391 // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length). 1392 locations->SetInAt(0, Location::RequiresRegister()); 1393 locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); 1394 locations->SetInAt(2, Location::RequiresRegister()); 1395 locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3))); 1396 locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4))); 1397 1398 locations->AddTemp(Location::RequiresRegister()); 1399 locations->AddTemp(Location::RequiresRegister()); 1400 locations->AddTemp(Location::RequiresRegister()); 1401 } 1402 1403 uint32_t CodeGenerator::GetReferenceSlowFlagOffset() const { 1404 ScopedObjectAccess soa(Thread::Current()); 1405 mirror::Class* klass = mirror::Reference::GetJavaLangRefReference(); 1406 DCHECK(klass->IsInitialized()); 1407 return klass->GetSlowPathFlagOffset().Uint32Value(); 1408 } 1409 1410 uint32_t CodeGenerator::GetReferenceDisableFlagOffset() const { 1411 ScopedObjectAccess soa(Thread::Current()); 1412 mirror::Class* klass = mirror::Reference::GetJavaLangRefReference(); 1413 DCHECK(klass->IsInitialized()); 1414 return klass->GetDisableIntrinsicFlagOffset().Uint32Value(); 1415 } 1416 1417 void CodeGenerator::EmitJitRoots(uint8_t* code, 1418 Handle<mirror::ObjectArray<mirror::Object>> roots, 1419 const uint8_t* roots_data) { 1420 DCHECK_EQ(static_cast<size_t>(roots->GetLength()), GetNumberOfJitRoots()); 1421 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 1422 size_t index = 0; 1423 for (auto& entry : jit_string_roots_) { 1424 // Update the `roots` with the string, and replace the address temporarily 1425 // stored to the index in the table. 1426 uint64_t address = entry.second; 1427 roots->Set(index, reinterpret_cast<StackReference<mirror::String>*>(address)->AsMirrorPtr()); 1428 DCHECK(roots->Get(index) != nullptr); 1429 entry.second = index; 1430 // Ensure the string is strongly interned. This is a requirement on how the JIT 1431 // handles strings. b/32995596 1432 class_linker->GetInternTable()->InternStrong( 1433 reinterpret_cast<mirror::String*>(roots->Get(index))); 1434 ++index; 1435 } 1436 for (auto& entry : jit_class_roots_) { 1437 // Update the `roots` with the class, and replace the address temporarily 1438 // stored to the index in the table. 1439 uint64_t address = entry.second; 1440 roots->Set(index, reinterpret_cast<StackReference<mirror::Class>*>(address)->AsMirrorPtr()); 1441 DCHECK(roots->Get(index) != nullptr); 1442 entry.second = index; 1443 ++index; 1444 } 1445 EmitJitRootPatches(code, roots_data); 1446 } 1447 1448 QuickEntrypointEnum CodeGenerator::GetArrayAllocationEntrypoint(Handle<mirror::Class> array_klass) { 1449 ScopedObjectAccess soa(Thread::Current()); 1450 if (array_klass == nullptr) { 1451 // This can only happen for non-primitive arrays, as primitive arrays can always 1452 // be resolved. 1453 return kQuickAllocArrayResolved32; 1454 } 1455 1456 switch (array_klass->GetComponentSize()) { 1457 case 1: return kQuickAllocArrayResolved8; 1458 case 2: return kQuickAllocArrayResolved16; 1459 case 4: return kQuickAllocArrayResolved32; 1460 case 8: return kQuickAllocArrayResolved64; 1461 } 1462 LOG(FATAL) << "Unreachable"; 1463 return kQuickAllocArrayResolved; 1464 } 1465 1466 } // namespace art 1467
