1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "code_generator_arm64.h" 18 19 #include "arch/arm64/instruction_set_features_arm64.h" 20 #include "art_method.h" 21 #include "code_generator_utils.h" 22 #include "compiled_method.h" 23 #include "entrypoints/quick/quick_entrypoints.h" 24 #include "entrypoints/quick/quick_entrypoints_enum.h" 25 #include "gc/accounting/card_table.h" 26 #include "intrinsics.h" 27 #include "intrinsics_arm64.h" 28 #include "mirror/array-inl.h" 29 #include "mirror/class-inl.h" 30 #include "offsets.h" 31 #include "thread.h" 32 #include "utils/arm64/assembler_arm64.h" 33 #include "utils/assembler.h" 34 #include "utils/stack_checks.h" 35 36 37 using namespace vixl; // NOLINT(build/namespaces) 38 39 #ifdef __ 40 #error "ARM64 Codegen VIXL macro-assembler macro already defined." 41 #endif 42 43 namespace art { 44 45 template<class MirrorType> 46 class GcRoot; 47 48 namespace arm64 { 49 50 using helpers::CPURegisterFrom; 51 using helpers::DRegisterFrom; 52 using helpers::FPRegisterFrom; 53 using helpers::HeapOperand; 54 using helpers::HeapOperandFrom; 55 using helpers::InputCPURegisterAt; 56 using helpers::InputFPRegisterAt; 57 using helpers::InputRegisterAt; 58 using helpers::InputOperandAt; 59 using helpers::Int64ConstantFrom; 60 using helpers::LocationFrom; 61 using helpers::OperandFromMemOperand; 62 using helpers::OutputCPURegister; 63 using helpers::OutputFPRegister; 64 using helpers::OutputRegister; 65 using helpers::RegisterFrom; 66 using helpers::StackOperandFrom; 67 using helpers::VIXLRegCodeFromART; 68 using helpers::WRegisterFrom; 69 using helpers::XRegisterFrom; 70 using helpers::ARM64EncodableConstantOrRegister; 71 using helpers::ArtVixlRegCodeCoherentForRegSet; 72 73 static constexpr int kCurrentMethodStackOffset = 0; 74 // The compare/jump sequence will generate about (1.5 * num_entries + 3) instructions. While jump 75 // table version generates 7 instructions and num_entries literals. Compare/jump sequence will 76 // generates less code/data with a small num_entries. 77 static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 78 79 inline Condition ARM64Condition(IfCondition cond) { 80 switch (cond) { 81 case kCondEQ: return eq; 82 case kCondNE: return ne; 83 case kCondLT: return lt; 84 case kCondLE: return le; 85 case kCondGT: return gt; 86 case kCondGE: return ge; 87 case kCondB: return lo; 88 case kCondBE: return ls; 89 case kCondA: return hi; 90 case kCondAE: return hs; 91 } 92 LOG(FATAL) << "Unreachable"; 93 UNREACHABLE(); 94 } 95 96 inline Condition ARM64FPCondition(IfCondition cond, bool gt_bias) { 97 // The ARM64 condition codes can express all the necessary branches, see the 98 // "Meaning (floating-point)" column in the table C1-1 in the ARMv8 reference manual. 99 // There is no dex instruction or HIR that would need the missing conditions 100 // "equal or unordered" or "not equal". 101 switch (cond) { 102 case kCondEQ: return eq; 103 case kCondNE: return ne /* unordered */; 104 case kCondLT: return gt_bias ? cc : lt /* unordered */; 105 case kCondLE: return gt_bias ? ls : le /* unordered */; 106 case kCondGT: return gt_bias ? hi /* unordered */ : gt; 107 case kCondGE: return gt_bias ? cs /* unordered */ : ge; 108 default: 109 LOG(FATAL) << "UNREACHABLE"; 110 UNREACHABLE(); 111 } 112 } 113 114 Location ARM64ReturnLocation(Primitive::Type return_type) { 115 // Note that in practice, `LocationFrom(x0)` and `LocationFrom(w0)` create the 116 // same Location object, and so do `LocationFrom(d0)` and `LocationFrom(s0)`, 117 // but we use the exact registers for clarity. 118 if (return_type == Primitive::kPrimFloat) { 119 return LocationFrom(s0); 120 } else if (return_type == Primitive::kPrimDouble) { 121 return LocationFrom(d0); 122 } else if (return_type == Primitive::kPrimLong) { 123 return LocationFrom(x0); 124 } else if (return_type == Primitive::kPrimVoid) { 125 return Location::NoLocation(); 126 } else { 127 return LocationFrom(w0); 128 } 129 } 130 131 Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) { 132 return ARM64ReturnLocation(return_type); 133 } 134 135 #define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()-> 136 #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value() 137 138 // Calculate memory accessing operand for save/restore live registers. 139 static void SaveRestoreLiveRegistersHelper(CodeGenerator* codegen, 140 RegisterSet* register_set, 141 int64_t spill_offset, 142 bool is_save) { 143 DCHECK(ArtVixlRegCodeCoherentForRegSet(register_set->GetCoreRegisters(), 144 codegen->GetNumberOfCoreRegisters(), 145 register_set->GetFloatingPointRegisters(), 146 codegen->GetNumberOfFloatingPointRegisters())); 147 148 CPURegList core_list = CPURegList(CPURegister::kRegister, kXRegSize, 149 register_set->GetCoreRegisters() & (~callee_saved_core_registers.list())); 150 CPURegList fp_list = CPURegList(CPURegister::kFPRegister, kDRegSize, 151 register_set->GetFloatingPointRegisters() & (~callee_saved_fp_registers.list())); 152 153 MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler(); 154 UseScratchRegisterScope temps(masm); 155 156 Register base = masm->StackPointer(); 157 int64_t core_spill_size = core_list.TotalSizeInBytes(); 158 int64_t fp_spill_size = fp_list.TotalSizeInBytes(); 159 int64_t reg_size = kXRegSizeInBytes; 160 int64_t max_ls_pair_offset = spill_offset + core_spill_size + fp_spill_size - 2 * reg_size; 161 uint32_t ls_access_size = WhichPowerOf2(reg_size); 162 if (((core_list.Count() > 1) || (fp_list.Count() > 1)) && 163 !masm->IsImmLSPair(max_ls_pair_offset, ls_access_size)) { 164 // If the offset does not fit in the instruction's immediate field, use an alternate register 165 // to compute the base address(float point registers spill base address). 166 Register new_base = temps.AcquireSameSizeAs(base); 167 __ Add(new_base, base, Operand(spill_offset + core_spill_size)); 168 base = new_base; 169 spill_offset = -core_spill_size; 170 int64_t new_max_ls_pair_offset = fp_spill_size - 2 * reg_size; 171 DCHECK(masm->IsImmLSPair(spill_offset, ls_access_size)); 172 DCHECK(masm->IsImmLSPair(new_max_ls_pair_offset, ls_access_size)); 173 } 174 175 if (is_save) { 176 __ StoreCPURegList(core_list, MemOperand(base, spill_offset)); 177 __ StoreCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size)); 178 } else { 179 __ LoadCPURegList(core_list, MemOperand(base, spill_offset)); 180 __ LoadCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size)); 181 } 182 } 183 184 void SlowPathCodeARM64::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 185 RegisterSet* register_set = locations->GetLiveRegisters(); 186 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 187 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 188 if (!codegen->IsCoreCalleeSaveRegister(i) && register_set->ContainsCoreRegister(i)) { 189 // If the register holds an object, update the stack mask. 190 if (locations->RegisterContainsObject(i)) { 191 locations->SetStackBit(stack_offset / kVRegSize); 192 } 193 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 194 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 195 saved_core_stack_offsets_[i] = stack_offset; 196 stack_offset += kXRegSizeInBytes; 197 } 198 } 199 200 for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) { 201 if (!codegen->IsFloatingPointCalleeSaveRegister(i) && 202 register_set->ContainsFloatingPointRegister(i)) { 203 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 204 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 205 saved_fpu_stack_offsets_[i] = stack_offset; 206 stack_offset += kDRegSizeInBytes; 207 } 208 } 209 210 SaveRestoreLiveRegistersHelper(codegen, register_set, 211 codegen->GetFirstRegisterSlotInSlowPath(), true /* is_save */); 212 } 213 214 void SlowPathCodeARM64::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 215 RegisterSet* register_set = locations->GetLiveRegisters(); 216 SaveRestoreLiveRegistersHelper(codegen, register_set, 217 codegen->GetFirstRegisterSlotInSlowPath(), false /* is_save */); 218 } 219 220 class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 { 221 public: 222 explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction) : SlowPathCodeARM64(instruction) {} 223 224 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 225 LocationSummary* locations = instruction_->GetLocations(); 226 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 227 228 __ Bind(GetEntryLabel()); 229 if (instruction_->CanThrowIntoCatchBlock()) { 230 // Live registers will be restored in the catch block if caught. 231 SaveLiveRegisters(codegen, instruction_->GetLocations()); 232 } 233 // We're moving two locations to locations that could overlap, so we need a parallel 234 // move resolver. 235 InvokeRuntimeCallingConvention calling_convention; 236 codegen->EmitParallelMoves( 237 locations->InAt(0), LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimInt, 238 locations->InAt(1), LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt); 239 arm64_codegen->InvokeRuntime( 240 QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this); 241 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 242 } 243 244 bool IsFatal() const OVERRIDE { return true; } 245 246 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; } 247 248 private: 249 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64); 250 }; 251 252 class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 { 253 public: 254 explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : SlowPathCodeARM64(instruction) {} 255 256 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 257 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 258 __ Bind(GetEntryLabel()); 259 if (instruction_->CanThrowIntoCatchBlock()) { 260 // Live registers will be restored in the catch block if caught. 261 SaveLiveRegisters(codegen, instruction_->GetLocations()); 262 } 263 arm64_codegen->InvokeRuntime( 264 QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this); 265 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 266 } 267 268 bool IsFatal() const OVERRIDE { return true; } 269 270 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; } 271 272 private: 273 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64); 274 }; 275 276 class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { 277 public: 278 LoadClassSlowPathARM64(HLoadClass* cls, 279 HInstruction* at, 280 uint32_t dex_pc, 281 bool do_clinit) 282 : SlowPathCodeARM64(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 283 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 284 } 285 286 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 287 LocationSummary* locations = at_->GetLocations(); 288 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 289 290 __ Bind(GetEntryLabel()); 291 SaveLiveRegisters(codegen, locations); 292 293 InvokeRuntimeCallingConvention calling_convention; 294 __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex()); 295 int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage) 296 : QUICK_ENTRY_POINT(pInitializeType); 297 arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_, this); 298 if (do_clinit_) { 299 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 300 } else { 301 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 302 } 303 304 // Move the class to the desired location. 305 Location out = locations->Out(); 306 if (out.IsValid()) { 307 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 308 Primitive::Type type = at_->GetType(); 309 arm64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type); 310 } 311 312 RestoreLiveRegisters(codegen, locations); 313 __ B(GetExitLabel()); 314 } 315 316 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM64"; } 317 318 private: 319 // The class this slow path will load. 320 HLoadClass* const cls_; 321 322 // The instruction where this slow path is happening. 323 // (Might be the load class or an initialization check). 324 HInstruction* const at_; 325 326 // The dex PC of `at_`. 327 const uint32_t dex_pc_; 328 329 // Whether to initialize the class. 330 const bool do_clinit_; 331 332 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64); 333 }; 334 335 class LoadStringSlowPathARM64 : public SlowPathCodeARM64 { 336 public: 337 explicit LoadStringSlowPathARM64(HLoadString* instruction) : SlowPathCodeARM64(instruction) {} 338 339 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 340 LocationSummary* locations = instruction_->GetLocations(); 341 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 342 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 343 344 __ Bind(GetEntryLabel()); 345 SaveLiveRegisters(codegen, locations); 346 347 InvokeRuntimeCallingConvention calling_convention; 348 const uint32_t string_index = instruction_->AsLoadString()->GetStringIndex(); 349 __ Mov(calling_convention.GetRegisterAt(0).W(), string_index); 350 arm64_codegen->InvokeRuntime( 351 QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc(), this); 352 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 353 Primitive::Type type = instruction_->GetType(); 354 arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type); 355 356 RestoreLiveRegisters(codegen, locations); 357 __ B(GetExitLabel()); 358 } 359 360 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM64"; } 361 362 private: 363 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64); 364 }; 365 366 class NullCheckSlowPathARM64 : public SlowPathCodeARM64 { 367 public: 368 explicit NullCheckSlowPathARM64(HNullCheck* instr) : SlowPathCodeARM64(instr) {} 369 370 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 371 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 372 __ Bind(GetEntryLabel()); 373 if (instruction_->CanThrowIntoCatchBlock()) { 374 // Live registers will be restored in the catch block if caught. 375 SaveLiveRegisters(codegen, instruction_->GetLocations()); 376 } 377 arm64_codegen->InvokeRuntime( 378 QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this); 379 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 380 } 381 382 bool IsFatal() const OVERRIDE { return true; } 383 384 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; } 385 386 private: 387 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64); 388 }; 389 390 class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 { 391 public: 392 SuspendCheckSlowPathARM64(HSuspendCheck* instruction, HBasicBlock* successor) 393 : SlowPathCodeARM64(instruction), successor_(successor) {} 394 395 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 396 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 397 __ Bind(GetEntryLabel()); 398 SaveLiveRegisters(codegen, instruction_->GetLocations()); 399 arm64_codegen->InvokeRuntime( 400 QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc(), this); 401 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 402 RestoreLiveRegisters(codegen, instruction_->GetLocations()); 403 if (successor_ == nullptr) { 404 __ B(GetReturnLabel()); 405 } else { 406 __ B(arm64_codegen->GetLabelOf(successor_)); 407 } 408 } 409 410 vixl::Label* GetReturnLabel() { 411 DCHECK(successor_ == nullptr); 412 return &return_label_; 413 } 414 415 HBasicBlock* GetSuccessor() const { 416 return successor_; 417 } 418 419 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM64"; } 420 421 private: 422 // If not null, the block to branch to after the suspend check. 423 HBasicBlock* const successor_; 424 425 // If `successor_` is null, the label to branch to after the suspend check. 426 vixl::Label return_label_; 427 428 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64); 429 }; 430 431 class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 { 432 public: 433 TypeCheckSlowPathARM64(HInstruction* instruction, bool is_fatal) 434 : SlowPathCodeARM64(instruction), is_fatal_(is_fatal) {} 435 436 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 437 LocationSummary* locations = instruction_->GetLocations(); 438 Location class_to_check = locations->InAt(1); 439 Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0) 440 : locations->Out(); 441 DCHECK(instruction_->IsCheckCast() 442 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 443 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 444 uint32_t dex_pc = instruction_->GetDexPc(); 445 446 __ Bind(GetEntryLabel()); 447 448 if (!is_fatal_) { 449 SaveLiveRegisters(codegen, locations); 450 } 451 452 // We're moving two locations to locations that could overlap, so we need a parallel 453 // move resolver. 454 InvokeRuntimeCallingConvention calling_convention; 455 codegen->EmitParallelMoves( 456 class_to_check, LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimNot, 457 object_class, LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimNot); 458 459 if (instruction_->IsInstanceOf()) { 460 arm64_codegen->InvokeRuntime( 461 QUICK_ENTRY_POINT(pInstanceofNonTrivial), instruction_, dex_pc, this); 462 CheckEntrypointTypes<kQuickInstanceofNonTrivial, uint32_t, 463 const mirror::Class*, const mirror::Class*>(); 464 Primitive::Type ret_type = instruction_->GetType(); 465 Location ret_loc = calling_convention.GetReturnLocation(ret_type); 466 arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type); 467 } else { 468 DCHECK(instruction_->IsCheckCast()); 469 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), instruction_, dex_pc, this); 470 CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>(); 471 } 472 473 if (!is_fatal_) { 474 RestoreLiveRegisters(codegen, locations); 475 __ B(GetExitLabel()); 476 } 477 } 478 479 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM64"; } 480 bool IsFatal() const { return is_fatal_; } 481 482 private: 483 const bool is_fatal_; 484 485 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64); 486 }; 487 488 class DeoptimizationSlowPathARM64 : public SlowPathCodeARM64 { 489 public: 490 explicit DeoptimizationSlowPathARM64(HDeoptimize* instruction) 491 : SlowPathCodeARM64(instruction) {} 492 493 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 494 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 495 __ Bind(GetEntryLabel()); 496 SaveLiveRegisters(codegen, instruction_->GetLocations()); 497 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), 498 instruction_, 499 instruction_->GetDexPc(), 500 this); 501 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 502 } 503 504 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM64"; } 505 506 private: 507 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM64); 508 }; 509 510 class ArraySetSlowPathARM64 : public SlowPathCodeARM64 { 511 public: 512 explicit ArraySetSlowPathARM64(HInstruction* instruction) : SlowPathCodeARM64(instruction) {} 513 514 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 515 LocationSummary* locations = instruction_->GetLocations(); 516 __ Bind(GetEntryLabel()); 517 SaveLiveRegisters(codegen, locations); 518 519 InvokeRuntimeCallingConvention calling_convention; 520 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 521 parallel_move.AddMove( 522 locations->InAt(0), 523 LocationFrom(calling_convention.GetRegisterAt(0)), 524 Primitive::kPrimNot, 525 nullptr); 526 parallel_move.AddMove( 527 locations->InAt(1), 528 LocationFrom(calling_convention.GetRegisterAt(1)), 529 Primitive::kPrimInt, 530 nullptr); 531 parallel_move.AddMove( 532 locations->InAt(2), 533 LocationFrom(calling_convention.GetRegisterAt(2)), 534 Primitive::kPrimNot, 535 nullptr); 536 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 537 538 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 539 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject), 540 instruction_, 541 instruction_->GetDexPc(), 542 this); 543 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 544 RestoreLiveRegisters(codegen, locations); 545 __ B(GetExitLabel()); 546 } 547 548 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM64"; } 549 550 private: 551 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM64); 552 }; 553 554 void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { 555 uint32_t num_entries = switch_instr_->GetNumEntries(); 556 DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold); 557 558 // We are about to use the assembler to place literals directly. Make sure we have enough 559 // underlying code buffer and we have generated the jump table with right size. 560 CodeBufferCheckScope scope(codegen->GetVIXLAssembler(), num_entries * sizeof(int32_t), 561 CodeBufferCheckScope::kCheck, CodeBufferCheckScope::kExactSize); 562 563 __ Bind(&table_start_); 564 const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors(); 565 for (uint32_t i = 0; i < num_entries; i++) { 566 vixl::Label* target_label = codegen->GetLabelOf(successors[i]); 567 DCHECK(target_label->IsBound()); 568 ptrdiff_t jump_offset = target_label->location() - table_start_.location(); 569 DCHECK_GT(jump_offset, std::numeric_limits<int32_t>::min()); 570 DCHECK_LE(jump_offset, std::numeric_limits<int32_t>::max()); 571 Literal<int32_t> literal(jump_offset); 572 __ place(&literal); 573 } 574 } 575 576 // Slow path marking an object during a read barrier. 577 class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { 578 public: 579 ReadBarrierMarkSlowPathARM64(HInstruction* instruction, Location out, Location obj) 580 : SlowPathCodeARM64(instruction), out_(out), obj_(obj) { 581 DCHECK(kEmitCompilerReadBarrier); 582 } 583 584 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM64"; } 585 586 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 587 LocationSummary* locations = instruction_->GetLocations(); 588 Primitive::Type type = Primitive::kPrimNot; 589 DCHECK(locations->CanCall()); 590 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg())); 591 DCHECK(instruction_->IsInstanceFieldGet() || 592 instruction_->IsStaticFieldGet() || 593 instruction_->IsArrayGet() || 594 instruction_->IsLoadClass() || 595 instruction_->IsLoadString() || 596 instruction_->IsInstanceOf() || 597 instruction_->IsCheckCast()) 598 << "Unexpected instruction in read barrier marking slow path: " 599 << instruction_->DebugName(); 600 601 __ Bind(GetEntryLabel()); 602 SaveLiveRegisters(codegen, locations); 603 604 InvokeRuntimeCallingConvention calling_convention; 605 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 606 arm64_codegen->MoveLocation(LocationFrom(calling_convention.GetRegisterAt(0)), obj_, type); 607 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), 608 instruction_, 609 instruction_->GetDexPc(), 610 this); 611 CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); 612 arm64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); 613 614 RestoreLiveRegisters(codegen, locations); 615 __ B(GetExitLabel()); 616 } 617 618 private: 619 const Location out_; 620 const Location obj_; 621 622 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64); 623 }; 624 625 // Slow path generating a read barrier for a heap reference. 626 class ReadBarrierForHeapReferenceSlowPathARM64 : public SlowPathCodeARM64 { 627 public: 628 ReadBarrierForHeapReferenceSlowPathARM64(HInstruction* instruction, 629 Location out, 630 Location ref, 631 Location obj, 632 uint32_t offset, 633 Location index) 634 : SlowPathCodeARM64(instruction), 635 out_(out), 636 ref_(ref), 637 obj_(obj), 638 offset_(offset), 639 index_(index) { 640 DCHECK(kEmitCompilerReadBarrier); 641 // If `obj` is equal to `out` or `ref`, it means the initial object 642 // has been overwritten by (or after) the heap object reference load 643 // to be instrumented, e.g.: 644 // 645 // __ Ldr(out, HeapOperand(out, class_offset); 646 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 647 // 648 // In that case, we have lost the information about the original 649 // object, and the emitted read barrier cannot work properly. 650 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 651 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 652 } 653 654 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 655 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 656 LocationSummary* locations = instruction_->GetLocations(); 657 Primitive::Type type = Primitive::kPrimNot; 658 DCHECK(locations->CanCall()); 659 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg())); 660 DCHECK(!instruction_->IsInvoke() || 661 (instruction_->IsInvokeStaticOrDirect() && 662 instruction_->GetLocations()->Intrinsified())) 663 << "Unexpected instruction in read barrier for heap reference slow path: " 664 << instruction_->DebugName(); 665 // The read barrier instrumentation does not support the 666 // HArm64IntermediateAddress instruction yet. 667 DCHECK(!(instruction_->IsArrayGet() && 668 instruction_->AsArrayGet()->GetArray()->IsArm64IntermediateAddress())); 669 670 __ Bind(GetEntryLabel()); 671 672 SaveLiveRegisters(codegen, locations); 673 674 // We may have to change the index's value, but as `index_` is a 675 // constant member (like other "inputs" of this slow path), 676 // introduce a copy of it, `index`. 677 Location index = index_; 678 if (index_.IsValid()) { 679 // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject. 680 if (instruction_->IsArrayGet()) { 681 // Compute the actual memory offset and store it in `index`. 682 Register index_reg = RegisterFrom(index_, Primitive::kPrimInt); 683 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_.reg())); 684 if (codegen->IsCoreCalleeSaveRegister(index_.reg())) { 685 // We are about to change the value of `index_reg` (see the 686 // calls to vixl::MacroAssembler::Lsl and 687 // vixl::MacroAssembler::Mov below), but it has 688 // not been saved by the previous call to 689 // art::SlowPathCode::SaveLiveRegisters, as it is a 690 // callee-save register -- 691 // art::SlowPathCode::SaveLiveRegisters does not consider 692 // callee-save registers, as it has been designed with the 693 // assumption that callee-save registers are supposed to be 694 // handled by the called function. So, as a callee-save 695 // register, `index_reg` _would_ eventually be saved onto 696 // the stack, but it would be too late: we would have 697 // changed its value earlier. Therefore, we manually save 698 // it here into another freely available register, 699 // `free_reg`, chosen of course among the caller-save 700 // registers (as a callee-save `free_reg` register would 701 // exhibit the same problem). 702 // 703 // Note we could have requested a temporary register from 704 // the register allocator instead; but we prefer not to, as 705 // this is a slow path, and we know we can find a 706 // caller-save register that is available. 707 Register free_reg = FindAvailableCallerSaveRegister(codegen); 708 __ Mov(free_reg.W(), index_reg); 709 index_reg = free_reg; 710 index = LocationFrom(index_reg); 711 } else { 712 // The initial register stored in `index_` has already been 713 // saved in the call to art::SlowPathCode::SaveLiveRegisters 714 // (as it is not a callee-save register), so we can freely 715 // use it. 716 } 717 // Shifting the index value contained in `index_reg` by the scale 718 // factor (2) cannot overflow in practice, as the runtime is 719 // unable to allocate object arrays with a size larger than 720 // 2^26 - 1 (that is, 2^28 - 4 bytes). 721 __ Lsl(index_reg, index_reg, Primitive::ComponentSizeShift(type)); 722 static_assert( 723 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 724 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 725 __ Add(index_reg, index_reg, Operand(offset_)); 726 } else { 727 DCHECK(instruction_->IsInvoke()); 728 DCHECK(instruction_->GetLocations()->Intrinsified()); 729 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 730 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 731 << instruction_->AsInvoke()->GetIntrinsic(); 732 DCHECK_EQ(offset_, 0U); 733 DCHECK(index_.IsRegisterPair()); 734 // UnsafeGet's offset location is a register pair, the low 735 // part contains the correct offset. 736 index = index_.ToLow(); 737 } 738 } 739 740 // We're moving two or three locations to locations that could 741 // overlap, so we need a parallel move resolver. 742 InvokeRuntimeCallingConvention calling_convention; 743 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 744 parallel_move.AddMove(ref_, 745 LocationFrom(calling_convention.GetRegisterAt(0)), 746 type, 747 nullptr); 748 parallel_move.AddMove(obj_, 749 LocationFrom(calling_convention.GetRegisterAt(1)), 750 type, 751 nullptr); 752 if (index.IsValid()) { 753 parallel_move.AddMove(index, 754 LocationFrom(calling_convention.GetRegisterAt(2)), 755 Primitive::kPrimInt, 756 nullptr); 757 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 758 } else { 759 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 760 arm64_codegen->MoveConstant(LocationFrom(calling_convention.GetRegisterAt(2)), offset_); 761 } 762 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow), 763 instruction_, 764 instruction_->GetDexPc(), 765 this); 766 CheckEntrypointTypes< 767 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 768 arm64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); 769 770 RestoreLiveRegisters(codegen, locations); 771 772 __ B(GetExitLabel()); 773 } 774 775 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM64"; } 776 777 private: 778 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 779 size_t ref = static_cast<int>(XRegisterFrom(ref_).code()); 780 size_t obj = static_cast<int>(XRegisterFrom(obj_).code()); 781 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 782 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 783 return Register(VIXLRegCodeFromART(i), kXRegSize); 784 } 785 } 786 // We shall never fail to find a free caller-save register, as 787 // there are more than two core caller-save registers on ARM64 788 // (meaning it is possible to find one which is different from 789 // `ref` and `obj`). 790 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 791 LOG(FATAL) << "Could not find a free register"; 792 UNREACHABLE(); 793 } 794 795 const Location out_; 796 const Location ref_; 797 const Location obj_; 798 const uint32_t offset_; 799 // An additional location containing an index to an array. 800 // Only used for HArrayGet and the UnsafeGetObject & 801 // UnsafeGetObjectVolatile intrinsics. 802 const Location index_; 803 804 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM64); 805 }; 806 807 // Slow path generating a read barrier for a GC root. 808 class ReadBarrierForRootSlowPathARM64 : public SlowPathCodeARM64 { 809 public: 810 ReadBarrierForRootSlowPathARM64(HInstruction* instruction, Location out, Location root) 811 : SlowPathCodeARM64(instruction), out_(out), root_(root) { 812 DCHECK(kEmitCompilerReadBarrier); 813 } 814 815 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 816 LocationSummary* locations = instruction_->GetLocations(); 817 Primitive::Type type = Primitive::kPrimNot; 818 DCHECK(locations->CanCall()); 819 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg())); 820 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 821 << "Unexpected instruction in read barrier for GC root slow path: " 822 << instruction_->DebugName(); 823 824 __ Bind(GetEntryLabel()); 825 SaveLiveRegisters(codegen, locations); 826 827 InvokeRuntimeCallingConvention calling_convention; 828 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); 829 // The argument of the ReadBarrierForRootSlow is not a managed 830 // reference (`mirror::Object*`), but a `GcRoot<mirror::Object>*`; 831 // thus we need a 64-bit move here, and we cannot use 832 // 833 // arm64_codegen->MoveLocation( 834 // LocationFrom(calling_convention.GetRegisterAt(0)), 835 // root_, 836 // type); 837 // 838 // which would emit a 32-bit move, as `type` is a (32-bit wide) 839 // reference type (`Primitive::kPrimNot`). 840 __ Mov(calling_convention.GetRegisterAt(0), XRegisterFrom(out_)); 841 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow), 842 instruction_, 843 instruction_->GetDexPc(), 844 this); 845 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 846 arm64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); 847 848 RestoreLiveRegisters(codegen, locations); 849 __ B(GetExitLabel()); 850 } 851 852 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM64"; } 853 854 private: 855 const Location out_; 856 const Location root_; 857 858 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM64); 859 }; 860 861 #undef __ 862 863 Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) { 864 Location next_location; 865 if (type == Primitive::kPrimVoid) { 866 LOG(FATAL) << "Unreachable type " << type; 867 } 868 869 if (Primitive::IsFloatingPointType(type) && 870 (float_index_ < calling_convention.GetNumberOfFpuRegisters())) { 871 next_location = LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++)); 872 } else if (!Primitive::IsFloatingPointType(type) && 873 (gp_index_ < calling_convention.GetNumberOfRegisters())) { 874 next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++)); 875 } else { 876 size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_); 877 next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset) 878 : Location::StackSlot(stack_offset); 879 } 880 881 // Space on the stack is reserved for all arguments. 882 stack_index_ += Primitive::Is64BitType(type) ? 2 : 1; 883 return next_location; 884 } 885 886 Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const { 887 return LocationFrom(kArtMethodRegister); 888 } 889 890 CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, 891 const Arm64InstructionSetFeatures& isa_features, 892 const CompilerOptions& compiler_options, 893 OptimizingCompilerStats* stats) 894 : CodeGenerator(graph, 895 kNumberOfAllocatableRegisters, 896 kNumberOfAllocatableFPRegisters, 897 kNumberOfAllocatableRegisterPairs, 898 callee_saved_core_registers.list(), 899 callee_saved_fp_registers.list(), 900 compiler_options, 901 stats), 902 block_labels_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 903 jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 904 location_builder_(graph, this), 905 instruction_visitor_(graph, this), 906 move_resolver_(graph->GetArena(), this), 907 assembler_(graph->GetArena()), 908 isa_features_(isa_features), 909 uint32_literals_(std::less<uint32_t>(), 910 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 911 uint64_literals_(std::less<uint64_t>(), 912 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 913 method_patches_(MethodReferenceComparator(), 914 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 915 call_patches_(MethodReferenceComparator(), 916 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 917 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 918 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 919 boot_image_string_patches_(StringReferenceValueComparator(), 920 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 921 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 922 boot_image_address_patches_(std::less<uint32_t>(), 923 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 924 // Save the link register (containing the return address) to mimic Quick. 925 AddAllocatedRegister(LocationFrom(lr)); 926 } 927 928 #define __ GetVIXLAssembler()-> 929 930 void CodeGeneratorARM64::EmitJumpTables() { 931 for (auto&& jump_table : jump_tables_) { 932 jump_table->EmitTable(this); 933 } 934 } 935 936 void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) { 937 EmitJumpTables(); 938 // Ensure we emit the literal pool. 939 __ FinalizeCode(); 940 941 CodeGenerator::Finalize(allocator); 942 } 943 944 void ParallelMoveResolverARM64::PrepareForEmitNativeCode() { 945 // Note: There are 6 kinds of moves: 946 // 1. constant -> GPR/FPR (non-cycle) 947 // 2. constant -> stack (non-cycle) 948 // 3. GPR/FPR -> GPR/FPR 949 // 4. GPR/FPR -> stack 950 // 5. stack -> GPR/FPR 951 // 6. stack -> stack (non-cycle) 952 // Case 1, 2 and 6 should never be included in a dependency cycle on ARM64. For case 3, 4, and 5 953 // VIXL uses at most 1 GPR. VIXL has 2 GPR and 1 FPR temps, and there should be no intersecting 954 // cycles on ARM64, so we always have 1 GPR and 1 FPR available VIXL temps to resolve the 955 // dependency. 956 vixl_temps_.Open(GetVIXLAssembler()); 957 } 958 959 void ParallelMoveResolverARM64::FinishEmitNativeCode() { 960 vixl_temps_.Close(); 961 } 962 963 Location ParallelMoveResolverARM64::AllocateScratchLocationFor(Location::Kind kind) { 964 DCHECK(kind == Location::kRegister || kind == Location::kFpuRegister || 965 kind == Location::kStackSlot || kind == Location::kDoubleStackSlot); 966 kind = (kind == Location::kFpuRegister) ? Location::kFpuRegister : Location::kRegister; 967 Location scratch = GetScratchLocation(kind); 968 if (!scratch.Equals(Location::NoLocation())) { 969 return scratch; 970 } 971 // Allocate from VIXL temp registers. 972 if (kind == Location::kRegister) { 973 scratch = LocationFrom(vixl_temps_.AcquireX()); 974 } else { 975 DCHECK(kind == Location::kFpuRegister); 976 scratch = LocationFrom(vixl_temps_.AcquireD()); 977 } 978 AddScratchLocation(scratch); 979 return scratch; 980 } 981 982 void ParallelMoveResolverARM64::FreeScratchLocation(Location loc) { 983 if (loc.IsRegister()) { 984 vixl_temps_.Release(XRegisterFrom(loc)); 985 } else { 986 DCHECK(loc.IsFpuRegister()); 987 vixl_temps_.Release(DRegisterFrom(loc)); 988 } 989 RemoveScratchLocation(loc); 990 } 991 992 void ParallelMoveResolverARM64::EmitMove(size_t index) { 993 MoveOperands* move = moves_[index]; 994 codegen_->MoveLocation(move->GetDestination(), move->GetSource(), Primitive::kPrimVoid); 995 } 996 997 void CodeGeneratorARM64::GenerateFrameEntry() { 998 MacroAssembler* masm = GetVIXLAssembler(); 999 BlockPoolsScope block_pools(masm); 1000 __ Bind(&frame_entry_label_); 1001 1002 bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod(); 1003 if (do_overflow_check) { 1004 UseScratchRegisterScope temps(masm); 1005 Register temp = temps.AcquireX(); 1006 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1007 __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64))); 1008 __ Ldr(wzr, MemOperand(temp, 0)); 1009 RecordPcInfo(nullptr, 0); 1010 } 1011 1012 if (!HasEmptyFrame()) { 1013 int frame_size = GetFrameSize(); 1014 // Stack layout: 1015 // sp[frame_size - 8] : lr. 1016 // ... : other preserved core registers. 1017 // ... : other preserved fp registers. 1018 // ... : reserved frame space. 1019 // sp[0] : current method. 1020 __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex)); 1021 GetAssembler()->cfi().AdjustCFAOffset(frame_size); 1022 GetAssembler()->SpillRegisters(GetFramePreservedCoreRegisters(), 1023 frame_size - GetCoreSpillSize()); 1024 GetAssembler()->SpillRegisters(GetFramePreservedFPRegisters(), 1025 frame_size - FrameEntrySpillSize()); 1026 } 1027 } 1028 1029 void CodeGeneratorARM64::GenerateFrameExit() { 1030 BlockPoolsScope block_pools(GetVIXLAssembler()); 1031 GetAssembler()->cfi().RememberState(); 1032 if (!HasEmptyFrame()) { 1033 int frame_size = GetFrameSize(); 1034 GetAssembler()->UnspillRegisters(GetFramePreservedFPRegisters(), 1035 frame_size - FrameEntrySpillSize()); 1036 GetAssembler()->UnspillRegisters(GetFramePreservedCoreRegisters(), 1037 frame_size - GetCoreSpillSize()); 1038 __ Drop(frame_size); 1039 GetAssembler()->cfi().AdjustCFAOffset(-frame_size); 1040 } 1041 __ Ret(); 1042 GetAssembler()->cfi().RestoreState(); 1043 GetAssembler()->cfi().DefCFAOffset(GetFrameSize()); 1044 } 1045 1046 vixl::CPURegList CodeGeneratorARM64::GetFramePreservedCoreRegisters() const { 1047 DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spill_mask_, GetNumberOfCoreRegisters(), 0, 0)); 1048 return vixl::CPURegList(vixl::CPURegister::kRegister, vixl::kXRegSize, 1049 core_spill_mask_); 1050 } 1051 1052 vixl::CPURegList CodeGeneratorARM64::GetFramePreservedFPRegisters() const { 1053 DCHECK(ArtVixlRegCodeCoherentForRegSet(0, 0, fpu_spill_mask_, 1054 GetNumberOfFloatingPointRegisters())); 1055 return vixl::CPURegList(vixl::CPURegister::kFPRegister, vixl::kDRegSize, 1056 fpu_spill_mask_); 1057 } 1058 1059 void CodeGeneratorARM64::Bind(HBasicBlock* block) { 1060 __ Bind(GetLabelOf(block)); 1061 } 1062 1063 void CodeGeneratorARM64::MoveConstant(Location location, int32_t value) { 1064 DCHECK(location.IsRegister()); 1065 __ Mov(RegisterFrom(location, Primitive::kPrimInt), value); 1066 } 1067 1068 void CodeGeneratorARM64::AddLocationAsTemp(Location location, LocationSummary* locations) { 1069 if (location.IsRegister()) { 1070 locations->AddTemp(location); 1071 } else { 1072 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1073 } 1074 } 1075 1076 void CodeGeneratorARM64::MarkGCCard(Register object, Register value, bool value_can_be_null) { 1077 UseScratchRegisterScope temps(GetVIXLAssembler()); 1078 Register card = temps.AcquireX(); 1079 Register temp = temps.AcquireW(); // Index within the CardTable - 32bit. 1080 vixl::Label done; 1081 if (value_can_be_null) { 1082 __ Cbz(value, &done); 1083 } 1084 __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64WordSize>().Int32Value())); 1085 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 1086 __ Strb(card, MemOperand(card, temp.X())); 1087 if (value_can_be_null) { 1088 __ Bind(&done); 1089 } 1090 } 1091 1092 void CodeGeneratorARM64::SetupBlockedRegisters() const { 1093 // Blocked core registers: 1094 // lr : Runtime reserved. 1095 // tr : Runtime reserved. 1096 // xSuspend : Runtime reserved. TODO: Unblock this when the runtime stops using it. 1097 // ip1 : VIXL core temp. 1098 // ip0 : VIXL core temp. 1099 // 1100 // Blocked fp registers: 1101 // d31 : VIXL fp temp. 1102 CPURegList reserved_core_registers = vixl_reserved_core_registers; 1103 reserved_core_registers.Combine(runtime_reserved_core_registers); 1104 while (!reserved_core_registers.IsEmpty()) { 1105 blocked_core_registers_[reserved_core_registers.PopLowestIndex().code()] = true; 1106 } 1107 1108 CPURegList reserved_fp_registers = vixl_reserved_fp_registers; 1109 while (!reserved_fp_registers.IsEmpty()) { 1110 blocked_fpu_registers_[reserved_fp_registers.PopLowestIndex().code()] = true; 1111 } 1112 1113 if (GetGraph()->IsDebuggable()) { 1114 // Stubs do not save callee-save floating point registers. If the graph 1115 // is debuggable, we need to deal with these registers differently. For 1116 // now, just block them. 1117 CPURegList reserved_fp_registers_debuggable = callee_saved_fp_registers; 1118 while (!reserved_fp_registers_debuggable.IsEmpty()) { 1119 blocked_fpu_registers_[reserved_fp_registers_debuggable.PopLowestIndex().code()] = true; 1120 } 1121 } 1122 } 1123 1124 size_t CodeGeneratorARM64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1125 Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize); 1126 __ Str(reg, MemOperand(sp, stack_index)); 1127 return kArm64WordSize; 1128 } 1129 1130 size_t CodeGeneratorARM64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1131 Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize); 1132 __ Ldr(reg, MemOperand(sp, stack_index)); 1133 return kArm64WordSize; 1134 } 1135 1136 size_t CodeGeneratorARM64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1137 FPRegister reg = FPRegister(reg_id, kDRegSize); 1138 __ Str(reg, MemOperand(sp, stack_index)); 1139 return kArm64WordSize; 1140 } 1141 1142 size_t CodeGeneratorARM64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1143 FPRegister reg = FPRegister(reg_id, kDRegSize); 1144 __ Ldr(reg, MemOperand(sp, stack_index)); 1145 return kArm64WordSize; 1146 } 1147 1148 void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const { 1149 stream << XRegister(reg); 1150 } 1151 1152 void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1153 stream << DRegister(reg); 1154 } 1155 1156 void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) { 1157 if (constant->IsIntConstant()) { 1158 __ Mov(Register(destination), constant->AsIntConstant()->GetValue()); 1159 } else if (constant->IsLongConstant()) { 1160 __ Mov(Register(destination), constant->AsLongConstant()->GetValue()); 1161 } else if (constant->IsNullConstant()) { 1162 __ Mov(Register(destination), 0); 1163 } else if (constant->IsFloatConstant()) { 1164 __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue()); 1165 } else { 1166 DCHECK(constant->IsDoubleConstant()); 1167 __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue()); 1168 } 1169 } 1170 1171 1172 static bool CoherentConstantAndType(Location constant, Primitive::Type type) { 1173 DCHECK(constant.IsConstant()); 1174 HConstant* cst = constant.GetConstant(); 1175 return (cst->IsIntConstant() && type == Primitive::kPrimInt) || 1176 // Null is mapped to a core W register, which we associate with kPrimInt. 1177 (cst->IsNullConstant() && type == Primitive::kPrimInt) || 1178 (cst->IsLongConstant() && type == Primitive::kPrimLong) || 1179 (cst->IsFloatConstant() && type == Primitive::kPrimFloat) || 1180 (cst->IsDoubleConstant() && type == Primitive::kPrimDouble); 1181 } 1182 1183 void CodeGeneratorARM64::MoveLocation(Location destination, 1184 Location source, 1185 Primitive::Type dst_type) { 1186 if (source.Equals(destination)) { 1187 return; 1188 } 1189 1190 // A valid move can always be inferred from the destination and source 1191 // locations. When moving from and to a register, the argument type can be 1192 // used to generate 32bit instead of 64bit moves. In debug mode we also 1193 // checks the coherency of the locations and the type. 1194 bool unspecified_type = (dst_type == Primitive::kPrimVoid); 1195 1196 if (destination.IsRegister() || destination.IsFpuRegister()) { 1197 if (unspecified_type) { 1198 HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr; 1199 if (source.IsStackSlot() || 1200 (src_cst != nullptr && (src_cst->IsIntConstant() 1201 || src_cst->IsFloatConstant() 1202 || src_cst->IsNullConstant()))) { 1203 // For stack slots and 32bit constants, a 64bit type is appropriate. 1204 dst_type = destination.IsRegister() ? Primitive::kPrimInt : Primitive::kPrimFloat; 1205 } else { 1206 // If the source is a double stack slot or a 64bit constant, a 64bit 1207 // type is appropriate. Else the source is a register, and since the 1208 // type has not been specified, we chose a 64bit type to force a 64bit 1209 // move. 1210 dst_type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble; 1211 } 1212 } 1213 DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(dst_type)) || 1214 (destination.IsRegister() && !Primitive::IsFloatingPointType(dst_type))); 1215 CPURegister dst = CPURegisterFrom(destination, dst_type); 1216 if (source.IsStackSlot() || source.IsDoubleStackSlot()) { 1217 DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot()); 1218 __ Ldr(dst, StackOperandFrom(source)); 1219 } else if (source.IsConstant()) { 1220 DCHECK(CoherentConstantAndType(source, dst_type)); 1221 MoveConstant(dst, source.GetConstant()); 1222 } else if (source.IsRegister()) { 1223 if (destination.IsRegister()) { 1224 __ Mov(Register(dst), RegisterFrom(source, dst_type)); 1225 } else { 1226 DCHECK(destination.IsFpuRegister()); 1227 Primitive::Type source_type = Primitive::Is64BitType(dst_type) 1228 ? Primitive::kPrimLong 1229 : Primitive::kPrimInt; 1230 __ Fmov(FPRegisterFrom(destination, dst_type), RegisterFrom(source, source_type)); 1231 } 1232 } else { 1233 DCHECK(source.IsFpuRegister()); 1234 if (destination.IsRegister()) { 1235 Primitive::Type source_type = Primitive::Is64BitType(dst_type) 1236 ? Primitive::kPrimDouble 1237 : Primitive::kPrimFloat; 1238 __ Fmov(RegisterFrom(destination, dst_type), FPRegisterFrom(source, source_type)); 1239 } else { 1240 DCHECK(destination.IsFpuRegister()); 1241 __ Fmov(FPRegister(dst), FPRegisterFrom(source, dst_type)); 1242 } 1243 } 1244 } else { // The destination is not a register. It must be a stack slot. 1245 DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot()); 1246 if (source.IsRegister() || source.IsFpuRegister()) { 1247 if (unspecified_type) { 1248 if (source.IsRegister()) { 1249 dst_type = destination.IsStackSlot() ? Primitive::kPrimInt : Primitive::kPrimLong; 1250 } else { 1251 dst_type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble; 1252 } 1253 } 1254 DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(dst_type)) && 1255 (source.IsFpuRegister() == Primitive::IsFloatingPointType(dst_type))); 1256 __ Str(CPURegisterFrom(source, dst_type), StackOperandFrom(destination)); 1257 } else if (source.IsConstant()) { 1258 DCHECK(unspecified_type || CoherentConstantAndType(source, dst_type)) 1259 << source << " " << dst_type; 1260 UseScratchRegisterScope temps(GetVIXLAssembler()); 1261 HConstant* src_cst = source.GetConstant(); 1262 CPURegister temp; 1263 if (src_cst->IsIntConstant() || src_cst->IsNullConstant()) { 1264 temp = temps.AcquireW(); 1265 } else if (src_cst->IsLongConstant()) { 1266 temp = temps.AcquireX(); 1267 } else if (src_cst->IsFloatConstant()) { 1268 temp = temps.AcquireS(); 1269 } else { 1270 DCHECK(src_cst->IsDoubleConstant()); 1271 temp = temps.AcquireD(); 1272 } 1273 MoveConstant(temp, src_cst); 1274 __ Str(temp, StackOperandFrom(destination)); 1275 } else { 1276 DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot()); 1277 DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot()); 1278 UseScratchRegisterScope temps(GetVIXLAssembler()); 1279 // There is generally less pressure on FP registers. 1280 FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS(); 1281 __ Ldr(temp, StackOperandFrom(source)); 1282 __ Str(temp, StackOperandFrom(destination)); 1283 } 1284 } 1285 } 1286 1287 void CodeGeneratorARM64::Load(Primitive::Type type, 1288 CPURegister dst, 1289 const MemOperand& src) { 1290 switch (type) { 1291 case Primitive::kPrimBoolean: 1292 __ Ldrb(Register(dst), src); 1293 break; 1294 case Primitive::kPrimByte: 1295 __ Ldrsb(Register(dst), src); 1296 break; 1297 case Primitive::kPrimShort: 1298 __ Ldrsh(Register(dst), src); 1299 break; 1300 case Primitive::kPrimChar: 1301 __ Ldrh(Register(dst), src); 1302 break; 1303 case Primitive::kPrimInt: 1304 case Primitive::kPrimNot: 1305 case Primitive::kPrimLong: 1306 case Primitive::kPrimFloat: 1307 case Primitive::kPrimDouble: 1308 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); 1309 __ Ldr(dst, src); 1310 break; 1311 case Primitive::kPrimVoid: 1312 LOG(FATAL) << "Unreachable type " << type; 1313 } 1314 } 1315 1316 void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction, 1317 CPURegister dst, 1318 const MemOperand& src, 1319 bool needs_null_check) { 1320 MacroAssembler* masm = GetVIXLAssembler(); 1321 BlockPoolsScope block_pools(masm); 1322 UseScratchRegisterScope temps(masm); 1323 Register temp_base = temps.AcquireX(); 1324 Primitive::Type type = instruction->GetType(); 1325 1326 DCHECK(!src.IsPreIndex()); 1327 DCHECK(!src.IsPostIndex()); 1328 1329 // TODO(vixl): Let the MacroAssembler handle MemOperand. 1330 __ Add(temp_base, src.base(), OperandFromMemOperand(src)); 1331 MemOperand base = MemOperand(temp_base); 1332 switch (type) { 1333 case Primitive::kPrimBoolean: 1334 __ Ldarb(Register(dst), base); 1335 if (needs_null_check) { 1336 MaybeRecordImplicitNullCheck(instruction); 1337 } 1338 break; 1339 case Primitive::kPrimByte: 1340 __ Ldarb(Register(dst), base); 1341 if (needs_null_check) { 1342 MaybeRecordImplicitNullCheck(instruction); 1343 } 1344 __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte); 1345 break; 1346 case Primitive::kPrimChar: 1347 __ Ldarh(Register(dst), base); 1348 if (needs_null_check) { 1349 MaybeRecordImplicitNullCheck(instruction); 1350 } 1351 break; 1352 case Primitive::kPrimShort: 1353 __ Ldarh(Register(dst), base); 1354 if (needs_null_check) { 1355 MaybeRecordImplicitNullCheck(instruction); 1356 } 1357 __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte); 1358 break; 1359 case Primitive::kPrimInt: 1360 case Primitive::kPrimNot: 1361 case Primitive::kPrimLong: 1362 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); 1363 __ Ldar(Register(dst), base); 1364 if (needs_null_check) { 1365 MaybeRecordImplicitNullCheck(instruction); 1366 } 1367 break; 1368 case Primitive::kPrimFloat: 1369 case Primitive::kPrimDouble: { 1370 DCHECK(dst.IsFPRegister()); 1371 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); 1372 1373 Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); 1374 __ Ldar(temp, base); 1375 if (needs_null_check) { 1376 MaybeRecordImplicitNullCheck(instruction); 1377 } 1378 __ Fmov(FPRegister(dst), temp); 1379 break; 1380 } 1381 case Primitive::kPrimVoid: 1382 LOG(FATAL) << "Unreachable type " << type; 1383 } 1384 } 1385 1386 void CodeGeneratorARM64::Store(Primitive::Type type, 1387 CPURegister src, 1388 const MemOperand& dst) { 1389 switch (type) { 1390 case Primitive::kPrimBoolean: 1391 case Primitive::kPrimByte: 1392 __ Strb(Register(src), dst); 1393 break; 1394 case Primitive::kPrimChar: 1395 case Primitive::kPrimShort: 1396 __ Strh(Register(src), dst); 1397 break; 1398 case Primitive::kPrimInt: 1399 case Primitive::kPrimNot: 1400 case Primitive::kPrimLong: 1401 case Primitive::kPrimFloat: 1402 case Primitive::kPrimDouble: 1403 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); 1404 __ Str(src, dst); 1405 break; 1406 case Primitive::kPrimVoid: 1407 LOG(FATAL) << "Unreachable type " << type; 1408 } 1409 } 1410 1411 void CodeGeneratorARM64::StoreRelease(Primitive::Type type, 1412 CPURegister src, 1413 const MemOperand& dst) { 1414 UseScratchRegisterScope temps(GetVIXLAssembler()); 1415 Register temp_base = temps.AcquireX(); 1416 1417 DCHECK(!dst.IsPreIndex()); 1418 DCHECK(!dst.IsPostIndex()); 1419 1420 // TODO(vixl): Let the MacroAssembler handle this. 1421 Operand op = OperandFromMemOperand(dst); 1422 __ Add(temp_base, dst.base(), op); 1423 MemOperand base = MemOperand(temp_base); 1424 switch (type) { 1425 case Primitive::kPrimBoolean: 1426 case Primitive::kPrimByte: 1427 __ Stlrb(Register(src), base); 1428 break; 1429 case Primitive::kPrimChar: 1430 case Primitive::kPrimShort: 1431 __ Stlrh(Register(src), base); 1432 break; 1433 case Primitive::kPrimInt: 1434 case Primitive::kPrimNot: 1435 case Primitive::kPrimLong: 1436 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); 1437 __ Stlr(Register(src), base); 1438 break; 1439 case Primitive::kPrimFloat: 1440 case Primitive::kPrimDouble: { 1441 DCHECK(src.IsFPRegister()); 1442 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); 1443 1444 Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); 1445 __ Fmov(temp, FPRegister(src)); 1446 __ Stlr(temp, base); 1447 break; 1448 } 1449 case Primitive::kPrimVoid: 1450 LOG(FATAL) << "Unreachable type " << type; 1451 } 1452 } 1453 1454 void CodeGeneratorARM64::InvokeRuntime(QuickEntrypointEnum entrypoint, 1455 HInstruction* instruction, 1456 uint32_t dex_pc, 1457 SlowPathCode* slow_path) { 1458 InvokeRuntime(GetThreadOffset<kArm64WordSize>(entrypoint).Int32Value(), 1459 instruction, 1460 dex_pc, 1461 slow_path); 1462 } 1463 1464 void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset, 1465 HInstruction* instruction, 1466 uint32_t dex_pc, 1467 SlowPathCode* slow_path) { 1468 ValidateInvokeRuntime(instruction, slow_path); 1469 BlockPoolsScope block_pools(GetVIXLAssembler()); 1470 __ Ldr(lr, MemOperand(tr, entry_point_offset)); 1471 __ Blr(lr); 1472 RecordPcInfo(instruction, dex_pc, slow_path); 1473 } 1474 1475 void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path, 1476 vixl::Register class_reg) { 1477 UseScratchRegisterScope temps(GetVIXLAssembler()); 1478 Register temp = temps.AcquireW(); 1479 size_t status_offset = mirror::Class::StatusOffset().SizeValue(); 1480 1481 // Even if the initialized flag is set, we need to ensure consistent memory ordering. 1482 // TODO(vixl): Let the MacroAssembler handle MemOperand. 1483 __ Add(temp, class_reg, status_offset); 1484 __ Ldar(temp, HeapOperand(temp)); 1485 __ Cmp(temp, mirror::Class::kStatusInitialized); 1486 __ B(lt, slow_path->GetEntryLabel()); 1487 __ Bind(slow_path->GetExitLabel()); 1488 } 1489 1490 void CodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) { 1491 BarrierType type = BarrierAll; 1492 1493 switch (kind) { 1494 case MemBarrierKind::kAnyAny: 1495 case MemBarrierKind::kAnyStore: { 1496 type = BarrierAll; 1497 break; 1498 } 1499 case MemBarrierKind::kLoadAny: { 1500 type = BarrierReads; 1501 break; 1502 } 1503 case MemBarrierKind::kStoreStore: { 1504 type = BarrierWrites; 1505 break; 1506 } 1507 default: 1508 LOG(FATAL) << "Unexpected memory barrier " << kind; 1509 } 1510 __ Dmb(InnerShareable, type); 1511 } 1512 1513 void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction, 1514 HBasicBlock* successor) { 1515 SuspendCheckSlowPathARM64* slow_path = 1516 down_cast<SuspendCheckSlowPathARM64*>(instruction->GetSlowPath()); 1517 if (slow_path == nullptr) { 1518 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, successor); 1519 instruction->SetSlowPath(slow_path); 1520 codegen_->AddSlowPath(slow_path); 1521 if (successor != nullptr) { 1522 DCHECK(successor->IsLoopHeader()); 1523 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 1524 } 1525 } else { 1526 DCHECK_EQ(slow_path->GetSuccessor(), successor); 1527 } 1528 1529 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); 1530 Register temp = temps.AcquireW(); 1531 1532 __ Ldrh(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64WordSize>().SizeValue())); 1533 if (successor == nullptr) { 1534 __ Cbnz(temp, slow_path->GetEntryLabel()); 1535 __ Bind(slow_path->GetReturnLabel()); 1536 } else { 1537 __ Cbz(temp, codegen_->GetLabelOf(successor)); 1538 __ B(slow_path->GetEntryLabel()); 1539 // slow_path will return to GetLabelOf(successor). 1540 } 1541 } 1542 1543 InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph, 1544 CodeGeneratorARM64* codegen) 1545 : InstructionCodeGenerator(graph, codegen), 1546 assembler_(codegen->GetAssembler()), 1547 codegen_(codegen) {} 1548 1549 #define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M) \ 1550 /* No unimplemented IR. */ 1551 1552 #define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode 1553 1554 enum UnimplementedInstructionBreakCode { 1555 // Using a base helps identify when we hit such breakpoints. 1556 UnimplementedInstructionBreakCodeBaseCode = 0x900, 1557 #define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name), 1558 FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION) 1559 #undef ENUM_UNIMPLEMENTED_INSTRUCTION 1560 }; 1561 1562 #define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name) \ 1563 void InstructionCodeGeneratorARM64::Visit##name(H##name* instr ATTRIBUTE_UNUSED) { \ 1564 __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name)); \ 1565 } \ 1566 void LocationsBuilderARM64::Visit##name(H##name* instr) { \ 1567 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \ 1568 locations->SetOut(Location::Any()); \ 1569 } 1570 FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS) 1571 #undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS 1572 1573 #undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE 1574 #undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION 1575 1576 void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) { 1577 DCHECK_EQ(instr->InputCount(), 2U); 1578 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); 1579 Primitive::Type type = instr->GetResultType(); 1580 switch (type) { 1581 case Primitive::kPrimInt: 1582 case Primitive::kPrimLong: 1583 locations->SetInAt(0, Location::RequiresRegister()); 1584 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instr->InputAt(1), instr)); 1585 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 1586 break; 1587 1588 case Primitive::kPrimFloat: 1589 case Primitive::kPrimDouble: 1590 locations->SetInAt(0, Location::RequiresFpuRegister()); 1591 locations->SetInAt(1, Location::RequiresFpuRegister()); 1592 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 1593 break; 1594 1595 default: 1596 LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type; 1597 } 1598 } 1599 1600 void LocationsBuilderARM64::HandleFieldGet(HInstruction* instruction) { 1601 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 1602 1603 bool object_field_get_with_read_barrier = 1604 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 1605 LocationSummary* locations = 1606 new (GetGraph()->GetArena()) LocationSummary(instruction, 1607 object_field_get_with_read_barrier ? 1608 LocationSummary::kCallOnSlowPath : 1609 LocationSummary::kNoCall); 1610 locations->SetInAt(0, Location::RequiresRegister()); 1611 if (Primitive::IsFloatingPointType(instruction->GetType())) { 1612 locations->SetOut(Location::RequiresFpuRegister()); 1613 } else { 1614 // The output overlaps for an object field get when read barriers 1615 // are enabled: we do not want the load to overwrite the object's 1616 // location, as we need it to emit the read barrier. 1617 locations->SetOut( 1618 Location::RequiresRegister(), 1619 object_field_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 1620 } 1621 } 1622 1623 void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction, 1624 const FieldInfo& field_info) { 1625 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 1626 LocationSummary* locations = instruction->GetLocations(); 1627 Location base_loc = locations->InAt(0); 1628 Location out = locations->Out(); 1629 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 1630 Primitive::Type field_type = field_info.GetFieldType(); 1631 BlockPoolsScope block_pools(GetVIXLAssembler()); 1632 MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset()); 1633 1634 if (field_type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 1635 // Object FieldGet with Baker's read barrier case. 1636 MacroAssembler* masm = GetVIXLAssembler(); 1637 UseScratchRegisterScope temps(masm); 1638 // /* HeapReference<Object> */ out = *(base + offset) 1639 Register base = RegisterFrom(base_loc, Primitive::kPrimNot); 1640 Register temp = temps.AcquireW(); 1641 // Note that potential implicit null checks are handled in this 1642 // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier call. 1643 codegen_->GenerateFieldLoadWithBakerReadBarrier( 1644 instruction, 1645 out, 1646 base, 1647 offset, 1648 temp, 1649 /* needs_null_check */ true, 1650 field_info.IsVolatile()); 1651 } else { 1652 // General case. 1653 if (field_info.IsVolatile()) { 1654 // Note that a potential implicit null check is handled in this 1655 // CodeGeneratorARM64::LoadAcquire call. 1656 // NB: LoadAcquire will record the pc info if needed. 1657 codegen_->LoadAcquire( 1658 instruction, OutputCPURegister(instruction), field, /* needs_null_check */ true); 1659 } else { 1660 codegen_->Load(field_type, OutputCPURegister(instruction), field); 1661 codegen_->MaybeRecordImplicitNullCheck(instruction); 1662 } 1663 if (field_type == Primitive::kPrimNot) { 1664 // If read barriers are enabled, emit read barriers other than 1665 // Baker's using a slow path (and also unpoison the loaded 1666 // reference, if heap poisoning is enabled). 1667 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 1668 } 1669 } 1670 } 1671 1672 void LocationsBuilderARM64::HandleFieldSet(HInstruction* instruction) { 1673 LocationSummary* locations = 1674 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 1675 locations->SetInAt(0, Location::RequiresRegister()); 1676 if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) { 1677 locations->SetInAt(1, Location::RequiresFpuRegister()); 1678 } else { 1679 locations->SetInAt(1, Location::RequiresRegister()); 1680 } 1681 } 1682 1683 void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction, 1684 const FieldInfo& field_info, 1685 bool value_can_be_null) { 1686 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 1687 BlockPoolsScope block_pools(GetVIXLAssembler()); 1688 1689 Register obj = InputRegisterAt(instruction, 0); 1690 CPURegister value = InputCPURegisterAt(instruction, 1); 1691 CPURegister source = value; 1692 Offset offset = field_info.GetFieldOffset(); 1693 Primitive::Type field_type = field_info.GetFieldType(); 1694 1695 { 1696 // We use a block to end the scratch scope before the write barrier, thus 1697 // freeing the temporary registers so they can be used in `MarkGCCard`. 1698 UseScratchRegisterScope temps(GetVIXLAssembler()); 1699 1700 if (kPoisonHeapReferences && field_type == Primitive::kPrimNot) { 1701 DCHECK(value.IsW()); 1702 Register temp = temps.AcquireW(); 1703 __ Mov(temp, value.W()); 1704 GetAssembler()->PoisonHeapReference(temp.W()); 1705 source = temp; 1706 } 1707 1708 if (field_info.IsVolatile()) { 1709 codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset)); 1710 codegen_->MaybeRecordImplicitNullCheck(instruction); 1711 } else { 1712 codegen_->Store(field_type, source, HeapOperand(obj, offset)); 1713 codegen_->MaybeRecordImplicitNullCheck(instruction); 1714 } 1715 } 1716 1717 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 1718 codegen_->MarkGCCard(obj, Register(value), value_can_be_null); 1719 } 1720 } 1721 1722 void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) { 1723 Primitive::Type type = instr->GetType(); 1724 1725 switch (type) { 1726 case Primitive::kPrimInt: 1727 case Primitive::kPrimLong: { 1728 Register dst = OutputRegister(instr); 1729 Register lhs = InputRegisterAt(instr, 0); 1730 Operand rhs = InputOperandAt(instr, 1); 1731 if (instr->IsAdd()) { 1732 __ Add(dst, lhs, rhs); 1733 } else if (instr->IsAnd()) { 1734 __ And(dst, lhs, rhs); 1735 } else if (instr->IsOr()) { 1736 __ Orr(dst, lhs, rhs); 1737 } else if (instr->IsSub()) { 1738 __ Sub(dst, lhs, rhs); 1739 } else if (instr->IsRor()) { 1740 if (rhs.IsImmediate()) { 1741 uint32_t shift = rhs.immediate() & (lhs.SizeInBits() - 1); 1742 __ Ror(dst, lhs, shift); 1743 } else { 1744 // Ensure shift distance is in the same size register as the result. If 1745 // we are rotating a long and the shift comes in a w register originally, 1746 // we don't need to sxtw for use as an x since the shift distances are 1747 // all & reg_bits - 1. 1748 __ Ror(dst, lhs, RegisterFrom(instr->GetLocations()->InAt(1), type)); 1749 } 1750 } else { 1751 DCHECK(instr->IsXor()); 1752 __ Eor(dst, lhs, rhs); 1753 } 1754 break; 1755 } 1756 case Primitive::kPrimFloat: 1757 case Primitive::kPrimDouble: { 1758 FPRegister dst = OutputFPRegister(instr); 1759 FPRegister lhs = InputFPRegisterAt(instr, 0); 1760 FPRegister rhs = InputFPRegisterAt(instr, 1); 1761 if (instr->IsAdd()) { 1762 __ Fadd(dst, lhs, rhs); 1763 } else if (instr->IsSub()) { 1764 __ Fsub(dst, lhs, rhs); 1765 } else { 1766 LOG(FATAL) << "Unexpected floating-point binary operation"; 1767 } 1768 break; 1769 } 1770 default: 1771 LOG(FATAL) << "Unexpected binary operation type " << type; 1772 } 1773 } 1774 1775 void LocationsBuilderARM64::HandleShift(HBinaryOperation* instr) { 1776 DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr()); 1777 1778 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); 1779 Primitive::Type type = instr->GetResultType(); 1780 switch (type) { 1781 case Primitive::kPrimInt: 1782 case Primitive::kPrimLong: { 1783 locations->SetInAt(0, Location::RequiresRegister()); 1784 locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1))); 1785 locations->SetOut(Location::RequiresRegister()); 1786 break; 1787 } 1788 default: 1789 LOG(FATAL) << "Unexpected shift type " << type; 1790 } 1791 } 1792 1793 void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) { 1794 DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr()); 1795 1796 Primitive::Type type = instr->GetType(); 1797 switch (type) { 1798 case Primitive::kPrimInt: 1799 case Primitive::kPrimLong: { 1800 Register dst = OutputRegister(instr); 1801 Register lhs = InputRegisterAt(instr, 0); 1802 Operand rhs = InputOperandAt(instr, 1); 1803 if (rhs.IsImmediate()) { 1804 uint32_t shift_value = rhs.immediate() & 1805 (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance); 1806 if (instr->IsShl()) { 1807 __ Lsl(dst, lhs, shift_value); 1808 } else if (instr->IsShr()) { 1809 __ Asr(dst, lhs, shift_value); 1810 } else { 1811 __ Lsr(dst, lhs, shift_value); 1812 } 1813 } else { 1814 Register rhs_reg = dst.IsX() ? rhs.reg().X() : rhs.reg().W(); 1815 1816 if (instr->IsShl()) { 1817 __ Lsl(dst, lhs, rhs_reg); 1818 } else if (instr->IsShr()) { 1819 __ Asr(dst, lhs, rhs_reg); 1820 } else { 1821 __ Lsr(dst, lhs, rhs_reg); 1822 } 1823 } 1824 break; 1825 } 1826 default: 1827 LOG(FATAL) << "Unexpected shift operation type " << type; 1828 } 1829 } 1830 1831 void LocationsBuilderARM64::VisitAdd(HAdd* instruction) { 1832 HandleBinaryOp(instruction); 1833 } 1834 1835 void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) { 1836 HandleBinaryOp(instruction); 1837 } 1838 1839 void LocationsBuilderARM64::VisitAnd(HAnd* instruction) { 1840 HandleBinaryOp(instruction); 1841 } 1842 1843 void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) { 1844 HandleBinaryOp(instruction); 1845 } 1846 1847 void LocationsBuilderARM64::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instr) { 1848 DCHECK(Primitive::IsIntegralType(instr->GetType())) << instr->GetType(); 1849 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); 1850 locations->SetInAt(0, Location::RequiresRegister()); 1851 // There is no immediate variant of negated bitwise instructions in AArch64. 1852 locations->SetInAt(1, Location::RequiresRegister()); 1853 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 1854 } 1855 1856 void InstructionCodeGeneratorARM64::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instr) { 1857 Register dst = OutputRegister(instr); 1858 Register lhs = InputRegisterAt(instr, 0); 1859 Register rhs = InputRegisterAt(instr, 1); 1860 1861 switch (instr->GetOpKind()) { 1862 case HInstruction::kAnd: 1863 __ Bic(dst, lhs, rhs); 1864 break; 1865 case HInstruction::kOr: 1866 __ Orn(dst, lhs, rhs); 1867 break; 1868 case HInstruction::kXor: 1869 __ Eon(dst, lhs, rhs); 1870 break; 1871 default: 1872 LOG(FATAL) << "Unreachable"; 1873 } 1874 } 1875 1876 void LocationsBuilderARM64::VisitArm64DataProcWithShifterOp( 1877 HArm64DataProcWithShifterOp* instruction) { 1878 DCHECK(instruction->GetType() == Primitive::kPrimInt || 1879 instruction->GetType() == Primitive::kPrimLong); 1880 LocationSummary* locations = 1881 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 1882 if (instruction->GetInstrKind() == HInstruction::kNeg) { 1883 locations->SetInAt(0, Location::ConstantLocation(instruction->InputAt(0)->AsConstant())); 1884 } else { 1885 locations->SetInAt(0, Location::RequiresRegister()); 1886 } 1887 locations->SetInAt(1, Location::RequiresRegister()); 1888 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 1889 } 1890 1891 void InstructionCodeGeneratorARM64::VisitArm64DataProcWithShifterOp( 1892 HArm64DataProcWithShifterOp* instruction) { 1893 Primitive::Type type = instruction->GetType(); 1894 HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1895 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 1896 Register out = OutputRegister(instruction); 1897 Register left; 1898 if (kind != HInstruction::kNeg) { 1899 left = InputRegisterAt(instruction, 0); 1900 } 1901 // If this `HArm64DataProcWithShifterOp` was created by merging a type conversion as the 1902 // shifter operand operation, the IR generating `right_reg` (input to the type 1903 // conversion) can have a different type from the current instruction's type, 1904 // so we manually indicate the type. 1905 Register right_reg = RegisterFrom(instruction->GetLocations()->InAt(1), type); 1906 int64_t shift_amount = instruction->GetShiftAmount() & 1907 (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance); 1908 1909 Operand right_operand(0); 1910 1911 HArm64DataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 1912 if (HArm64DataProcWithShifterOp::IsExtensionOp(op_kind)) { 1913 right_operand = Operand(right_reg, helpers::ExtendFromOpKind(op_kind)); 1914 } else { 1915 right_operand = Operand(right_reg, helpers::ShiftFromOpKind(op_kind), shift_amount); 1916 } 1917 1918 // Logical binary operations do not support extension operations in the 1919 // operand. Note that VIXL would still manage if it was passed by generating 1920 // the extension as a separate instruction. 1921 // `HNeg` also does not support extension. See comments in `ShifterOperandSupportsExtension()`. 1922 DCHECK(!right_operand.IsExtendedRegister() || 1923 (kind != HInstruction::kAnd && kind != HInstruction::kOr && kind != HInstruction::kXor && 1924 kind != HInstruction::kNeg)); 1925 switch (kind) { 1926 case HInstruction::kAdd: 1927 __ Add(out, left, right_operand); 1928 break; 1929 case HInstruction::kAnd: 1930 __ And(out, left, right_operand); 1931 break; 1932 case HInstruction::kNeg: 1933 DCHECK(instruction->InputAt(0)->AsConstant()->IsArithmeticZero()); 1934 __ Neg(out, right_operand); 1935 break; 1936 case HInstruction::kOr: 1937 __ Orr(out, left, right_operand); 1938 break; 1939 case HInstruction::kSub: 1940 __ Sub(out, left, right_operand); 1941 break; 1942 case HInstruction::kXor: 1943 __ Eor(out, left, right_operand); 1944 break; 1945 default: 1946 LOG(FATAL) << "Unexpected operation kind: " << kind; 1947 UNREACHABLE(); 1948 } 1949 } 1950 1951 void LocationsBuilderARM64::VisitArm64IntermediateAddress(HArm64IntermediateAddress* instruction) { 1952 // The read barrier instrumentation does not support the 1953 // HArm64IntermediateAddress instruction yet. 1954 DCHECK(!kEmitCompilerReadBarrier); 1955 LocationSummary* locations = 1956 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 1957 locations->SetInAt(0, Location::RequiresRegister()); 1958 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->GetOffset(), instruction)); 1959 locations->SetOut(Location::RequiresRegister()); 1960 } 1961 1962 void InstructionCodeGeneratorARM64::VisitArm64IntermediateAddress( 1963 HArm64IntermediateAddress* instruction) { 1964 // The read barrier instrumentation does not support the 1965 // HArm64IntermediateAddress instruction yet. 1966 DCHECK(!kEmitCompilerReadBarrier); 1967 __ Add(OutputRegister(instruction), 1968 InputRegisterAt(instruction, 0), 1969 Operand(InputOperandAt(instruction, 1))); 1970 } 1971 1972 void LocationsBuilderARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 1973 LocationSummary* locations = 1974 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 1975 HInstruction* accumulator = instr->InputAt(HMultiplyAccumulate::kInputAccumulatorIndex); 1976 if (instr->GetOpKind() == HInstruction::kSub && 1977 accumulator->IsConstant() && 1978 accumulator->AsConstant()->IsArithmeticZero()) { 1979 // Don't allocate register for Mneg instruction. 1980 } else { 1981 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 1982 Location::RequiresRegister()); 1983 } 1984 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 1985 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 1986 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 1987 } 1988 1989 void InstructionCodeGeneratorARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 1990 Register res = OutputRegister(instr); 1991 Register mul_left = InputRegisterAt(instr, HMultiplyAccumulate::kInputMulLeftIndex); 1992 Register mul_right = InputRegisterAt(instr, HMultiplyAccumulate::kInputMulRightIndex); 1993 1994 // Avoid emitting code that could trigger Cortex A53's erratum 835769. 1995 // This fixup should be carried out for all multiply-accumulate instructions: 1996 // madd, msub, smaddl, smsubl, umaddl and umsubl. 1997 if (instr->GetType() == Primitive::kPrimLong && 1998 codegen_->GetInstructionSetFeatures().NeedFixCortexA53_835769()) { 1999 MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen_)->GetVIXLAssembler(); 2000 vixl::Instruction* prev = masm->GetCursorAddress<vixl::Instruction*>() - vixl::kInstructionSize; 2001 if (prev->IsLoadOrStore()) { 2002 // Make sure we emit only exactly one nop. 2003 vixl::CodeBufferCheckScope scope(masm, 2004 vixl::kInstructionSize, 2005 vixl::CodeBufferCheckScope::kCheck, 2006 vixl::CodeBufferCheckScope::kExactSize); 2007 __ nop(); 2008 } 2009 } 2010 2011 if (instr->GetOpKind() == HInstruction::kAdd) { 2012 Register accumulator = InputRegisterAt(instr, HMultiplyAccumulate::kInputAccumulatorIndex); 2013 __ Madd(res, mul_left, mul_right, accumulator); 2014 } else { 2015 DCHECK(instr->GetOpKind() == HInstruction::kSub); 2016 HInstruction* accum_instr = instr->InputAt(HMultiplyAccumulate::kInputAccumulatorIndex); 2017 if (accum_instr->IsConstant() && accum_instr->AsConstant()->IsArithmeticZero()) { 2018 __ Mneg(res, mul_left, mul_right); 2019 } else { 2020 Register accumulator = InputRegisterAt(instr, HMultiplyAccumulate::kInputAccumulatorIndex); 2021 __ Msub(res, mul_left, mul_right, accumulator); 2022 } 2023 } 2024 } 2025 2026 void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) { 2027 bool object_array_get_with_read_barrier = 2028 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 2029 LocationSummary* locations = 2030 new (GetGraph()->GetArena()) LocationSummary(instruction, 2031 object_array_get_with_read_barrier ? 2032 LocationSummary::kCallOnSlowPath : 2033 LocationSummary::kNoCall); 2034 locations->SetInAt(0, Location::RequiresRegister()); 2035 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 2036 if (Primitive::IsFloatingPointType(instruction->GetType())) { 2037 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2038 } else { 2039 // The output overlaps in the case of an object array get with 2040 // read barriers enabled: we do not want the move to overwrite the 2041 // array's location, as we need it to emit the read barrier. 2042 locations->SetOut( 2043 Location::RequiresRegister(), 2044 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 2045 } 2046 } 2047 2048 void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { 2049 Primitive::Type type = instruction->GetType(); 2050 Register obj = InputRegisterAt(instruction, 0); 2051 LocationSummary* locations = instruction->GetLocations(); 2052 Location index = locations->InAt(1); 2053 uint32_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(type)).Uint32Value(); 2054 Location out = locations->Out(); 2055 2056 MacroAssembler* masm = GetVIXLAssembler(); 2057 UseScratchRegisterScope temps(masm); 2058 // Block pools between `Load` and `MaybeRecordImplicitNullCheck`. 2059 BlockPoolsScope block_pools(masm); 2060 2061 if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 2062 // Object ArrayGet with Baker's read barrier case. 2063 Register temp = temps.AcquireW(); 2064 // The read barrier instrumentation does not support the 2065 // HArm64IntermediateAddress instruction yet. 2066 DCHECK(!instruction->GetArray()->IsArm64IntermediateAddress()); 2067 // Note that a potential implicit null check is handled in the 2068 // CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier call. 2069 codegen_->GenerateArrayLoadWithBakerReadBarrier( 2070 instruction, out, obj.W(), offset, index, temp, /* needs_null_check */ true); 2071 } else { 2072 // General case. 2073 MemOperand source = HeapOperand(obj); 2074 if (index.IsConstant()) { 2075 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type); 2076 source = HeapOperand(obj, offset); 2077 } else { 2078 Register temp = temps.AcquireSameSizeAs(obj); 2079 if (instruction->GetArray()->IsArm64IntermediateAddress()) { 2080 // The read barrier instrumentation does not support the 2081 // HArm64IntermediateAddress instruction yet. 2082 DCHECK(!kEmitCompilerReadBarrier); 2083 // We do not need to compute the intermediate address from the array: the 2084 // input instruction has done it already. See the comment in 2085 // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`. 2086 if (kIsDebugBuild) { 2087 HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress(); 2088 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), offset); 2089 } 2090 temp = obj; 2091 } else { 2092 __ Add(temp, obj, offset); 2093 } 2094 source = HeapOperand(temp, XRegisterFrom(index), LSL, Primitive::ComponentSizeShift(type)); 2095 } 2096 2097 codegen_->Load(type, OutputCPURegister(instruction), source); 2098 codegen_->MaybeRecordImplicitNullCheck(instruction); 2099 2100 if (type == Primitive::kPrimNot) { 2101 static_assert( 2102 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 2103 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 2104 Location obj_loc = locations->InAt(0); 2105 if (index.IsConstant()) { 2106 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, obj_loc, offset); 2107 } else { 2108 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, obj_loc, offset, index); 2109 } 2110 } 2111 } 2112 } 2113 2114 void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) { 2115 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 2116 locations->SetInAt(0, Location::RequiresRegister()); 2117 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2118 } 2119 2120 void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) { 2121 BlockPoolsScope block_pools(GetVIXLAssembler()); 2122 __ Ldr(OutputRegister(instruction), 2123 HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset())); 2124 codegen_->MaybeRecordImplicitNullCheck(instruction); 2125 } 2126 2127 void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) { 2128 Primitive::Type value_type = instruction->GetComponentType(); 2129 2130 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 2131 bool object_array_set_with_read_barrier = 2132 kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot); 2133 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 2134 instruction, 2135 (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ? 2136 LocationSummary::kCallOnSlowPath : 2137 LocationSummary::kNoCall); 2138 locations->SetInAt(0, Location::RequiresRegister()); 2139 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 2140 if (Primitive::IsFloatingPointType(value_type)) { 2141 locations->SetInAt(2, Location::RequiresFpuRegister()); 2142 } else { 2143 locations->SetInAt(2, Location::RequiresRegister()); 2144 } 2145 } 2146 2147 void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { 2148 Primitive::Type value_type = instruction->GetComponentType(); 2149 LocationSummary* locations = instruction->GetLocations(); 2150 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 2151 bool needs_write_barrier = 2152 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 2153 2154 Register array = InputRegisterAt(instruction, 0); 2155 CPURegister value = InputCPURegisterAt(instruction, 2); 2156 CPURegister source = value; 2157 Location index = locations->InAt(1); 2158 size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 2159 MemOperand destination = HeapOperand(array); 2160 MacroAssembler* masm = GetVIXLAssembler(); 2161 BlockPoolsScope block_pools(masm); 2162 2163 if (!needs_write_barrier) { 2164 DCHECK(!may_need_runtime_call_for_type_check); 2165 if (index.IsConstant()) { 2166 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type); 2167 destination = HeapOperand(array, offset); 2168 } else { 2169 UseScratchRegisterScope temps(masm); 2170 Register temp = temps.AcquireSameSizeAs(array); 2171 if (instruction->GetArray()->IsArm64IntermediateAddress()) { 2172 // The read barrier instrumentation does not support the 2173 // HArm64IntermediateAddress instruction yet. 2174 DCHECK(!kEmitCompilerReadBarrier); 2175 // We do not need to compute the intermediate address from the array: the 2176 // input instruction has done it already. See the comment in 2177 // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`. 2178 if (kIsDebugBuild) { 2179 HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress(); 2180 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset); 2181 } 2182 temp = array; 2183 } else { 2184 __ Add(temp, array, offset); 2185 } 2186 destination = HeapOperand(temp, 2187 XRegisterFrom(index), 2188 LSL, 2189 Primitive::ComponentSizeShift(value_type)); 2190 } 2191 codegen_->Store(value_type, value, destination); 2192 codegen_->MaybeRecordImplicitNullCheck(instruction); 2193 } else { 2194 DCHECK(needs_write_barrier); 2195 DCHECK(!instruction->GetArray()->IsArm64IntermediateAddress()); 2196 vixl::Label done; 2197 SlowPathCodeARM64* slow_path = nullptr; 2198 { 2199 // We use a block to end the scratch scope before the write barrier, thus 2200 // freeing the temporary registers so they can be used in `MarkGCCard`. 2201 UseScratchRegisterScope temps(masm); 2202 Register temp = temps.AcquireSameSizeAs(array); 2203 if (index.IsConstant()) { 2204 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type); 2205 destination = HeapOperand(array, offset); 2206 } else { 2207 destination = HeapOperand(temp, 2208 XRegisterFrom(index), 2209 LSL, 2210 Primitive::ComponentSizeShift(value_type)); 2211 } 2212 2213 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2214 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 2215 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 2216 2217 if (may_need_runtime_call_for_type_check) { 2218 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM64(instruction); 2219 codegen_->AddSlowPath(slow_path); 2220 if (instruction->GetValueCanBeNull()) { 2221 vixl::Label non_zero; 2222 __ Cbnz(Register(value), &non_zero); 2223 if (!index.IsConstant()) { 2224 __ Add(temp, array, offset); 2225 } 2226 __ Str(wzr, destination); 2227 codegen_->MaybeRecordImplicitNullCheck(instruction); 2228 __ B(&done); 2229 __ Bind(&non_zero); 2230 } 2231 2232 if (kEmitCompilerReadBarrier) { 2233 // When read barriers are enabled, the type checking 2234 // instrumentation requires two read barriers: 2235 // 2236 // __ Mov(temp2, temp); 2237 // // /* HeapReference<Class> */ temp = temp->component_type_ 2238 // __ Ldr(temp, HeapOperand(temp, component_offset)); 2239 // codegen_->GenerateReadBarrierSlow( 2240 // instruction, temp_loc, temp_loc, temp2_loc, component_offset); 2241 // 2242 // // /* HeapReference<Class> */ temp2 = value->klass_ 2243 // __ Ldr(temp2, HeapOperand(Register(value), class_offset)); 2244 // codegen_->GenerateReadBarrierSlow( 2245 // instruction, temp2_loc, temp2_loc, value_loc, class_offset, temp_loc); 2246 // 2247 // __ Cmp(temp, temp2); 2248 // 2249 // However, the second read barrier may trash `temp`, as it 2250 // is a temporary register, and as such would not be saved 2251 // along with live registers before calling the runtime (nor 2252 // restored afterwards). So in this case, we bail out and 2253 // delegate the work to the array set slow path. 2254 // 2255 // TODO: Extend the register allocator to support a new 2256 // "(locally) live temp" location so as to avoid always 2257 // going into the slow path when read barriers are enabled. 2258 __ B(slow_path->GetEntryLabel()); 2259 } else { 2260 Register temp2 = temps.AcquireSameSizeAs(array); 2261 // /* HeapReference<Class> */ temp = array->klass_ 2262 __ Ldr(temp, HeapOperand(array, class_offset)); 2263 codegen_->MaybeRecordImplicitNullCheck(instruction); 2264 GetAssembler()->MaybeUnpoisonHeapReference(temp); 2265 2266 // /* HeapReference<Class> */ temp = temp->component_type_ 2267 __ Ldr(temp, HeapOperand(temp, component_offset)); 2268 // /* HeapReference<Class> */ temp2 = value->klass_ 2269 __ Ldr(temp2, HeapOperand(Register(value), class_offset)); 2270 // If heap poisoning is enabled, no need to unpoison `temp` 2271 // nor `temp2`, as we are comparing two poisoned references. 2272 __ Cmp(temp, temp2); 2273 2274 if (instruction->StaticTypeOfArrayIsObjectArray()) { 2275 vixl::Label do_put; 2276 __ B(eq, &do_put); 2277 // If heap poisoning is enabled, the `temp` reference has 2278 // not been unpoisoned yet; unpoison it now. 2279 GetAssembler()->MaybeUnpoisonHeapReference(temp); 2280 2281 // /* HeapReference<Class> */ temp = temp->super_class_ 2282 __ Ldr(temp, HeapOperand(temp, super_offset)); 2283 // If heap poisoning is enabled, no need to unpoison 2284 // `temp`, as we are comparing against null below. 2285 __ Cbnz(temp, slow_path->GetEntryLabel()); 2286 __ Bind(&do_put); 2287 } else { 2288 __ B(ne, slow_path->GetEntryLabel()); 2289 } 2290 temps.Release(temp2); 2291 } 2292 } 2293 2294 if (kPoisonHeapReferences) { 2295 Register temp2 = temps.AcquireSameSizeAs(array); 2296 DCHECK(value.IsW()); 2297 __ Mov(temp2, value.W()); 2298 GetAssembler()->PoisonHeapReference(temp2); 2299 source = temp2; 2300 } 2301 2302 if (!index.IsConstant()) { 2303 __ Add(temp, array, offset); 2304 } 2305 __ Str(source, destination); 2306 2307 if (!may_need_runtime_call_for_type_check) { 2308 codegen_->MaybeRecordImplicitNullCheck(instruction); 2309 } 2310 } 2311 2312 codegen_->MarkGCCard(array, value.W(), instruction->GetValueCanBeNull()); 2313 2314 if (done.IsLinked()) { 2315 __ Bind(&done); 2316 } 2317 2318 if (slow_path != nullptr) { 2319 __ Bind(slow_path->GetExitLabel()); 2320 } 2321 } 2322 } 2323 2324 void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) { 2325 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock() 2326 ? LocationSummary::kCallOnSlowPath 2327 : LocationSummary::kNoCall; 2328 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 2329 locations->SetInAt(0, Location::RequiresRegister()); 2330 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction)); 2331 if (instruction->HasUses()) { 2332 locations->SetOut(Location::SameAsFirstInput()); 2333 } 2334 } 2335 2336 void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) { 2337 BoundsCheckSlowPathARM64* slow_path = 2338 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(instruction); 2339 codegen_->AddSlowPath(slow_path); 2340 2341 __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1)); 2342 __ B(slow_path->GetEntryLabel(), hs); 2343 } 2344 2345 void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) { 2346 LocationSummary* locations = 2347 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 2348 locations->SetInAt(0, Location::RequiresRegister()); 2349 if (check->HasUses()) { 2350 locations->SetOut(Location::SameAsFirstInput()); 2351 } 2352 } 2353 2354 void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) { 2355 // We assume the class is not null. 2356 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64( 2357 check->GetLoadClass(), check, check->GetDexPc(), true); 2358 codegen_->AddSlowPath(slow_path); 2359 GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0)); 2360 } 2361 2362 static bool IsFloatingPointZeroConstant(HInstruction* inst) { 2363 return (inst->IsFloatConstant() && (inst->AsFloatConstant()->IsArithmeticZero())) 2364 || (inst->IsDoubleConstant() && (inst->AsDoubleConstant()->IsArithmeticZero())); 2365 } 2366 2367 void InstructionCodeGeneratorARM64::GenerateFcmp(HInstruction* instruction) { 2368 FPRegister lhs_reg = InputFPRegisterAt(instruction, 0); 2369 Location rhs_loc = instruction->GetLocations()->InAt(1); 2370 if (rhs_loc.IsConstant()) { 2371 // 0.0 is the only immediate that can be encoded directly in 2372 // an FCMP instruction. 2373 // 2374 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 2375 // specify that in a floating-point comparison, positive zero 2376 // and negative zero are considered equal, so we can use the 2377 // literal 0.0 for both cases here. 2378 // 2379 // Note however that some methods (Float.equal, Float.compare, 2380 // Float.compareTo, Double.equal, Double.compare, 2381 // Double.compareTo, Math.max, Math.min, StrictMath.max, 2382 // StrictMath.min) consider 0.0 to be (strictly) greater than 2383 // -0.0. So if we ever translate calls to these methods into a 2384 // HCompare instruction, we must handle the -0.0 case with 2385 // care here. 2386 DCHECK(IsFloatingPointZeroConstant(rhs_loc.GetConstant())); 2387 __ Fcmp(lhs_reg, 0.0); 2388 } else { 2389 __ Fcmp(lhs_reg, InputFPRegisterAt(instruction, 1)); 2390 } 2391 } 2392 2393 void LocationsBuilderARM64::VisitCompare(HCompare* compare) { 2394 LocationSummary* locations = 2395 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 2396 Primitive::Type in_type = compare->InputAt(0)->GetType(); 2397 switch (in_type) { 2398 case Primitive::kPrimBoolean: 2399 case Primitive::kPrimByte: 2400 case Primitive::kPrimShort: 2401 case Primitive::kPrimChar: 2402 case Primitive::kPrimInt: 2403 case Primitive::kPrimLong: { 2404 locations->SetInAt(0, Location::RequiresRegister()); 2405 locations->SetInAt(1, ARM64EncodableConstantOrRegister(compare->InputAt(1), compare)); 2406 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2407 break; 2408 } 2409 case Primitive::kPrimFloat: 2410 case Primitive::kPrimDouble: { 2411 locations->SetInAt(0, Location::RequiresFpuRegister()); 2412 locations->SetInAt(1, 2413 IsFloatingPointZeroConstant(compare->InputAt(1)) 2414 ? Location::ConstantLocation(compare->InputAt(1)->AsConstant()) 2415 : Location::RequiresFpuRegister()); 2416 locations->SetOut(Location::RequiresRegister()); 2417 break; 2418 } 2419 default: 2420 LOG(FATAL) << "Unexpected type for compare operation " << in_type; 2421 } 2422 } 2423 2424 void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) { 2425 Primitive::Type in_type = compare->InputAt(0)->GetType(); 2426 2427 // 0 if: left == right 2428 // 1 if: left > right 2429 // -1 if: left < right 2430 switch (in_type) { 2431 case Primitive::kPrimBoolean: 2432 case Primitive::kPrimByte: 2433 case Primitive::kPrimShort: 2434 case Primitive::kPrimChar: 2435 case Primitive::kPrimInt: 2436 case Primitive::kPrimLong: { 2437 Register result = OutputRegister(compare); 2438 Register left = InputRegisterAt(compare, 0); 2439 Operand right = InputOperandAt(compare, 1); 2440 __ Cmp(left, right); 2441 __ Cset(result, ne); // result == +1 if NE or 0 otherwise 2442 __ Cneg(result, result, lt); // result == -1 if LT or unchanged otherwise 2443 break; 2444 } 2445 case Primitive::kPrimFloat: 2446 case Primitive::kPrimDouble: { 2447 Register result = OutputRegister(compare); 2448 GenerateFcmp(compare); 2449 __ Cset(result, ne); 2450 __ Cneg(result, result, ARM64FPCondition(kCondLT, compare->IsGtBias())); 2451 break; 2452 } 2453 default: 2454 LOG(FATAL) << "Unimplemented compare type " << in_type; 2455 } 2456 } 2457 2458 void LocationsBuilderARM64::HandleCondition(HCondition* instruction) { 2459 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 2460 2461 if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) { 2462 locations->SetInAt(0, Location::RequiresFpuRegister()); 2463 locations->SetInAt(1, 2464 IsFloatingPointZeroConstant(instruction->InputAt(1)) 2465 ? Location::ConstantLocation(instruction->InputAt(1)->AsConstant()) 2466 : Location::RequiresFpuRegister()); 2467 } else { 2468 // Integer cases. 2469 locations->SetInAt(0, Location::RequiresRegister()); 2470 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction)); 2471 } 2472 2473 if (!instruction->IsEmittedAtUseSite()) { 2474 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2475 } 2476 } 2477 2478 void InstructionCodeGeneratorARM64::HandleCondition(HCondition* instruction) { 2479 if (instruction->IsEmittedAtUseSite()) { 2480 return; 2481 } 2482 2483 LocationSummary* locations = instruction->GetLocations(); 2484 Register res = RegisterFrom(locations->Out(), instruction->GetType()); 2485 IfCondition if_cond = instruction->GetCondition(); 2486 2487 if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) { 2488 GenerateFcmp(instruction); 2489 __ Cset(res, ARM64FPCondition(if_cond, instruction->IsGtBias())); 2490 } else { 2491 // Integer cases. 2492 Register lhs = InputRegisterAt(instruction, 0); 2493 Operand rhs = InputOperandAt(instruction, 1); 2494 __ Cmp(lhs, rhs); 2495 __ Cset(res, ARM64Condition(if_cond)); 2496 } 2497 } 2498 2499 #define FOR_EACH_CONDITION_INSTRUCTION(M) \ 2500 M(Equal) \ 2501 M(NotEqual) \ 2502 M(LessThan) \ 2503 M(LessThanOrEqual) \ 2504 M(GreaterThan) \ 2505 M(GreaterThanOrEqual) \ 2506 M(Below) \ 2507 M(BelowOrEqual) \ 2508 M(Above) \ 2509 M(AboveOrEqual) 2510 #define DEFINE_CONDITION_VISITORS(Name) \ 2511 void LocationsBuilderARM64::Visit##Name(H##Name* comp) { HandleCondition(comp); } \ 2512 void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { HandleCondition(comp); } 2513 FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS) 2514 #undef DEFINE_CONDITION_VISITORS 2515 #undef FOR_EACH_CONDITION_INSTRUCTION 2516 2517 void InstructionCodeGeneratorARM64::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 2518 DCHECK(instruction->IsDiv() || instruction->IsRem()); 2519 2520 LocationSummary* locations = instruction->GetLocations(); 2521 Location second = locations->InAt(1); 2522 DCHECK(second.IsConstant()); 2523 2524 Register out = OutputRegister(instruction); 2525 Register dividend = InputRegisterAt(instruction, 0); 2526 int64_t imm = Int64FromConstant(second.GetConstant()); 2527 DCHECK(imm == 1 || imm == -1); 2528 2529 if (instruction->IsRem()) { 2530 __ Mov(out, 0); 2531 } else { 2532 if (imm == 1) { 2533 __ Mov(out, dividend); 2534 } else { 2535 __ Neg(out, dividend); 2536 } 2537 } 2538 } 2539 2540 void InstructionCodeGeneratorARM64::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 2541 DCHECK(instruction->IsDiv() || instruction->IsRem()); 2542 2543 LocationSummary* locations = instruction->GetLocations(); 2544 Location second = locations->InAt(1); 2545 DCHECK(second.IsConstant()); 2546 2547 Register out = OutputRegister(instruction); 2548 Register dividend = InputRegisterAt(instruction, 0); 2549 int64_t imm = Int64FromConstant(second.GetConstant()); 2550 uint64_t abs_imm = static_cast<uint64_t>(AbsOrMin(imm)); 2551 int ctz_imm = CTZ(abs_imm); 2552 2553 UseScratchRegisterScope temps(GetVIXLAssembler()); 2554 Register temp = temps.AcquireSameSizeAs(out); 2555 2556 if (instruction->IsDiv()) { 2557 __ Add(temp, dividend, abs_imm - 1); 2558 __ Cmp(dividend, 0); 2559 __ Csel(out, temp, dividend, lt); 2560 if (imm > 0) { 2561 __ Asr(out, out, ctz_imm); 2562 } else { 2563 __ Neg(out, Operand(out, ASR, ctz_imm)); 2564 } 2565 } else { 2566 int bits = instruction->GetResultType() == Primitive::kPrimInt ? 32 : 64; 2567 __ Asr(temp, dividend, bits - 1); 2568 __ Lsr(temp, temp, bits - ctz_imm); 2569 __ Add(out, dividend, temp); 2570 __ And(out, out, abs_imm - 1); 2571 __ Sub(out, out, temp); 2572 } 2573 } 2574 2575 void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 2576 DCHECK(instruction->IsDiv() || instruction->IsRem()); 2577 2578 LocationSummary* locations = instruction->GetLocations(); 2579 Location second = locations->InAt(1); 2580 DCHECK(second.IsConstant()); 2581 2582 Register out = OutputRegister(instruction); 2583 Register dividend = InputRegisterAt(instruction, 0); 2584 int64_t imm = Int64FromConstant(second.GetConstant()); 2585 2586 Primitive::Type type = instruction->GetResultType(); 2587 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 2588 2589 int64_t magic; 2590 int shift; 2591 CalculateMagicAndShiftForDivRem(imm, type == Primitive::kPrimLong /* is_long */, &magic, &shift); 2592 2593 UseScratchRegisterScope temps(GetVIXLAssembler()); 2594 Register temp = temps.AcquireSameSizeAs(out); 2595 2596 // temp = get_high(dividend * magic) 2597 __ Mov(temp, magic); 2598 if (type == Primitive::kPrimLong) { 2599 __ Smulh(temp, dividend, temp); 2600 } else { 2601 __ Smull(temp.X(), dividend, temp); 2602 __ Lsr(temp.X(), temp.X(), 32); 2603 } 2604 2605 if (imm > 0 && magic < 0) { 2606 __ Add(temp, temp, dividend); 2607 } else if (imm < 0 && magic > 0) { 2608 __ Sub(temp, temp, dividend); 2609 } 2610 2611 if (shift != 0) { 2612 __ Asr(temp, temp, shift); 2613 } 2614 2615 if (instruction->IsDiv()) { 2616 __ Sub(out, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31)); 2617 } else { 2618 __ Sub(temp, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31)); 2619 // TODO: Strength reduction for msub. 2620 Register temp_imm = temps.AcquireSameSizeAs(out); 2621 __ Mov(temp_imm, imm); 2622 __ Msub(out, temp, temp_imm, dividend); 2623 } 2624 } 2625 2626 void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) { 2627 DCHECK(instruction->IsDiv() || instruction->IsRem()); 2628 Primitive::Type type = instruction->GetResultType(); 2629 DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong); 2630 2631 LocationSummary* locations = instruction->GetLocations(); 2632 Register out = OutputRegister(instruction); 2633 Location second = locations->InAt(1); 2634 2635 if (second.IsConstant()) { 2636 int64_t imm = Int64FromConstant(second.GetConstant()); 2637 2638 if (imm == 0) { 2639 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 2640 } else if (imm == 1 || imm == -1) { 2641 DivRemOneOrMinusOne(instruction); 2642 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 2643 DivRemByPowerOfTwo(instruction); 2644 } else { 2645 DCHECK(imm <= -2 || imm >= 2); 2646 GenerateDivRemWithAnyConstant(instruction); 2647 } 2648 } else { 2649 Register dividend = InputRegisterAt(instruction, 0); 2650 Register divisor = InputRegisterAt(instruction, 1); 2651 if (instruction->IsDiv()) { 2652 __ Sdiv(out, dividend, divisor); 2653 } else { 2654 UseScratchRegisterScope temps(GetVIXLAssembler()); 2655 Register temp = temps.AcquireSameSizeAs(out); 2656 __ Sdiv(temp, dividend, divisor); 2657 __ Msub(out, temp, divisor, dividend); 2658 } 2659 } 2660 } 2661 2662 void LocationsBuilderARM64::VisitDiv(HDiv* div) { 2663 LocationSummary* locations = 2664 new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall); 2665 switch (div->GetResultType()) { 2666 case Primitive::kPrimInt: 2667 case Primitive::kPrimLong: 2668 locations->SetInAt(0, Location::RequiresRegister()); 2669 locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1))); 2670 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2671 break; 2672 2673 case Primitive::kPrimFloat: 2674 case Primitive::kPrimDouble: 2675 locations->SetInAt(0, Location::RequiresFpuRegister()); 2676 locations->SetInAt(1, Location::RequiresFpuRegister()); 2677 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2678 break; 2679 2680 default: 2681 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 2682 } 2683 } 2684 2685 void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) { 2686 Primitive::Type type = div->GetResultType(); 2687 switch (type) { 2688 case Primitive::kPrimInt: 2689 case Primitive::kPrimLong: 2690 GenerateDivRemIntegral(div); 2691 break; 2692 2693 case Primitive::kPrimFloat: 2694 case Primitive::kPrimDouble: 2695 __ Fdiv(OutputFPRegister(div), InputFPRegisterAt(div, 0), InputFPRegisterAt(div, 1)); 2696 break; 2697 2698 default: 2699 LOG(FATAL) << "Unexpected div type " << type; 2700 } 2701 } 2702 2703 void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) { 2704 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock() 2705 ? LocationSummary::kCallOnSlowPath 2706 : LocationSummary::kNoCall; 2707 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 2708 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 2709 if (instruction->HasUses()) { 2710 locations->SetOut(Location::SameAsFirstInput()); 2711 } 2712 } 2713 2714 void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) { 2715 SlowPathCodeARM64* slow_path = 2716 new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction); 2717 codegen_->AddSlowPath(slow_path); 2718 Location value = instruction->GetLocations()->InAt(0); 2719 2720 Primitive::Type type = instruction->GetType(); 2721 2722 if (!Primitive::IsIntegralType(type)) { 2723 LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck."; 2724 return; 2725 } 2726 2727 if (value.IsConstant()) { 2728 int64_t divisor = Int64ConstantFrom(value); 2729 if (divisor == 0) { 2730 __ B(slow_path->GetEntryLabel()); 2731 } else { 2732 // A division by a non-null constant is valid. We don't need to perform 2733 // any check, so simply fall through. 2734 } 2735 } else { 2736 __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel()); 2737 } 2738 } 2739 2740 void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) { 2741 LocationSummary* locations = 2742 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2743 locations->SetOut(Location::ConstantLocation(constant)); 2744 } 2745 2746 void InstructionCodeGeneratorARM64::VisitDoubleConstant( 2747 HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2748 // Will be generated at use site. 2749 } 2750 2751 void LocationsBuilderARM64::VisitExit(HExit* exit) { 2752 exit->SetLocations(nullptr); 2753 } 2754 2755 void InstructionCodeGeneratorARM64::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 2756 } 2757 2758 void LocationsBuilderARM64::VisitFloatConstant(HFloatConstant* constant) { 2759 LocationSummary* locations = 2760 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2761 locations->SetOut(Location::ConstantLocation(constant)); 2762 } 2763 2764 void InstructionCodeGeneratorARM64::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2765 // Will be generated at use site. 2766 } 2767 2768 void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* successor) { 2769 DCHECK(!successor->IsExitBlock()); 2770 HBasicBlock* block = got->GetBlock(); 2771 HInstruction* previous = got->GetPrevious(); 2772 HLoopInformation* info = block->GetLoopInformation(); 2773 2774 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 2775 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 2776 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 2777 return; 2778 } 2779 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 2780 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 2781 } 2782 if (!codegen_->GoesToNextBlock(block, successor)) { 2783 __ B(codegen_->GetLabelOf(successor)); 2784 } 2785 } 2786 2787 void LocationsBuilderARM64::VisitGoto(HGoto* got) { 2788 got->SetLocations(nullptr); 2789 } 2790 2791 void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) { 2792 HandleGoto(got, got->GetSuccessor()); 2793 } 2794 2795 void LocationsBuilderARM64::VisitTryBoundary(HTryBoundary* try_boundary) { 2796 try_boundary->SetLocations(nullptr); 2797 } 2798 2799 void InstructionCodeGeneratorARM64::VisitTryBoundary(HTryBoundary* try_boundary) { 2800 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 2801 if (!successor->IsExitBlock()) { 2802 HandleGoto(try_boundary, successor); 2803 } 2804 } 2805 2806 void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction, 2807 size_t condition_input_index, 2808 vixl::Label* true_target, 2809 vixl::Label* false_target) { 2810 // FP branching requires both targets to be explicit. If either of the targets 2811 // is nullptr (fallthrough) use and bind `fallthrough_target` instead. 2812 vixl::Label fallthrough_target; 2813 HInstruction* cond = instruction->InputAt(condition_input_index); 2814 2815 if (true_target == nullptr && false_target == nullptr) { 2816 // Nothing to do. The code always falls through. 2817 return; 2818 } else if (cond->IsIntConstant()) { 2819 // Constant condition, statically compared against "true" (integer value 1). 2820 if (cond->AsIntConstant()->IsTrue()) { 2821 if (true_target != nullptr) { 2822 __ B(true_target); 2823 } 2824 } else { 2825 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 2826 if (false_target != nullptr) { 2827 __ B(false_target); 2828 } 2829 } 2830 return; 2831 } 2832 2833 // The following code generates these patterns: 2834 // (1) true_target == nullptr && false_target != nullptr 2835 // - opposite condition true => branch to false_target 2836 // (2) true_target != nullptr && false_target == nullptr 2837 // - condition true => branch to true_target 2838 // (3) true_target != nullptr && false_target != nullptr 2839 // - condition true => branch to true_target 2840 // - branch to false_target 2841 if (IsBooleanValueOrMaterializedCondition(cond)) { 2842 // The condition instruction has been materialized, compare the output to 0. 2843 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 2844 DCHECK(cond_val.IsRegister()); 2845 if (true_target == nullptr) { 2846 __ Cbz(InputRegisterAt(instruction, condition_input_index), false_target); 2847 } else { 2848 __ Cbnz(InputRegisterAt(instruction, condition_input_index), true_target); 2849 } 2850 } else { 2851 // The condition instruction has not been materialized, use its inputs as 2852 // the comparison and its condition as the branch condition. 2853 HCondition* condition = cond->AsCondition(); 2854 2855 Primitive::Type type = condition->InputAt(0)->GetType(); 2856 if (Primitive::IsFloatingPointType(type)) { 2857 GenerateFcmp(condition); 2858 if (true_target == nullptr) { 2859 IfCondition opposite_condition = condition->GetOppositeCondition(); 2860 __ B(ARM64FPCondition(opposite_condition, condition->IsGtBias()), false_target); 2861 } else { 2862 __ B(ARM64FPCondition(condition->GetCondition(), condition->IsGtBias()), true_target); 2863 } 2864 } else { 2865 // Integer cases. 2866 Register lhs = InputRegisterAt(condition, 0); 2867 Operand rhs = InputOperandAt(condition, 1); 2868 2869 Condition arm64_cond; 2870 vixl::Label* non_fallthrough_target; 2871 if (true_target == nullptr) { 2872 arm64_cond = ARM64Condition(condition->GetOppositeCondition()); 2873 non_fallthrough_target = false_target; 2874 } else { 2875 arm64_cond = ARM64Condition(condition->GetCondition()); 2876 non_fallthrough_target = true_target; 2877 } 2878 2879 if ((arm64_cond == eq || arm64_cond == ne || arm64_cond == lt || arm64_cond == ge) && 2880 rhs.IsImmediate() && (rhs.immediate() == 0)) { 2881 switch (arm64_cond) { 2882 case eq: 2883 __ Cbz(lhs, non_fallthrough_target); 2884 break; 2885 case ne: 2886 __ Cbnz(lhs, non_fallthrough_target); 2887 break; 2888 case lt: 2889 // Test the sign bit and branch accordingly. 2890 __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target); 2891 break; 2892 case ge: 2893 // Test the sign bit and branch accordingly. 2894 __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target); 2895 break; 2896 default: 2897 // Without the `static_cast` the compiler throws an error for 2898 // `-Werror=sign-promo`. 2899 LOG(FATAL) << "Unexpected condition: " << static_cast<int>(arm64_cond); 2900 } 2901 } else { 2902 __ Cmp(lhs, rhs); 2903 __ B(arm64_cond, non_fallthrough_target); 2904 } 2905 } 2906 } 2907 2908 // If neither branch falls through (case 3), the conditional branch to `true_target` 2909 // was already emitted (case 2) and we need to emit a jump to `false_target`. 2910 if (true_target != nullptr && false_target != nullptr) { 2911 __ B(false_target); 2912 } 2913 2914 if (fallthrough_target.IsLinked()) { 2915 __ Bind(&fallthrough_target); 2916 } 2917 } 2918 2919 void LocationsBuilderARM64::VisitIf(HIf* if_instr) { 2920 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 2921 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 2922 locations->SetInAt(0, Location::RequiresRegister()); 2923 } 2924 } 2925 2926 void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) { 2927 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 2928 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 2929 vixl::Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 2930 nullptr : codegen_->GetLabelOf(true_successor); 2931 vixl::Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 2932 nullptr : codegen_->GetLabelOf(false_successor); 2933 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 2934 } 2935 2936 void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) { 2937 LocationSummary* locations = new (GetGraph()->GetArena()) 2938 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 2939 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 2940 locations->SetInAt(0, Location::RequiresRegister()); 2941 } 2942 } 2943 2944 void InstructionCodeGeneratorARM64::VisitDeoptimize(HDeoptimize* deoptimize) { 2945 SlowPathCodeARM64* slow_path = 2946 deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM64>(deoptimize); 2947 GenerateTestAndBranch(deoptimize, 2948 /* condition_input_index */ 0, 2949 slow_path->GetEntryLabel(), 2950 /* false_target */ nullptr); 2951 } 2952 2953 enum SelectVariant { 2954 kCsel, 2955 kCselFalseConst, 2956 kCselTrueConst, 2957 kFcsel, 2958 }; 2959 2960 static inline bool IsConditionOnFloatingPointValues(HInstruction* condition) { 2961 return condition->IsCondition() && 2962 Primitive::IsFloatingPointType(condition->InputAt(0)->GetType()); 2963 } 2964 2965 static inline bool IsRecognizedCselConstant(HInstruction* constant) { 2966 if (constant->IsConstant()) { 2967 int64_t value = Int64FromConstant(constant->AsConstant()); 2968 if ((value == -1) || (value == 0) || (value == 1)) { 2969 return true; 2970 } 2971 } 2972 return false; 2973 } 2974 2975 static inline SelectVariant GetSelectVariant(HSelect* select) { 2976 if (Primitive::IsFloatingPointType(select->GetType())) { 2977 return kFcsel; 2978 } else if (IsRecognizedCselConstant(select->GetFalseValue())) { 2979 return kCselFalseConst; 2980 } else if (IsRecognizedCselConstant(select->GetTrueValue())) { 2981 return kCselTrueConst; 2982 } else { 2983 return kCsel; 2984 } 2985 } 2986 2987 static inline bool HasSwappedInputs(SelectVariant variant) { 2988 return variant == kCselTrueConst; 2989 } 2990 2991 static inline Condition GetConditionForSelect(HCondition* condition, SelectVariant variant) { 2992 IfCondition cond = HasSwappedInputs(variant) ? condition->GetOppositeCondition() 2993 : condition->GetCondition(); 2994 return IsConditionOnFloatingPointValues(condition) ? ARM64FPCondition(cond, condition->IsGtBias()) 2995 : ARM64Condition(cond); 2996 } 2997 2998 void LocationsBuilderARM64::VisitSelect(HSelect* select) { 2999 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 3000 switch (GetSelectVariant(select)) { 3001 case kCsel: 3002 locations->SetInAt(0, Location::RequiresRegister()); 3003 locations->SetInAt(1, Location::RequiresRegister()); 3004 locations->SetOut(Location::RequiresRegister()); 3005 break; 3006 case kCselFalseConst: 3007 locations->SetInAt(0, Location::ConstantLocation(select->InputAt(0)->AsConstant())); 3008 locations->SetInAt(1, Location::RequiresRegister()); 3009 locations->SetOut(Location::RequiresRegister()); 3010 break; 3011 case kCselTrueConst: 3012 locations->SetInAt(0, Location::RequiresRegister()); 3013 locations->SetInAt(1, Location::ConstantLocation(select->InputAt(1)->AsConstant())); 3014 locations->SetOut(Location::RequiresRegister()); 3015 break; 3016 case kFcsel: 3017 locations->SetInAt(0, Location::RequiresFpuRegister()); 3018 locations->SetInAt(1, Location::RequiresFpuRegister()); 3019 locations->SetOut(Location::RequiresFpuRegister()); 3020 break; 3021 } 3022 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 3023 locations->SetInAt(2, Location::RequiresRegister()); 3024 } 3025 } 3026 3027 void InstructionCodeGeneratorARM64::VisitSelect(HSelect* select) { 3028 HInstruction* cond = select->GetCondition(); 3029 SelectVariant variant = GetSelectVariant(select); 3030 Condition csel_cond; 3031 3032 if (IsBooleanValueOrMaterializedCondition(cond)) { 3033 if (cond->IsCondition() && cond->GetNext() == select) { 3034 // Condition codes set from previous instruction. 3035 csel_cond = GetConditionForSelect(cond->AsCondition(), variant); 3036 } else { 3037 __ Cmp(InputRegisterAt(select, 2), 0); 3038 csel_cond = HasSwappedInputs(variant) ? eq : ne; 3039 } 3040 } else if (IsConditionOnFloatingPointValues(cond)) { 3041 GenerateFcmp(cond); 3042 csel_cond = GetConditionForSelect(cond->AsCondition(), variant); 3043 } else { 3044 __ Cmp(InputRegisterAt(cond, 0), InputOperandAt(cond, 1)); 3045 csel_cond = GetConditionForSelect(cond->AsCondition(), variant); 3046 } 3047 3048 switch (variant) { 3049 case kCsel: 3050 case kCselFalseConst: 3051 __ Csel(OutputRegister(select), 3052 InputRegisterAt(select, 1), 3053 InputOperandAt(select, 0), 3054 csel_cond); 3055 break; 3056 case kCselTrueConst: 3057 __ Csel(OutputRegister(select), 3058 InputRegisterAt(select, 0), 3059 InputOperandAt(select, 1), 3060 csel_cond); 3061 break; 3062 case kFcsel: 3063 __ Fcsel(OutputFPRegister(select), 3064 InputFPRegisterAt(select, 1), 3065 InputFPRegisterAt(select, 0), 3066 csel_cond); 3067 break; 3068 } 3069 } 3070 3071 void LocationsBuilderARM64::VisitNativeDebugInfo(HNativeDebugInfo* info) { 3072 new (GetGraph()->GetArena()) LocationSummary(info); 3073 } 3074 3075 void InstructionCodeGeneratorARM64::VisitNativeDebugInfo(HNativeDebugInfo*) { 3076 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 3077 } 3078 3079 void CodeGeneratorARM64::GenerateNop() { 3080 __ Nop(); 3081 } 3082 3083 void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 3084 HandleFieldGet(instruction); 3085 } 3086 3087 void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 3088 HandleFieldGet(instruction, instruction->GetFieldInfo()); 3089 } 3090 3091 void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 3092 HandleFieldSet(instruction); 3093 } 3094 3095 void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 3096 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 3097 } 3098 3099 static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { 3100 return kEmitCompilerReadBarrier && 3101 (kUseBakerReadBarrier || 3102 type_check_kind == TypeCheckKind::kAbstractClassCheck || 3103 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 3104 type_check_kind == TypeCheckKind::kArrayObjectCheck); 3105 } 3106 3107 void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) { 3108 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3109 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 3110 switch (type_check_kind) { 3111 case TypeCheckKind::kExactCheck: 3112 case TypeCheckKind::kAbstractClassCheck: 3113 case TypeCheckKind::kClassHierarchyCheck: 3114 case TypeCheckKind::kArrayObjectCheck: 3115 call_kind = 3116 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 3117 break; 3118 case TypeCheckKind::kArrayCheck: 3119 case TypeCheckKind::kUnresolvedCheck: 3120 case TypeCheckKind::kInterfaceCheck: 3121 call_kind = LocationSummary::kCallOnSlowPath; 3122 break; 3123 } 3124 3125 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 3126 locations->SetInAt(0, Location::RequiresRegister()); 3127 locations->SetInAt(1, Location::RequiresRegister()); 3128 // The "out" register is used as a temporary, so it overlaps with the inputs. 3129 // Note that TypeCheckSlowPathARM64 uses this register too. 3130 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3131 // When read barriers are enabled, we need a temporary register for 3132 // some cases. 3133 if (TypeCheckNeedsATemporary(type_check_kind)) { 3134 locations->AddTemp(Location::RequiresRegister()); 3135 } 3136 } 3137 3138 void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { 3139 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 3140 LocationSummary* locations = instruction->GetLocations(); 3141 Location obj_loc = locations->InAt(0); 3142 Register obj = InputRegisterAt(instruction, 0); 3143 Register cls = InputRegisterAt(instruction, 1); 3144 Location out_loc = locations->Out(); 3145 Register out = OutputRegister(instruction); 3146 Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? 3147 locations->GetTemp(0) : 3148 Location::NoLocation(); 3149 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 3150 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 3151 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 3152 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 3153 3154 vixl::Label done, zero; 3155 SlowPathCodeARM64* slow_path = nullptr; 3156 3157 // Return 0 if `obj` is null. 3158 // Avoid null check if we know `obj` is not null. 3159 if (instruction->MustDoNullCheck()) { 3160 __ Cbz(obj, &zero); 3161 } 3162 3163 // /* HeapReference<Class> */ out = obj->klass_ 3164 GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc); 3165 3166 switch (type_check_kind) { 3167 case TypeCheckKind::kExactCheck: { 3168 __ Cmp(out, cls); 3169 __ Cset(out, eq); 3170 if (zero.IsLinked()) { 3171 __ B(&done); 3172 } 3173 break; 3174 } 3175 3176 case TypeCheckKind::kAbstractClassCheck: { 3177 // If the class is abstract, we eagerly fetch the super class of the 3178 // object to avoid doing a comparison we know will fail. 3179 vixl::Label loop, success; 3180 __ Bind(&loop); 3181 // /* HeapReference<Class> */ out = out->super_class_ 3182 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc); 3183 // If `out` is null, we use it for the result, and jump to `done`. 3184 __ Cbz(out, &done); 3185 __ Cmp(out, cls); 3186 __ B(ne, &loop); 3187 __ Mov(out, 1); 3188 if (zero.IsLinked()) { 3189 __ B(&done); 3190 } 3191 break; 3192 } 3193 3194 case TypeCheckKind::kClassHierarchyCheck: { 3195 // Walk over the class hierarchy to find a match. 3196 vixl::Label loop, success; 3197 __ Bind(&loop); 3198 __ Cmp(out, cls); 3199 __ B(eq, &success); 3200 // /* HeapReference<Class> */ out = out->super_class_ 3201 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc); 3202 __ Cbnz(out, &loop); 3203 // If `out` is null, we use it for the result, and jump to `done`. 3204 __ B(&done); 3205 __ Bind(&success); 3206 __ Mov(out, 1); 3207 if (zero.IsLinked()) { 3208 __ B(&done); 3209 } 3210 break; 3211 } 3212 3213 case TypeCheckKind::kArrayObjectCheck: { 3214 // Do an exact check. 3215 vixl::Label exact_check; 3216 __ Cmp(out, cls); 3217 __ B(eq, &exact_check); 3218 // Otherwise, we need to check that the object's class is a non-primitive array. 3219 // /* HeapReference<Class> */ out = out->component_type_ 3220 GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, maybe_temp_loc); 3221 // If `out` is null, we use it for the result, and jump to `done`. 3222 __ Cbz(out, &done); 3223 __ Ldrh(out, HeapOperand(out, primitive_offset)); 3224 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 3225 __ Cbnz(out, &zero); 3226 __ Bind(&exact_check); 3227 __ Mov(out, 1); 3228 __ B(&done); 3229 break; 3230 } 3231 3232 case TypeCheckKind::kArrayCheck: { 3233 __ Cmp(out, cls); 3234 DCHECK(locations->OnlyCallsOnSlowPath()); 3235 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction, 3236 /* is_fatal */ false); 3237 codegen_->AddSlowPath(slow_path); 3238 __ B(ne, slow_path->GetEntryLabel()); 3239 __ Mov(out, 1); 3240 if (zero.IsLinked()) { 3241 __ B(&done); 3242 } 3243 break; 3244 } 3245 3246 case TypeCheckKind::kUnresolvedCheck: 3247 case TypeCheckKind::kInterfaceCheck: { 3248 // Note that we indeed only call on slow path, but we always go 3249 // into the slow path for the unresolved and interface check 3250 // cases. 3251 // 3252 // We cannot directly call the InstanceofNonTrivial runtime 3253 // entry point without resorting to a type checking slow path 3254 // here (i.e. by calling InvokeRuntime directly), as it would 3255 // require to assign fixed registers for the inputs of this 3256 // HInstanceOf instruction (following the runtime calling 3257 // convention), which might be cluttered by the potential first 3258 // read barrier emission at the beginning of this method. 3259 // 3260 // TODO: Introduce a new runtime entry point taking the object 3261 // to test (instead of its class) as argument, and let it deal 3262 // with the read barrier issues. This will let us refactor this 3263 // case of the `switch` code as it was previously (with a direct 3264 // call to the runtime not using a type checking slow path). 3265 // This should also be beneficial for the other cases above. 3266 DCHECK(locations->OnlyCallsOnSlowPath()); 3267 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction, 3268 /* is_fatal */ false); 3269 codegen_->AddSlowPath(slow_path); 3270 __ B(slow_path->GetEntryLabel()); 3271 if (zero.IsLinked()) { 3272 __ B(&done); 3273 } 3274 break; 3275 } 3276 } 3277 3278 if (zero.IsLinked()) { 3279 __ Bind(&zero); 3280 __ Mov(out, 0); 3281 } 3282 3283 if (done.IsLinked()) { 3284 __ Bind(&done); 3285 } 3286 3287 if (slow_path != nullptr) { 3288 __ Bind(slow_path->GetExitLabel()); 3289 } 3290 } 3291 3292 void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) { 3293 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3294 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 3295 3296 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 3297 switch (type_check_kind) { 3298 case TypeCheckKind::kExactCheck: 3299 case TypeCheckKind::kAbstractClassCheck: 3300 case TypeCheckKind::kClassHierarchyCheck: 3301 case TypeCheckKind::kArrayObjectCheck: 3302 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 3303 LocationSummary::kCallOnSlowPath : 3304 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 3305 break; 3306 case TypeCheckKind::kArrayCheck: 3307 case TypeCheckKind::kUnresolvedCheck: 3308 case TypeCheckKind::kInterfaceCheck: 3309 call_kind = LocationSummary::kCallOnSlowPath; 3310 break; 3311 } 3312 3313 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 3314 locations->SetInAt(0, Location::RequiresRegister()); 3315 locations->SetInAt(1, Location::RequiresRegister()); 3316 // Note that TypeCheckSlowPathARM64 uses this "temp" register too. 3317 locations->AddTemp(Location::RequiresRegister()); 3318 // When read barriers are enabled, we need an additional temporary 3319 // register for some cases. 3320 if (TypeCheckNeedsATemporary(type_check_kind)) { 3321 locations->AddTemp(Location::RequiresRegister()); 3322 } 3323 } 3324 3325 void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { 3326 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 3327 LocationSummary* locations = instruction->GetLocations(); 3328 Location obj_loc = locations->InAt(0); 3329 Register obj = InputRegisterAt(instruction, 0); 3330 Register cls = InputRegisterAt(instruction, 1); 3331 Location temp_loc = locations->GetTemp(0); 3332 Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? 3333 locations->GetTemp(1) : 3334 Location::NoLocation(); 3335 Register temp = WRegisterFrom(temp_loc); 3336 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 3337 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 3338 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 3339 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 3340 3341 bool is_type_check_slow_path_fatal = 3342 (type_check_kind == TypeCheckKind::kExactCheck || 3343 type_check_kind == TypeCheckKind::kAbstractClassCheck || 3344 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 3345 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 3346 !instruction->CanThrowIntoCatchBlock(); 3347 SlowPathCodeARM64* type_check_slow_path = 3348 new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction, 3349 is_type_check_slow_path_fatal); 3350 codegen_->AddSlowPath(type_check_slow_path); 3351 3352 vixl::Label done; 3353 // Avoid null check if we know obj is not null. 3354 if (instruction->MustDoNullCheck()) { 3355 __ Cbz(obj, &done); 3356 } 3357 3358 // /* HeapReference<Class> */ temp = obj->klass_ 3359 GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 3360 3361 switch (type_check_kind) { 3362 case TypeCheckKind::kExactCheck: 3363 case TypeCheckKind::kArrayCheck: { 3364 __ Cmp(temp, cls); 3365 // Jump to slow path for throwing the exception or doing a 3366 // more involved array check. 3367 __ B(ne, type_check_slow_path->GetEntryLabel()); 3368 break; 3369 } 3370 3371 case TypeCheckKind::kAbstractClassCheck: { 3372 // If the class is abstract, we eagerly fetch the super class of the 3373 // object to avoid doing a comparison we know will fail. 3374 vixl::Label loop, compare_classes; 3375 __ Bind(&loop); 3376 // /* HeapReference<Class> */ temp = temp->super_class_ 3377 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc); 3378 3379 // If the class reference currently in `temp` is not null, jump 3380 // to the `compare_classes` label to compare it with the checked 3381 // class. 3382 __ Cbnz(temp, &compare_classes); 3383 // Otherwise, jump to the slow path to throw the exception. 3384 // 3385 // But before, move back the object's class into `temp` before 3386 // going into the slow path, as it has been overwritten in the 3387 // meantime. 3388 // /* HeapReference<Class> */ temp = obj->klass_ 3389 GenerateReferenceLoadTwoRegisters( 3390 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 3391 __ B(type_check_slow_path->GetEntryLabel()); 3392 3393 __ Bind(&compare_classes); 3394 __ Cmp(temp, cls); 3395 __ B(ne, &loop); 3396 break; 3397 } 3398 3399 case TypeCheckKind::kClassHierarchyCheck: { 3400 // Walk over the class hierarchy to find a match. 3401 vixl::Label loop; 3402 __ Bind(&loop); 3403 __ Cmp(temp, cls); 3404 __ B(eq, &done); 3405 3406 // /* HeapReference<Class> */ temp = temp->super_class_ 3407 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc); 3408 3409 // If the class reference currently in `temp` is not null, jump 3410 // back at the beginning of the loop. 3411 __ Cbnz(temp, &loop); 3412 // Otherwise, jump to the slow path to throw the exception. 3413 // 3414 // But before, move back the object's class into `temp` before 3415 // going into the slow path, as it has been overwritten in the 3416 // meantime. 3417 // /* HeapReference<Class> */ temp = obj->klass_ 3418 GenerateReferenceLoadTwoRegisters( 3419 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 3420 __ B(type_check_slow_path->GetEntryLabel()); 3421 break; 3422 } 3423 3424 case TypeCheckKind::kArrayObjectCheck: { 3425 // Do an exact check. 3426 vixl::Label check_non_primitive_component_type; 3427 __ Cmp(temp, cls); 3428 __ B(eq, &done); 3429 3430 // Otherwise, we need to check that the object's class is a non-primitive array. 3431 // /* HeapReference<Class> */ temp = temp->component_type_ 3432 GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, maybe_temp2_loc); 3433 3434 // If the component type is not null (i.e. the object is indeed 3435 // an array), jump to label `check_non_primitive_component_type` 3436 // to further check that this component type is not a primitive 3437 // type. 3438 __ Cbnz(temp, &check_non_primitive_component_type); 3439 // Otherwise, jump to the slow path to throw the exception. 3440 // 3441 // But before, move back the object's class into `temp` before 3442 // going into the slow path, as it has been overwritten in the 3443 // meantime. 3444 // /* HeapReference<Class> */ temp = obj->klass_ 3445 GenerateReferenceLoadTwoRegisters( 3446 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 3447 __ B(type_check_slow_path->GetEntryLabel()); 3448 3449 __ Bind(&check_non_primitive_component_type); 3450 __ Ldrh(temp, HeapOperand(temp, primitive_offset)); 3451 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 3452 __ Cbz(temp, &done); 3453 // Same comment as above regarding `temp` and the slow path. 3454 // /* HeapReference<Class> */ temp = obj->klass_ 3455 GenerateReferenceLoadTwoRegisters( 3456 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 3457 __ B(type_check_slow_path->GetEntryLabel()); 3458 break; 3459 } 3460 3461 case TypeCheckKind::kUnresolvedCheck: 3462 case TypeCheckKind::kInterfaceCheck: 3463 // We always go into the type check slow path for the unresolved 3464 // and interface check cases. 3465 // 3466 // We cannot directly call the CheckCast runtime entry point 3467 // without resorting to a type checking slow path here (i.e. by 3468 // calling InvokeRuntime directly), as it would require to 3469 // assign fixed registers for the inputs of this HInstanceOf 3470 // instruction (following the runtime calling convention), which 3471 // might be cluttered by the potential first read barrier 3472 // emission at the beginning of this method. 3473 // 3474 // TODO: Introduce a new runtime entry point taking the object 3475 // to test (instead of its class) as argument, and let it deal 3476 // with the read barrier issues. This will let us refactor this 3477 // case of the `switch` code as it was previously (with a direct 3478 // call to the runtime not using a type checking slow path). 3479 // This should also be beneficial for the other cases above. 3480 __ B(type_check_slow_path->GetEntryLabel()); 3481 break; 3482 } 3483 __ Bind(&done); 3484 3485 __ Bind(type_check_slow_path->GetExitLabel()); 3486 } 3487 3488 void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) { 3489 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant); 3490 locations->SetOut(Location::ConstantLocation(constant)); 3491 } 3492 3493 void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 3494 // Will be generated at use site. 3495 } 3496 3497 void LocationsBuilderARM64::VisitNullConstant(HNullConstant* constant) { 3498 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant); 3499 locations->SetOut(Location::ConstantLocation(constant)); 3500 } 3501 3502 void InstructionCodeGeneratorARM64::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 3503 // Will be generated at use site. 3504 } 3505 3506 void LocationsBuilderARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3507 // The trampoline uses the same calling convention as dex calling conventions, 3508 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 3509 // the method_idx. 3510 HandleInvoke(invoke); 3511 } 3512 3513 void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3514 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 3515 } 3516 3517 void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) { 3518 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor; 3519 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 3520 } 3521 3522 void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) { 3523 HandleInvoke(invoke); 3524 } 3525 3526 void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) { 3527 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 3528 LocationSummary* locations = invoke->GetLocations(); 3529 Register temp = XRegisterFrom(locations->GetTemp(0)); 3530 Location receiver = locations->InAt(0); 3531 Offset class_offset = mirror::Object::ClassOffset(); 3532 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize); 3533 3534 // The register ip1 is required to be used for the hidden argument in 3535 // art_quick_imt_conflict_trampoline, so prevent VIXL from using it. 3536 MacroAssembler* masm = GetVIXLAssembler(); 3537 UseScratchRegisterScope scratch_scope(masm); 3538 BlockPoolsScope block_pools(masm); 3539 scratch_scope.Exclude(ip1); 3540 __ Mov(ip1, invoke->GetDexMethodIndex()); 3541 3542 if (receiver.IsStackSlot()) { 3543 __ Ldr(temp.W(), StackOperandFrom(receiver)); 3544 // /* HeapReference<Class> */ temp = temp->klass_ 3545 __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset)); 3546 } else { 3547 // /* HeapReference<Class> */ temp = receiver->klass_ 3548 __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset)); 3549 } 3550 codegen_->MaybeRecordImplicitNullCheck(invoke); 3551 // Instead of simply (possibly) unpoisoning `temp` here, we should 3552 // emit a read barrier for the previous class reference load. 3553 // However this is not required in practice, as this is an 3554 // intermediate/temporary reference and because the current 3555 // concurrent copying collector keeps the from-space memory 3556 // intact/accessible until the end of the marking phase (the 3557 // concurrent copying collector may not in the future). 3558 GetAssembler()->MaybeUnpoisonHeapReference(temp.W()); 3559 __ Ldr(temp, 3560 MemOperand(temp, mirror::Class::ImtPtrOffset(kArm64PointerSize).Uint32Value())); 3561 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 3562 invoke->GetImtIndex() % ImTable::kSize, kArm64PointerSize)); 3563 // temp = temp->GetImtEntryAt(method_offset); 3564 __ Ldr(temp, MemOperand(temp, method_offset)); 3565 // lr = temp->GetEntryPoint(); 3566 __ Ldr(lr, MemOperand(temp, entry_point.Int32Value())); 3567 // lr(); 3568 __ Blr(lr); 3569 DCHECK(!codegen_->IsLeafMethod()); 3570 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3571 } 3572 3573 void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3574 IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena()); 3575 if (intrinsic.TryDispatch(invoke)) { 3576 return; 3577 } 3578 3579 HandleInvoke(invoke); 3580 } 3581 3582 void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3583 // Explicit clinit checks triggered by static invokes must have been pruned by 3584 // art::PrepareForRegisterAllocation. 3585 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3586 3587 IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena()); 3588 if (intrinsic.TryDispatch(invoke)) { 3589 return; 3590 } 3591 3592 HandleInvoke(invoke); 3593 } 3594 3595 static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) { 3596 if (invoke->GetLocations()->Intrinsified()) { 3597 IntrinsicCodeGeneratorARM64 intrinsic(codegen); 3598 intrinsic.Dispatch(invoke); 3599 return true; 3600 } 3601 return false; 3602 } 3603 3604 HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStaticOrDirectDispatch( 3605 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 3606 MethodReference target_method ATTRIBUTE_UNUSED) { 3607 // On ARM64 we support all dispatch types. 3608 return desired_dispatch_info; 3609 } 3610 3611 void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 3612 // For better instruction scheduling we load the direct code pointer before the method pointer. 3613 bool direct_code_loaded = false; 3614 switch (invoke->GetCodePtrLocation()) { 3615 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 3616 // LR = code address from literal pool with link-time patch. 3617 __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); 3618 direct_code_loaded = true; 3619 break; 3620 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 3621 // LR = invoke->GetDirectCodePtr(); 3622 __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr())); 3623 direct_code_loaded = true; 3624 break; 3625 default: 3626 break; 3627 } 3628 3629 // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention. 3630 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 3631 switch (invoke->GetMethodLoadKind()) { 3632 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: 3633 // temp = thread->string_init_entrypoint 3634 __ Ldr(XRegisterFrom(temp), MemOperand(tr, invoke->GetStringInitOffset())); 3635 break; 3636 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 3637 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 3638 break; 3639 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 3640 // Load method address from literal pool. 3641 __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress())); 3642 break; 3643 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: 3644 // Load method address from literal pool with a link-time patch. 3645 __ Ldr(XRegisterFrom(temp), 3646 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); 3647 break; 3648 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 3649 // Add ADRP with its PC-relative DexCache access patch. 3650 const DexFile& dex_file = *invoke->GetTargetMethod().dex_file; 3651 uint32_t element_offset = invoke->GetDexCacheArrayOffset(); 3652 vixl::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); 3653 { 3654 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 3655 __ Bind(adrp_label); 3656 __ adrp(XRegisterFrom(temp), /* offset placeholder */ 0); 3657 } 3658 // Add LDR with its PC-relative DexCache access patch. 3659 vixl::Label* ldr_label = 3660 NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); 3661 { 3662 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 3663 __ Bind(ldr_label); 3664 __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), /* offset placeholder */ 0)); 3665 } 3666 break; 3667 } 3668 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 3669 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 3670 Register reg = XRegisterFrom(temp); 3671 Register method_reg; 3672 if (current_method.IsRegister()) { 3673 method_reg = XRegisterFrom(current_method); 3674 } else { 3675 DCHECK(invoke->GetLocations()->Intrinsified()); 3676 DCHECK(!current_method.IsValid()); 3677 method_reg = reg; 3678 __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset)); 3679 } 3680 3681 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 3682 __ Ldr(reg.X(), 3683 MemOperand(method_reg.X(), 3684 ArtMethod::DexCacheResolvedMethodsOffset(kArm64WordSize).Int32Value())); 3685 // temp = temp[index_in_cache]; 3686 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 3687 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 3688 __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache))); 3689 break; 3690 } 3691 } 3692 3693 switch (invoke->GetCodePtrLocation()) { 3694 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 3695 __ Bl(&frame_entry_label_); 3696 break; 3697 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: { 3698 relative_call_patches_.emplace_back(invoke->GetTargetMethod()); 3699 vixl::Label* label = &relative_call_patches_.back().label; 3700 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 3701 __ Bind(label); 3702 __ bl(0); // Branch and link to itself. This will be overriden at link time. 3703 break; 3704 } 3705 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 3706 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 3707 // LR prepared above for better instruction scheduling. 3708 DCHECK(direct_code_loaded); 3709 // lr() 3710 __ Blr(lr); 3711 break; 3712 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 3713 // LR = callee_method->entry_point_from_quick_compiled_code_; 3714 __ Ldr(lr, MemOperand( 3715 XRegisterFrom(callee_method), 3716 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value())); 3717 // lr() 3718 __ Blr(lr); 3719 break; 3720 } 3721 3722 DCHECK(!IsLeafMethod()); 3723 } 3724 3725 void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { 3726 // Use the calling convention instead of the location of the receiver, as 3727 // intrinsics may have put the receiver in a different register. In the intrinsics 3728 // slow path, the arguments have been moved to the right place, so here we are 3729 // guaranteed that the receiver is the first register of the calling convention. 3730 InvokeDexCallingConvention calling_convention; 3731 Register receiver = calling_convention.GetRegisterAt(0); 3732 Register temp = XRegisterFrom(temp_in); 3733 size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 3734 invoke->GetVTableIndex(), kArm64PointerSize).SizeValue(); 3735 Offset class_offset = mirror::Object::ClassOffset(); 3736 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize); 3737 3738 BlockPoolsScope block_pools(GetVIXLAssembler()); 3739 3740 DCHECK(receiver.IsRegister()); 3741 // /* HeapReference<Class> */ temp = receiver->klass_ 3742 __ Ldr(temp.W(), HeapOperandFrom(LocationFrom(receiver), class_offset)); 3743 MaybeRecordImplicitNullCheck(invoke); 3744 // Instead of simply (possibly) unpoisoning `temp` here, we should 3745 // emit a read barrier for the previous class reference load. 3746 // intermediate/temporary reference and because the current 3747 // concurrent copying collector keeps the from-space memory 3748 // intact/accessible until the end of the marking phase (the 3749 // concurrent copying collector may not in the future). 3750 GetAssembler()->MaybeUnpoisonHeapReference(temp.W()); 3751 // temp = temp->GetMethodAt(method_offset); 3752 __ Ldr(temp, MemOperand(temp, method_offset)); 3753 // lr = temp->GetEntryPoint(); 3754 __ Ldr(lr, MemOperand(temp, entry_point.SizeValue())); 3755 // lr(); 3756 __ Blr(lr); 3757 } 3758 3759 vixl::Label* CodeGeneratorARM64::NewPcRelativeStringPatch(const DexFile& dex_file, 3760 uint32_t string_index, 3761 vixl::Label* adrp_label) { 3762 return NewPcRelativePatch(dex_file, string_index, adrp_label, &pc_relative_string_patches_); 3763 } 3764 3765 vixl::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, 3766 uint32_t element_offset, 3767 vixl::Label* adrp_label) { 3768 return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_); 3769 } 3770 3771 vixl::Label* CodeGeneratorARM64::NewPcRelativePatch(const DexFile& dex_file, 3772 uint32_t offset_or_index, 3773 vixl::Label* adrp_label, 3774 ArenaDeque<PcRelativePatchInfo>* patches) { 3775 // Add a patch entry and return the label. 3776 patches->emplace_back(dex_file, offset_or_index); 3777 PcRelativePatchInfo* info = &patches->back(); 3778 vixl::Label* label = &info->label; 3779 // If adrp_label is null, this is the ADRP patch and needs to point to its own label. 3780 info->pc_insn_label = (adrp_label != nullptr) ? adrp_label : label; 3781 return label; 3782 } 3783 3784 vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageStringLiteral( 3785 const DexFile& dex_file, uint32_t string_index) { 3786 return boot_image_string_patches_.GetOrCreate( 3787 StringReference(&dex_file, string_index), 3788 [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); 3789 } 3790 3791 vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(uint64_t address) { 3792 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 3793 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 3794 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 3795 } 3796 3797 vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral(uint64_t address) { 3798 return DeduplicateUint64Literal(address); 3799 } 3800 3801 void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 3802 DCHECK(linker_patches->empty()); 3803 size_t size = 3804 method_patches_.size() + 3805 call_patches_.size() + 3806 relative_call_patches_.size() + 3807 pc_relative_dex_cache_patches_.size() + 3808 boot_image_string_patches_.size() + 3809 pc_relative_string_patches_.size() + 3810 boot_image_address_patches_.size(); 3811 linker_patches->reserve(size); 3812 for (const auto& entry : method_patches_) { 3813 const MethodReference& target_method = entry.first; 3814 vixl::Literal<uint64_t>* literal = entry.second; 3815 linker_patches->push_back(LinkerPatch::MethodPatch(literal->offset(), 3816 target_method.dex_file, 3817 target_method.dex_method_index)); 3818 } 3819 for (const auto& entry : call_patches_) { 3820 const MethodReference& target_method = entry.first; 3821 vixl::Literal<uint64_t>* literal = entry.second; 3822 linker_patches->push_back(LinkerPatch::CodePatch(literal->offset(), 3823 target_method.dex_file, 3824 target_method.dex_method_index)); 3825 } 3826 for (const MethodPatchInfo<vixl::Label>& info : relative_call_patches_) { 3827 linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location(), 3828 info.target_method.dex_file, 3829 info.target_method.dex_method_index)); 3830 } 3831 for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { 3832 linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(), 3833 &info.target_dex_file, 3834 info.pc_insn_label->location(), 3835 info.offset_or_index)); 3836 } 3837 for (const auto& entry : boot_image_string_patches_) { 3838 const StringReference& target_string = entry.first; 3839 vixl::Literal<uint32_t>* literal = entry.second; 3840 linker_patches->push_back(LinkerPatch::StringPatch(literal->offset(), 3841 target_string.dex_file, 3842 target_string.string_index)); 3843 } 3844 for (const PcRelativePatchInfo& info : pc_relative_string_patches_) { 3845 linker_patches->push_back(LinkerPatch::RelativeStringPatch(info.label.location(), 3846 &info.target_dex_file, 3847 info.pc_insn_label->location(), 3848 info.offset_or_index)); 3849 } 3850 for (const auto& entry : boot_image_address_patches_) { 3851 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 3852 vixl::Literal<uint32_t>* literal = entry.second; 3853 linker_patches->push_back(LinkerPatch::RecordPosition(literal->offset())); 3854 } 3855 } 3856 3857 vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value, 3858 Uint32ToLiteralMap* map) { 3859 return map->GetOrCreate( 3860 value, 3861 [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); }); 3862 } 3863 3864 vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) { 3865 return uint64_literals_.GetOrCreate( 3866 value, 3867 [this, value]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(value); }); 3868 } 3869 3870 vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral( 3871 MethodReference target_method, 3872 MethodToLiteralMap* map) { 3873 return map->GetOrCreate( 3874 target_method, 3875 [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); }); 3876 } 3877 3878 vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral( 3879 MethodReference target_method) { 3880 return DeduplicateMethodLiteral(target_method, &method_patches_); 3881 } 3882 3883 vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral( 3884 MethodReference target_method) { 3885 return DeduplicateMethodLiteral(target_method, &call_patches_); 3886 } 3887 3888 3889 void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3890 // Explicit clinit checks triggered by static invokes must have been pruned by 3891 // art::PrepareForRegisterAllocation. 3892 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3893 3894 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3895 return; 3896 } 3897 3898 BlockPoolsScope block_pools(GetVIXLAssembler()); 3899 LocationSummary* locations = invoke->GetLocations(); 3900 codegen_->GenerateStaticOrDirectCall( 3901 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 3902 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3903 } 3904 3905 void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3906 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3907 return; 3908 } 3909 3910 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 3911 DCHECK(!codegen_->IsLeafMethod()); 3912 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3913 } 3914 3915 void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) { 3916 InvokeRuntimeCallingConvention calling_convention; 3917 CodeGenerator::CreateLoadClassLocationSummary( 3918 cls, 3919 LocationFrom(calling_convention.GetRegisterAt(0)), 3920 LocationFrom(vixl::x0), 3921 /* code_generator_supports_read_barrier */ true); 3922 } 3923 3924 void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) { 3925 if (cls->NeedsAccessCheck()) { 3926 codegen_->MoveConstant(cls->GetLocations()->GetTemp(0), cls->GetTypeIndex()); 3927 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess), 3928 cls, 3929 cls->GetDexPc(), 3930 nullptr); 3931 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 3932 return; 3933 } 3934 3935 Location out_loc = cls->GetLocations()->Out(); 3936 Register out = OutputRegister(cls); 3937 Register current_method = InputRegisterAt(cls, 0); 3938 if (cls->IsReferrersClass()) { 3939 DCHECK(!cls->CanCallRuntime()); 3940 DCHECK(!cls->MustGenerateClinitCheck()); 3941 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 3942 GenerateGcRootFieldLoad( 3943 cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); 3944 } else { 3945 MemberOffset resolved_types_offset = ArtMethod::DexCacheResolvedTypesOffset(kArm64PointerSize); 3946 // /* GcRoot<mirror::Class>[] */ out = 3947 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 3948 __ Ldr(out.X(), MemOperand(current_method, resolved_types_offset.Int32Value())); 3949 // /* GcRoot<mirror::Class> */ out = out[type_index] 3950 GenerateGcRootFieldLoad( 3951 cls, out_loc, out.X(), CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); 3952 3953 if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { 3954 DCHECK(cls->CanCallRuntime()); 3955 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64( 3956 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 3957 codegen_->AddSlowPath(slow_path); 3958 if (!cls->IsInDexCache()) { 3959 __ Cbz(out, slow_path->GetEntryLabel()); 3960 } 3961 if (cls->MustGenerateClinitCheck()) { 3962 GenerateClassInitializationCheck(slow_path, out); 3963 } else { 3964 __ Bind(slow_path->GetExitLabel()); 3965 } 3966 } 3967 } 3968 } 3969 3970 static MemOperand GetExceptionTlsAddress() { 3971 return MemOperand(tr, Thread::ExceptionOffset<kArm64WordSize>().Int32Value()); 3972 } 3973 3974 void LocationsBuilderARM64::VisitLoadException(HLoadException* load) { 3975 LocationSummary* locations = 3976 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 3977 locations->SetOut(Location::RequiresRegister()); 3978 } 3979 3980 void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) { 3981 __ Ldr(OutputRegister(instruction), GetExceptionTlsAddress()); 3982 } 3983 3984 void LocationsBuilderARM64::VisitClearException(HClearException* clear) { 3985 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 3986 } 3987 3988 void InstructionCodeGeneratorARM64::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 3989 __ Str(wzr, GetExceptionTlsAddress()); 3990 } 3991 3992 HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind( 3993 HLoadString::LoadKind desired_string_load_kind) { 3994 if (kEmitCompilerReadBarrier) { 3995 switch (desired_string_load_kind) { 3996 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 3997 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 3998 case HLoadString::LoadKind::kBootImageAddress: 3999 // TODO: Implement for read barrier. 4000 return HLoadString::LoadKind::kDexCacheViaMethod; 4001 default: 4002 break; 4003 } 4004 } 4005 switch (desired_string_load_kind) { 4006 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 4007 DCHECK(!GetCompilerOptions().GetCompilePic()); 4008 break; 4009 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 4010 DCHECK(GetCompilerOptions().GetCompilePic()); 4011 break; 4012 case HLoadString::LoadKind::kBootImageAddress: 4013 break; 4014 case HLoadString::LoadKind::kDexCacheAddress: 4015 DCHECK(Runtime::Current()->UseJitCompilation()); 4016 break; 4017 case HLoadString::LoadKind::kDexCachePcRelative: 4018 DCHECK(!Runtime::Current()->UseJitCompilation()); 4019 break; 4020 case HLoadString::LoadKind::kDexCacheViaMethod: 4021 break; 4022 } 4023 return desired_string_load_kind; 4024 } 4025 4026 void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { 4027 LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) 4028 ? LocationSummary::kCallOnSlowPath 4029 : LocationSummary::kNoCall; 4030 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 4031 if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { 4032 locations->SetInAt(0, Location::RequiresRegister()); 4033 } 4034 locations->SetOut(Location::RequiresRegister()); 4035 } 4036 4037 void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { 4038 Location out_loc = load->GetLocations()->Out(); 4039 Register out = OutputRegister(load); 4040 4041 switch (load->GetLoadKind()) { 4042 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 4043 DCHECK(!kEmitCompilerReadBarrier); 4044 __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 4045 load->GetStringIndex())); 4046 return; // No dex cache slow path. 4047 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 4048 DCHECK(!kEmitCompilerReadBarrier); 4049 // Add ADRP with its PC-relative String patch. 4050 const DexFile& dex_file = load->GetDexFile(); 4051 uint32_t string_index = load->GetStringIndex(); 4052 vixl::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index); 4053 { 4054 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 4055 __ Bind(adrp_label); 4056 __ adrp(out.X(), /* offset placeholder */ 0); 4057 } 4058 // Add ADD with its PC-relative String patch. 4059 vixl::Label* add_label = 4060 codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label); 4061 { 4062 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 4063 __ Bind(add_label); 4064 __ add(out.X(), out.X(), Operand(/* offset placeholder */ 0)); 4065 } 4066 return; // No dex cache slow path. 4067 } 4068 case HLoadString::LoadKind::kBootImageAddress: { 4069 DCHECK(!kEmitCompilerReadBarrier); 4070 DCHECK(load->GetAddress() != 0u && IsUint<32>(load->GetAddress())); 4071 __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(load->GetAddress())); 4072 return; // No dex cache slow path. 4073 } 4074 case HLoadString::LoadKind::kDexCacheAddress: { 4075 DCHECK_NE(load->GetAddress(), 0u); 4076 // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads 4077 // that gives a 16KiB range. To try and reduce the number of literals if we load 4078 // multiple strings, simply split the dex cache address to a 16KiB aligned base 4079 // loaded from a literal and the remaining offset embedded in the load. 4080 static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes."); 4081 DCHECK_ALIGNED(load->GetAddress(), 4u); 4082 constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2; 4083 uint64_t base_address = load->GetAddress() & ~MaxInt<uint64_t>(offset_bits); 4084 uint32_t offset = load->GetAddress() & MaxInt<uint64_t>(offset_bits); 4085 __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address)); 4086 GenerateGcRootFieldLoad(load, out_loc, out.X(), offset); 4087 break; 4088 } 4089 case HLoadString::LoadKind::kDexCachePcRelative: { 4090 // Add ADRP with its PC-relative DexCache access patch. 4091 const DexFile& dex_file = load->GetDexFile(); 4092 uint32_t element_offset = load->GetDexCacheElementOffset(); 4093 vixl::Label* adrp_label = codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); 4094 { 4095 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 4096 __ Bind(adrp_label); 4097 __ adrp(out.X(), /* offset placeholder */ 0); 4098 } 4099 // Add LDR with its PC-relative DexCache access patch. 4100 vixl::Label* ldr_label = 4101 codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); 4102 GenerateGcRootFieldLoad(load, out_loc, out.X(), /* offset placeholder */ 0, ldr_label); 4103 break; 4104 } 4105 case HLoadString::LoadKind::kDexCacheViaMethod: { 4106 Register current_method = InputRegisterAt(load, 0); 4107 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 4108 GenerateGcRootFieldLoad( 4109 load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); 4110 // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ 4111 __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); 4112 // /* GcRoot<mirror::String> */ out = out[string_index] 4113 GenerateGcRootFieldLoad( 4114 load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())); 4115 break; 4116 } 4117 default: 4118 LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); 4119 UNREACHABLE(); 4120 } 4121 4122 if (!load->IsInDexCache()) { 4123 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load); 4124 codegen_->AddSlowPath(slow_path); 4125 __ Cbz(out, slow_path->GetEntryLabel()); 4126 __ Bind(slow_path->GetExitLabel()); 4127 } 4128 } 4129 4130 void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) { 4131 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant); 4132 locations->SetOut(Location::ConstantLocation(constant)); 4133 } 4134 4135 void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 4136 // Will be generated at use site. 4137 } 4138 4139 void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) { 4140 LocationSummary* locations = 4141 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); 4142 InvokeRuntimeCallingConvention calling_convention; 4143 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 4144 } 4145 4146 void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) { 4147 codegen_->InvokeRuntime(instruction->IsEnter() 4148 ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject), 4149 instruction, 4150 instruction->GetDexPc(), 4151 nullptr); 4152 if (instruction->IsEnter()) { 4153 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 4154 } else { 4155 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 4156 } 4157 } 4158 4159 void LocationsBuilderARM64::VisitMul(HMul* mul) { 4160 LocationSummary* locations = 4161 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 4162 switch (mul->GetResultType()) { 4163 case Primitive::kPrimInt: 4164 case Primitive::kPrimLong: 4165 locations->SetInAt(0, Location::RequiresRegister()); 4166 locations->SetInAt(1, Location::RequiresRegister()); 4167 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4168 break; 4169 4170 case Primitive::kPrimFloat: 4171 case Primitive::kPrimDouble: 4172 locations->SetInAt(0, Location::RequiresFpuRegister()); 4173 locations->SetInAt(1, Location::RequiresFpuRegister()); 4174 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4175 break; 4176 4177 default: 4178 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 4179 } 4180 } 4181 4182 void InstructionCodeGeneratorARM64::VisitMul(HMul* mul) { 4183 switch (mul->GetResultType()) { 4184 case Primitive::kPrimInt: 4185 case Primitive::kPrimLong: 4186 __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1)); 4187 break; 4188 4189 case Primitive::kPrimFloat: 4190 case Primitive::kPrimDouble: 4191 __ Fmul(OutputFPRegister(mul), InputFPRegisterAt(mul, 0), InputFPRegisterAt(mul, 1)); 4192 break; 4193 4194 default: 4195 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 4196 } 4197 } 4198 4199 void LocationsBuilderARM64::VisitNeg(HNeg* neg) { 4200 LocationSummary* locations = 4201 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 4202 switch (neg->GetResultType()) { 4203 case Primitive::kPrimInt: 4204 case Primitive::kPrimLong: 4205 locations->SetInAt(0, ARM64EncodableConstantOrRegister(neg->InputAt(0), neg)); 4206 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4207 break; 4208 4209 case Primitive::kPrimFloat: 4210 case Primitive::kPrimDouble: 4211 locations->SetInAt(0, Location::RequiresFpuRegister()); 4212 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4213 break; 4214 4215 default: 4216 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 4217 } 4218 } 4219 4220 void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) { 4221 switch (neg->GetResultType()) { 4222 case Primitive::kPrimInt: 4223 case Primitive::kPrimLong: 4224 __ Neg(OutputRegister(neg), InputOperandAt(neg, 0)); 4225 break; 4226 4227 case Primitive::kPrimFloat: 4228 case Primitive::kPrimDouble: 4229 __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0)); 4230 break; 4231 4232 default: 4233 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 4234 } 4235 } 4236 4237 void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) { 4238 LocationSummary* locations = 4239 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); 4240 InvokeRuntimeCallingConvention calling_convention; 4241 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0))); 4242 locations->SetOut(LocationFrom(x0)); 4243 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1))); 4244 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2))); 4245 } 4246 4247 void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) { 4248 LocationSummary* locations = instruction->GetLocations(); 4249 InvokeRuntimeCallingConvention calling_convention; 4250 Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt); 4251 DCHECK(type_index.Is(w0)); 4252 __ Mov(type_index, instruction->GetTypeIndex()); 4253 // Note: if heap poisoning is enabled, the entry point takes cares 4254 // of poisoning the reference. 4255 codegen_->InvokeRuntime(instruction->GetEntrypoint(), 4256 instruction, 4257 instruction->GetDexPc(), 4258 nullptr); 4259 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 4260 } 4261 4262 void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) { 4263 LocationSummary* locations = 4264 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); 4265 InvokeRuntimeCallingConvention calling_convention; 4266 if (instruction->IsStringAlloc()) { 4267 locations->AddTemp(LocationFrom(kArtMethodRegister)); 4268 } else { 4269 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 4270 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 4271 } 4272 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 4273 } 4274 4275 void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) { 4276 // Note: if heap poisoning is enabled, the entry point takes cares 4277 // of poisoning the reference. 4278 if (instruction->IsStringAlloc()) { 4279 // String is allocated through StringFactory. Call NewEmptyString entry point. 4280 Location temp = instruction->GetLocations()->GetTemp(0); 4281 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize); 4282 __ Ldr(XRegisterFrom(temp), MemOperand(tr, QUICK_ENTRY_POINT(pNewEmptyString))); 4283 __ Ldr(lr, MemOperand(XRegisterFrom(temp), code_offset.Int32Value())); 4284 __ Blr(lr); 4285 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 4286 } else { 4287 codegen_->InvokeRuntime(instruction->GetEntrypoint(), 4288 instruction, 4289 instruction->GetDexPc(), 4290 nullptr); 4291 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); 4292 } 4293 } 4294 4295 void LocationsBuilderARM64::VisitNot(HNot* instruction) { 4296 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 4297 locations->SetInAt(0, Location::RequiresRegister()); 4298 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4299 } 4300 4301 void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) { 4302 switch (instruction->GetResultType()) { 4303 case Primitive::kPrimInt: 4304 case Primitive::kPrimLong: 4305 __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0)); 4306 break; 4307 4308 default: 4309 LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType(); 4310 } 4311 } 4312 4313 void LocationsBuilderARM64::VisitBooleanNot(HBooleanNot* instruction) { 4314 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 4315 locations->SetInAt(0, Location::RequiresRegister()); 4316 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4317 } 4318 4319 void InstructionCodeGeneratorARM64::VisitBooleanNot(HBooleanNot* instruction) { 4320 __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), vixl::Operand(1)); 4321 } 4322 4323 void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) { 4324 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock() 4325 ? LocationSummary::kCallOnSlowPath 4326 : LocationSummary::kNoCall; 4327 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 4328 locations->SetInAt(0, Location::RequiresRegister()); 4329 if (instruction->HasUses()) { 4330 locations->SetOut(Location::SameAsFirstInput()); 4331 } 4332 } 4333 4334 void CodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) { 4335 if (CanMoveNullCheckToUser(instruction)) { 4336 return; 4337 } 4338 4339 BlockPoolsScope block_pools(GetVIXLAssembler()); 4340 Location obj = instruction->GetLocations()->InAt(0); 4341 __ Ldr(wzr, HeapOperandFrom(obj, Offset(0))); 4342 RecordPcInfo(instruction, instruction->GetDexPc()); 4343 } 4344 4345 void CodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) { 4346 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction); 4347 AddSlowPath(slow_path); 4348 4349 LocationSummary* locations = instruction->GetLocations(); 4350 Location obj = locations->InAt(0); 4351 4352 __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel()); 4353 } 4354 4355 void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) { 4356 codegen_->GenerateNullCheck(instruction); 4357 } 4358 4359 void LocationsBuilderARM64::VisitOr(HOr* instruction) { 4360 HandleBinaryOp(instruction); 4361 } 4362 4363 void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) { 4364 HandleBinaryOp(instruction); 4365 } 4366 4367 void LocationsBuilderARM64::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 4368 LOG(FATAL) << "Unreachable"; 4369 } 4370 4371 void InstructionCodeGeneratorARM64::VisitParallelMove(HParallelMove* instruction) { 4372 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 4373 } 4374 4375 void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) { 4376 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 4377 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 4378 if (location.IsStackSlot()) { 4379 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4380 } else if (location.IsDoubleStackSlot()) { 4381 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4382 } 4383 locations->SetOut(location); 4384 } 4385 4386 void InstructionCodeGeneratorARM64::VisitParameterValue( 4387 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4388 // Nothing to do, the parameter is already at its location. 4389 } 4390 4391 void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) { 4392 LocationSummary* locations = 4393 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4394 locations->SetOut(LocationFrom(kArtMethodRegister)); 4395 } 4396 4397 void InstructionCodeGeneratorARM64::VisitCurrentMethod( 4398 HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4399 // Nothing to do, the method is already at its location. 4400 } 4401 4402 void LocationsBuilderARM64::VisitPhi(HPhi* instruction) { 4403 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 4404 for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) { 4405 locations->SetInAt(i, Location::Any()); 4406 } 4407 locations->SetOut(Location::Any()); 4408 } 4409 4410 void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4411 LOG(FATAL) << "Unreachable"; 4412 } 4413 4414 void LocationsBuilderARM64::VisitRem(HRem* rem) { 4415 Primitive::Type type = rem->GetResultType(); 4416 LocationSummary::CallKind call_kind = 4417 Primitive::IsFloatingPointType(type) ? LocationSummary::kCall : LocationSummary::kNoCall; 4418 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 4419 4420 switch (type) { 4421 case Primitive::kPrimInt: 4422 case Primitive::kPrimLong: 4423 locations->SetInAt(0, Location::RequiresRegister()); 4424 locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1))); 4425 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4426 break; 4427 4428 case Primitive::kPrimFloat: 4429 case Primitive::kPrimDouble: { 4430 InvokeRuntimeCallingConvention calling_convention; 4431 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0))); 4432 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1))); 4433 locations->SetOut(calling_convention.GetReturnLocation(type)); 4434 4435 break; 4436 } 4437 4438 default: 4439 LOG(FATAL) << "Unexpected rem type " << type; 4440 } 4441 } 4442 4443 void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) { 4444 Primitive::Type type = rem->GetResultType(); 4445 4446 switch (type) { 4447 case Primitive::kPrimInt: 4448 case Primitive::kPrimLong: { 4449 GenerateDivRemIntegral(rem); 4450 break; 4451 } 4452 4453 case Primitive::kPrimFloat: 4454 case Primitive::kPrimDouble: { 4455 int32_t entry_offset = (type == Primitive::kPrimFloat) ? QUICK_ENTRY_POINT(pFmodf) 4456 : QUICK_ENTRY_POINT(pFmod); 4457 codegen_->InvokeRuntime(entry_offset, rem, rem->GetDexPc(), nullptr); 4458 if (type == Primitive::kPrimFloat) { 4459 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 4460 } else { 4461 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 4462 } 4463 break; 4464 } 4465 4466 default: 4467 LOG(FATAL) << "Unexpected rem type " << type; 4468 UNREACHABLE(); 4469 } 4470 } 4471 4472 void LocationsBuilderARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 4473 memory_barrier->SetLocations(nullptr); 4474 } 4475 4476 void InstructionCodeGeneratorARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 4477 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 4478 } 4479 4480 void LocationsBuilderARM64::VisitReturn(HReturn* instruction) { 4481 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); 4482 Primitive::Type return_type = instruction->InputAt(0)->GetType(); 4483 locations->SetInAt(0, ARM64ReturnLocation(return_type)); 4484 } 4485 4486 void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction ATTRIBUTE_UNUSED) { 4487 codegen_->GenerateFrameExit(); 4488 } 4489 4490 void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) { 4491 instruction->SetLocations(nullptr); 4492 } 4493 4494 void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction ATTRIBUTE_UNUSED) { 4495 codegen_->GenerateFrameExit(); 4496 } 4497 4498 void LocationsBuilderARM64::VisitRor(HRor* ror) { 4499 HandleBinaryOp(ror); 4500 } 4501 4502 void InstructionCodeGeneratorARM64::VisitRor(HRor* ror) { 4503 HandleBinaryOp(ror); 4504 } 4505 4506 void LocationsBuilderARM64::VisitShl(HShl* shl) { 4507 HandleShift(shl); 4508 } 4509 4510 void InstructionCodeGeneratorARM64::VisitShl(HShl* shl) { 4511 HandleShift(shl); 4512 } 4513 4514 void LocationsBuilderARM64::VisitShr(HShr* shr) { 4515 HandleShift(shr); 4516 } 4517 4518 void InstructionCodeGeneratorARM64::VisitShr(HShr* shr) { 4519 HandleShift(shr); 4520 } 4521 4522 void LocationsBuilderARM64::VisitSub(HSub* instruction) { 4523 HandleBinaryOp(instruction); 4524 } 4525 4526 void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) { 4527 HandleBinaryOp(instruction); 4528 } 4529 4530 void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4531 HandleFieldGet(instruction); 4532 } 4533 4534 void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4535 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4536 } 4537 4538 void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4539 HandleFieldSet(instruction); 4540 } 4541 4542 void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4543 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4544 } 4545 4546 void LocationsBuilderARM64::VisitUnresolvedInstanceFieldGet( 4547 HUnresolvedInstanceFieldGet* instruction) { 4548 FieldAccessCallingConventionARM64 calling_convention; 4549 codegen_->CreateUnresolvedFieldLocationSummary( 4550 instruction, instruction->GetFieldType(), calling_convention); 4551 } 4552 4553 void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldGet( 4554 HUnresolvedInstanceFieldGet* instruction) { 4555 FieldAccessCallingConventionARM64 calling_convention; 4556 codegen_->GenerateUnresolvedFieldAccess(instruction, 4557 instruction->GetFieldType(), 4558 instruction->GetFieldIndex(), 4559 instruction->GetDexPc(), 4560 calling_convention); 4561 } 4562 4563 void LocationsBuilderARM64::VisitUnresolvedInstanceFieldSet( 4564 HUnresolvedInstanceFieldSet* instruction) { 4565 FieldAccessCallingConventionARM64 calling_convention; 4566 codegen_->CreateUnresolvedFieldLocationSummary( 4567 instruction, instruction->GetFieldType(), calling_convention); 4568 } 4569 4570 void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldSet( 4571 HUnresolvedInstanceFieldSet* instruction) { 4572 FieldAccessCallingConventionARM64 calling_convention; 4573 codegen_->GenerateUnresolvedFieldAccess(instruction, 4574 instruction->GetFieldType(), 4575 instruction->GetFieldIndex(), 4576 instruction->GetDexPc(), 4577 calling_convention); 4578 } 4579 4580 void LocationsBuilderARM64::VisitUnresolvedStaticFieldGet( 4581 HUnresolvedStaticFieldGet* instruction) { 4582 FieldAccessCallingConventionARM64 calling_convention; 4583 codegen_->CreateUnresolvedFieldLocationSummary( 4584 instruction, instruction->GetFieldType(), calling_convention); 4585 } 4586 4587 void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldGet( 4588 HUnresolvedStaticFieldGet* instruction) { 4589 FieldAccessCallingConventionARM64 calling_convention; 4590 codegen_->GenerateUnresolvedFieldAccess(instruction, 4591 instruction->GetFieldType(), 4592 instruction->GetFieldIndex(), 4593 instruction->GetDexPc(), 4594 calling_convention); 4595 } 4596 4597 void LocationsBuilderARM64::VisitUnresolvedStaticFieldSet( 4598 HUnresolvedStaticFieldSet* instruction) { 4599 FieldAccessCallingConventionARM64 calling_convention; 4600 codegen_->CreateUnresolvedFieldLocationSummary( 4601 instruction, instruction->GetFieldType(), calling_convention); 4602 } 4603 4604 void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldSet( 4605 HUnresolvedStaticFieldSet* instruction) { 4606 FieldAccessCallingConventionARM64 calling_convention; 4607 codegen_->GenerateUnresolvedFieldAccess(instruction, 4608 instruction->GetFieldType(), 4609 instruction->GetFieldIndex(), 4610 instruction->GetDexPc(), 4611 calling_convention); 4612 } 4613 4614 void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) { 4615 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 4616 } 4617 4618 void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) { 4619 HBasicBlock* block = instruction->GetBlock(); 4620 if (block->GetLoopInformation() != nullptr) { 4621 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 4622 // The back edge will generate the suspend check. 4623 return; 4624 } 4625 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 4626 // The goto will generate the suspend check. 4627 return; 4628 } 4629 GenerateSuspendCheck(instruction, nullptr); 4630 } 4631 4632 void LocationsBuilderARM64::VisitThrow(HThrow* instruction) { 4633 LocationSummary* locations = 4634 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); 4635 InvokeRuntimeCallingConvention calling_convention; 4636 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 4637 } 4638 4639 void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) { 4640 codegen_->InvokeRuntime( 4641 QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc(), nullptr); 4642 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 4643 } 4644 4645 void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) { 4646 LocationSummary* locations = 4647 new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall); 4648 Primitive::Type input_type = conversion->GetInputType(); 4649 Primitive::Type result_type = conversion->GetResultType(); 4650 DCHECK_NE(input_type, result_type); 4651 if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) || 4652 (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) { 4653 LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type; 4654 } 4655 4656 if (Primitive::IsFloatingPointType(input_type)) { 4657 locations->SetInAt(0, Location::RequiresFpuRegister()); 4658 } else { 4659 locations->SetInAt(0, Location::RequiresRegister()); 4660 } 4661 4662 if (Primitive::IsFloatingPointType(result_type)) { 4663 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4664 } else { 4665 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4666 } 4667 } 4668 4669 void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) { 4670 Primitive::Type result_type = conversion->GetResultType(); 4671 Primitive::Type input_type = conversion->GetInputType(); 4672 4673 DCHECK_NE(input_type, result_type); 4674 4675 if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) { 4676 int result_size = Primitive::ComponentSize(result_type); 4677 int input_size = Primitive::ComponentSize(input_type); 4678 int min_size = std::min(result_size, input_size); 4679 Register output = OutputRegister(conversion); 4680 Register source = InputRegisterAt(conversion, 0); 4681 if (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimLong) { 4682 // 'int' values are used directly as W registers, discarding the top 4683 // bits, so we don't need to sign-extend and can just perform a move. 4684 // We do not pass the `kDiscardForSameWReg` argument to force clearing the 4685 // top 32 bits of the target register. We theoretically could leave those 4686 // bits unchanged, but we would have to make sure that no code uses a 4687 // 32bit input value as a 64bit value assuming that the top 32 bits are 4688 // zero. 4689 __ Mov(output.W(), source.W()); 4690 } else if (result_type == Primitive::kPrimChar || 4691 (input_type == Primitive::kPrimChar && input_size < result_size)) { 4692 __ Ubfx(output, 4693 output.IsX() ? source.X() : source.W(), 4694 0, Primitive::ComponentSize(Primitive::kPrimChar) * kBitsPerByte); 4695 } else { 4696 __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte); 4697 } 4698 } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) { 4699 __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0)); 4700 } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) { 4701 CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong); 4702 __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0)); 4703 } else if (Primitive::IsFloatingPointType(result_type) && 4704 Primitive::IsFloatingPointType(input_type)) { 4705 __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0)); 4706 } else { 4707 LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type 4708 << " to " << result_type; 4709 } 4710 } 4711 4712 void LocationsBuilderARM64::VisitUShr(HUShr* ushr) { 4713 HandleShift(ushr); 4714 } 4715 4716 void InstructionCodeGeneratorARM64::VisitUShr(HUShr* ushr) { 4717 HandleShift(ushr); 4718 } 4719 4720 void LocationsBuilderARM64::VisitXor(HXor* instruction) { 4721 HandleBinaryOp(instruction); 4722 } 4723 4724 void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) { 4725 HandleBinaryOp(instruction); 4726 } 4727 4728 void LocationsBuilderARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 4729 // Nothing to do, this should be removed during prepare for register allocator. 4730 LOG(FATAL) << "Unreachable"; 4731 } 4732 4733 void InstructionCodeGeneratorARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 4734 // Nothing to do, this should be removed during prepare for register allocator. 4735 LOG(FATAL) << "Unreachable"; 4736 } 4737 4738 // Simple implementation of packed switch - generate cascaded compare/jumps. 4739 void LocationsBuilderARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) { 4740 LocationSummary* locations = 4741 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 4742 locations->SetInAt(0, Location::RequiresRegister()); 4743 } 4744 4745 void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) { 4746 int32_t lower_bound = switch_instr->GetStartValue(); 4747 uint32_t num_entries = switch_instr->GetNumEntries(); 4748 Register value_reg = InputRegisterAt(switch_instr, 0); 4749 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 4750 4751 // Roughly set 16 as max average assemblies generated per HIR in a graph. 4752 static constexpr int32_t kMaxExpectedSizePerHInstruction = 16 * vixl::kInstructionSize; 4753 // ADR has a limited range(+/-1MB), so we set a threshold for the number of HIRs in the graph to 4754 // make sure we don't emit it if the target may run out of range. 4755 // TODO: Instead of emitting all jump tables at the end of the code, we could keep track of ADR 4756 // ranges and emit the tables only as required. 4757 static constexpr int32_t kJumpTableInstructionThreshold = 1* MB / kMaxExpectedSizePerHInstruction; 4758 4759 if (num_entries <= kPackedSwitchCompareJumpThreshold || 4760 // Current instruction id is an upper bound of the number of HIRs in the graph. 4761 GetGraph()->GetCurrentInstructionId() > kJumpTableInstructionThreshold) { 4762 // Create a series of compare/jumps. 4763 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); 4764 Register temp = temps.AcquireW(); 4765 __ Subs(temp, value_reg, Operand(lower_bound)); 4766 4767 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 4768 // Jump to successors[0] if value == lower_bound. 4769 __ B(eq, codegen_->GetLabelOf(successors[0])); 4770 int32_t last_index = 0; 4771 for (; num_entries - last_index > 2; last_index += 2) { 4772 __ Subs(temp, temp, Operand(2)); 4773 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 4774 __ B(lo, codegen_->GetLabelOf(successors[last_index + 1])); 4775 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 4776 __ B(eq, codegen_->GetLabelOf(successors[last_index + 2])); 4777 } 4778 if (num_entries - last_index == 2) { 4779 // The last missing case_value. 4780 __ Cmp(temp, Operand(1)); 4781 __ B(eq, codegen_->GetLabelOf(successors[last_index + 1])); 4782 } 4783 4784 // And the default for any other value. 4785 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 4786 __ B(codegen_->GetLabelOf(default_block)); 4787 } 4788 } else { 4789 JumpTableARM64* jump_table = codegen_->CreateJumpTable(switch_instr); 4790 4791 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); 4792 4793 // Below instructions should use at most one blocked register. Since there are two blocked 4794 // registers, we are free to block one. 4795 Register temp_w = temps.AcquireW(); 4796 Register index; 4797 // Remove the bias. 4798 if (lower_bound != 0) { 4799 index = temp_w; 4800 __ Sub(index, value_reg, Operand(lower_bound)); 4801 } else { 4802 index = value_reg; 4803 } 4804 4805 // Jump to default block if index is out of the range. 4806 __ Cmp(index, Operand(num_entries)); 4807 __ B(hs, codegen_->GetLabelOf(default_block)); 4808 4809 // In current VIXL implementation, it won't require any blocked registers to encode the 4810 // immediate value for Adr. So we are free to use both VIXL blocked registers to reduce the 4811 // register pressure. 4812 Register table_base = temps.AcquireX(); 4813 // Load jump offset from the table. 4814 __ Adr(table_base, jump_table->GetTableStartLabel()); 4815 Register jump_offset = temp_w; 4816 __ Ldr(jump_offset, MemOperand(table_base, index, UXTW, 2)); 4817 4818 // Jump to target block by branching to table_base(pc related) + offset. 4819 Register target_address = table_base; 4820 __ Add(target_address, table_base, Operand(jump_offset, SXTW)); 4821 __ Br(target_address); 4822 } 4823 } 4824 4825 void InstructionCodeGeneratorARM64::GenerateReferenceLoadOneRegister(HInstruction* instruction, 4826 Location out, 4827 uint32_t offset, 4828 Location maybe_temp) { 4829 Primitive::Type type = Primitive::kPrimNot; 4830 Register out_reg = RegisterFrom(out, type); 4831 if (kEmitCompilerReadBarrier) { 4832 Register temp_reg = RegisterFrom(maybe_temp, type); 4833 if (kUseBakerReadBarrier) { 4834 // Load with fast path based Baker's read barrier. 4835 // /* HeapReference<Object> */ out = *(out + offset) 4836 codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, 4837 out, 4838 out_reg, 4839 offset, 4840 temp_reg, 4841 /* needs_null_check */ false, 4842 /* use_load_acquire */ false); 4843 } else { 4844 // Load with slow path based read barrier. 4845 // Save the value of `out` into `maybe_temp` before overwriting it 4846 // in the following move operation, as we will need it for the 4847 // read barrier below. 4848 __ Mov(temp_reg, out_reg); 4849 // /* HeapReference<Object> */ out = *(out + offset) 4850 __ Ldr(out_reg, HeapOperand(out_reg, offset)); 4851 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 4852 } 4853 } else { 4854 // Plain load with no read barrier. 4855 // /* HeapReference<Object> */ out = *(out + offset) 4856 __ Ldr(out_reg, HeapOperand(out_reg, offset)); 4857 GetAssembler()->MaybeUnpoisonHeapReference(out_reg); 4858 } 4859 } 4860 4861 void InstructionCodeGeneratorARM64::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, 4862 Location out, 4863 Location obj, 4864 uint32_t offset, 4865 Location maybe_temp) { 4866 Primitive::Type type = Primitive::kPrimNot; 4867 Register out_reg = RegisterFrom(out, type); 4868 Register obj_reg = RegisterFrom(obj, type); 4869 if (kEmitCompilerReadBarrier) { 4870 if (kUseBakerReadBarrier) { 4871 // Load with fast path based Baker's read barrier. 4872 Register temp_reg = RegisterFrom(maybe_temp, type); 4873 // /* HeapReference<Object> */ out = *(obj + offset) 4874 codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, 4875 out, 4876 obj_reg, 4877 offset, 4878 temp_reg, 4879 /* needs_null_check */ false, 4880 /* use_load_acquire */ false); 4881 } else { 4882 // Load with slow path based read barrier. 4883 // /* HeapReference<Object> */ out = *(obj + offset) 4884 __ Ldr(out_reg, HeapOperand(obj_reg, offset)); 4885 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 4886 } 4887 } else { 4888 // Plain load with no read barrier. 4889 // /* HeapReference<Object> */ out = *(obj + offset) 4890 __ Ldr(out_reg, HeapOperand(obj_reg, offset)); 4891 GetAssembler()->MaybeUnpoisonHeapReference(out_reg); 4892 } 4893 } 4894 4895 void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instruction, 4896 Location root, 4897 vixl::Register obj, 4898 uint32_t offset, 4899 vixl::Label* fixup_label) { 4900 Register root_reg = RegisterFrom(root, Primitive::kPrimNot); 4901 if (kEmitCompilerReadBarrier) { 4902 if (kUseBakerReadBarrier) { 4903 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 4904 // Baker's read barrier are used: 4905 // 4906 // root = obj.field; 4907 // if (Thread::Current()->GetIsGcMarking()) { 4908 // root = ReadBarrier::Mark(root) 4909 // } 4910 4911 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 4912 if (fixup_label == nullptr) { 4913 __ Ldr(root_reg, MemOperand(obj, offset)); 4914 } else { 4915 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 4916 __ Bind(fixup_label); 4917 __ ldr(root_reg, MemOperand(obj, offset)); 4918 } 4919 static_assert( 4920 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 4921 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 4922 "have different sizes."); 4923 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 4924 "art::mirror::CompressedReference<mirror::Object> and int32_t " 4925 "have different sizes."); 4926 4927 // Slow path used to mark the GC root `root`. 4928 SlowPathCodeARM64* slow_path = 4929 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root, root); 4930 codegen_->AddSlowPath(slow_path); 4931 4932 MacroAssembler* masm = GetVIXLAssembler(); 4933 UseScratchRegisterScope temps(masm); 4934 Register temp = temps.AcquireW(); 4935 // temp = Thread::Current()->GetIsGcMarking() 4936 __ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArm64WordSize>().Int32Value())); 4937 __ Cbnz(temp, slow_path->GetEntryLabel()); 4938 __ Bind(slow_path->GetExitLabel()); 4939 } else { 4940 // GC root loaded through a slow path for read barriers other 4941 // than Baker's. 4942 // /* GcRoot<mirror::Object>* */ root = obj + offset 4943 if (fixup_label == nullptr) { 4944 __ Add(root_reg.X(), obj.X(), offset); 4945 } else { 4946 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 4947 __ Bind(fixup_label); 4948 __ add(root_reg.X(), obj.X(), offset); 4949 } 4950 // /* mirror::Object* */ root = root->Read() 4951 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 4952 } 4953 } else { 4954 // Plain GC root load with no read barrier. 4955 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 4956 if (fixup_label == nullptr) { 4957 __ Ldr(root_reg, MemOperand(obj, offset)); 4958 } else { 4959 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); 4960 __ Bind(fixup_label); 4961 __ ldr(root_reg, MemOperand(obj, offset)); 4962 } 4963 // Note that GC roots are not affected by heap poisoning, thus we 4964 // do not have to unpoison `root_reg` here. 4965 } 4966 } 4967 4968 void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 4969 Location ref, 4970 vixl::Register obj, 4971 uint32_t offset, 4972 Register temp, 4973 bool needs_null_check, 4974 bool use_load_acquire) { 4975 DCHECK(kEmitCompilerReadBarrier); 4976 DCHECK(kUseBakerReadBarrier); 4977 4978 // /* HeapReference<Object> */ ref = *(obj + offset) 4979 Location no_index = Location::NoLocation(); 4980 GenerateReferenceLoadWithBakerReadBarrier( 4981 instruction, ref, obj, offset, no_index, temp, needs_null_check, use_load_acquire); 4982 } 4983 4984 void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 4985 Location ref, 4986 vixl::Register obj, 4987 uint32_t data_offset, 4988 Location index, 4989 Register temp, 4990 bool needs_null_check) { 4991 DCHECK(kEmitCompilerReadBarrier); 4992 DCHECK(kUseBakerReadBarrier); 4993 4994 // Array cells are never volatile variables, therefore array loads 4995 // never use Load-Acquire instructions on ARM64. 4996 const bool use_load_acquire = false; 4997 4998 // /* HeapReference<Object> */ ref = 4999 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 5000 GenerateReferenceLoadWithBakerReadBarrier( 5001 instruction, ref, obj, data_offset, index, temp, needs_null_check, use_load_acquire); 5002 } 5003 5004 void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 5005 Location ref, 5006 vixl::Register obj, 5007 uint32_t offset, 5008 Location index, 5009 Register temp, 5010 bool needs_null_check, 5011 bool use_load_acquire) { 5012 DCHECK(kEmitCompilerReadBarrier); 5013 DCHECK(kUseBakerReadBarrier); 5014 // If `index` is a valid location, then we are emitting an array 5015 // load, so we shouldn't be using a Load Acquire instruction. 5016 // In other words: `index.IsValid()` => `!use_load_acquire`. 5017 DCHECK(!index.IsValid() || !use_load_acquire); 5018 5019 MacroAssembler* masm = GetVIXLAssembler(); 5020 UseScratchRegisterScope temps(masm); 5021 5022 // In slow path based read barriers, the read barrier call is 5023 // inserted after the original load. However, in fast path based 5024 // Baker's read barriers, we need to perform the load of 5025 // mirror::Object::monitor_ *before* the original reference load. 5026 // This load-load ordering is required by the read barrier. 5027 // The fast path/slow path (for Baker's algorithm) should look like: 5028 // 5029 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 5030 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 5031 // HeapReference<Object> ref = *src; // Original reference load. 5032 // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); 5033 // if (is_gray) { 5034 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 5035 // } 5036 // 5037 // Note: the original implementation in ReadBarrier::Barrier is 5038 // slightly more complex as it performs additional checks that we do 5039 // not do here for performance reasons. 5040 5041 Primitive::Type type = Primitive::kPrimNot; 5042 Register ref_reg = RegisterFrom(ref, type); 5043 DCHECK(obj.IsW()); 5044 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 5045 5046 // /* int32_t */ monitor = obj->monitor_ 5047 __ Ldr(temp, HeapOperand(obj, monitor_offset)); 5048 if (needs_null_check) { 5049 MaybeRecordImplicitNullCheck(instruction); 5050 } 5051 // /* LockWord */ lock_word = LockWord(monitor) 5052 static_assert(sizeof(LockWord) == sizeof(int32_t), 5053 "art::LockWord and int32_t have different sizes."); 5054 // /* uint32_t */ rb_state = lock_word.ReadBarrierState() 5055 __ Lsr(temp, temp, LockWord::kReadBarrierStateShift); 5056 __ And(temp, temp, Operand(LockWord::kReadBarrierStateMask)); 5057 static_assert( 5058 LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_, 5059 "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_."); 5060 5061 // Introduce a dependency on the high bits of rb_state, which shall 5062 // be all zeroes, to prevent load-load reordering, and without using 5063 // a memory barrier (which would be more expensive). 5064 // temp2 = rb_state & ~LockWord::kReadBarrierStateMask = 0 5065 Register temp2 = temps.AcquireW(); 5066 __ Bic(temp2, temp, Operand(LockWord::kReadBarrierStateMask)); 5067 // obj is unchanged by this operation, but its value now depends on 5068 // temp2, which depends on temp. 5069 __ Add(obj, obj, Operand(temp2)); 5070 temps.Release(temp2); 5071 5072 // The actual reference load. 5073 if (index.IsValid()) { 5074 static_assert( 5075 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 5076 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 5077 // /* HeapReference<Object> */ ref = 5078 // *(obj + offset + index * sizeof(HeapReference<Object>)) 5079 const size_t shift_amount = Primitive::ComponentSizeShift(type); 5080 if (index.IsConstant()) { 5081 uint32_t computed_offset = offset + (Int64ConstantFrom(index) << shift_amount); 5082 Load(type, ref_reg, HeapOperand(obj, computed_offset)); 5083 } else { 5084 temp2 = temps.AcquireW(); 5085 __ Add(temp2, obj, offset); 5086 Load(type, ref_reg, HeapOperand(temp2, XRegisterFrom(index), LSL, shift_amount)); 5087 temps.Release(temp2); 5088 } 5089 } else { 5090 // /* HeapReference<Object> */ ref = *(obj + offset) 5091 MemOperand field = HeapOperand(obj, offset); 5092 if (use_load_acquire) { 5093 LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false); 5094 } else { 5095 Load(type, ref_reg, field); 5096 } 5097 } 5098 5099 // Object* ref = ref_addr->AsMirrorPtr() 5100 GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); 5101 5102 // Slow path used to mark the object `ref` when it is gray. 5103 SlowPathCodeARM64* slow_path = 5104 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, ref, ref); 5105 AddSlowPath(slow_path); 5106 5107 // if (rb_state == ReadBarrier::gray_ptr_) 5108 // ref = ReadBarrier::Mark(ref); 5109 __ Cmp(temp, ReadBarrier::gray_ptr_); 5110 __ B(eq, slow_path->GetEntryLabel()); 5111 __ Bind(slow_path->GetExitLabel()); 5112 } 5113 5114 void CodeGeneratorARM64::GenerateReadBarrierSlow(HInstruction* instruction, 5115 Location out, 5116 Location ref, 5117 Location obj, 5118 uint32_t offset, 5119 Location index) { 5120 DCHECK(kEmitCompilerReadBarrier); 5121 5122 // Insert a slow path based read barrier *after* the reference load. 5123 // 5124 // If heap poisoning is enabled, the unpoisoning of the loaded 5125 // reference will be carried out by the runtime within the slow 5126 // path. 5127 // 5128 // Note that `ref` currently does not get unpoisoned (when heap 5129 // poisoning is enabled), which is alright as the `ref` argument is 5130 // not used by the artReadBarrierSlow entry point. 5131 // 5132 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 5133 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) 5134 ReadBarrierForHeapReferenceSlowPathARM64(instruction, out, ref, obj, offset, index); 5135 AddSlowPath(slow_path); 5136 5137 __ B(slow_path->GetEntryLabel()); 5138 __ Bind(slow_path->GetExitLabel()); 5139 } 5140 5141 void CodeGeneratorARM64::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 5142 Location out, 5143 Location ref, 5144 Location obj, 5145 uint32_t offset, 5146 Location index) { 5147 if (kEmitCompilerReadBarrier) { 5148 // Baker's read barriers shall be handled by the fast path 5149 // (CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier). 5150 DCHECK(!kUseBakerReadBarrier); 5151 // If heap poisoning is enabled, unpoisoning will be taken care of 5152 // by the runtime within the slow path. 5153 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 5154 } else if (kPoisonHeapReferences) { 5155 GetAssembler()->UnpoisonHeapReference(WRegisterFrom(out)); 5156 } 5157 } 5158 5159 void CodeGeneratorARM64::GenerateReadBarrierForRootSlow(HInstruction* instruction, 5160 Location out, 5161 Location root) { 5162 DCHECK(kEmitCompilerReadBarrier); 5163 5164 // Insert a slow path based read barrier *after* the GC root load. 5165 // 5166 // Note that GC roots are not affected by heap poisoning, so we do 5167 // not need to do anything special for this here. 5168 SlowPathCodeARM64* slow_path = 5169 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM64(instruction, out, root); 5170 AddSlowPath(slow_path); 5171 5172 __ B(slow_path->GetEntryLabel()); 5173 __ Bind(slow_path->GetExitLabel()); 5174 } 5175 5176 void LocationsBuilderARM64::VisitClassTableGet(HClassTableGet* instruction) { 5177 LocationSummary* locations = 5178 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5179 locations->SetInAt(0, Location::RequiresRegister()); 5180 locations->SetOut(Location::RequiresRegister()); 5181 } 5182 5183 void InstructionCodeGeneratorARM64::VisitClassTableGet(HClassTableGet* instruction) { 5184 LocationSummary* locations = instruction->GetLocations(); 5185 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 5186 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 5187 instruction->GetIndex(), kArm64PointerSize).SizeValue(); 5188 __ Ldr(XRegisterFrom(locations->Out()), 5189 MemOperand(XRegisterFrom(locations->InAt(0)), method_offset)); 5190 } else { 5191 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 5192 instruction->GetIndex() % ImTable::kSize, kArm64PointerSize)); 5193 __ Ldr(XRegisterFrom(locations->Out()), MemOperand(XRegisterFrom(locations->InAt(0)), 5194 mirror::Class::ImtPtrOffset(kArm64PointerSize).Uint32Value())); 5195 __ Ldr(XRegisterFrom(locations->Out()), 5196 MemOperand(XRegisterFrom(locations->Out()), method_offset)); 5197 } 5198 } 5199 5200 5201 5202 #undef __ 5203 #undef QUICK_ENTRY_POINT 5204 5205 } // namespace arm64 5206 } // namespace art 5207