1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ 18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ 19 20 #include "base/bit_field.h" 21 #include "code_generator.h" 22 #include "common_arm64.h" 23 #include "dex/dex_file_types.h" 24 #include "dex/string_reference.h" 25 #include "dex/type_reference.h" 26 #include "driver/compiler_options.h" 27 #include "nodes.h" 28 #include "parallel_move_resolver.h" 29 #include "utils/arm64/assembler_arm64.h" 30 31 // TODO(VIXL): Make VIXL compile with -Wshadow. 32 #pragma GCC diagnostic push 33 #pragma GCC diagnostic ignored "-Wshadow" 34 #include "aarch64/disasm-aarch64.h" 35 #include "aarch64/macro-assembler-aarch64.h" 36 #pragma GCC diagnostic pop 37 38 namespace art { 39 40 namespace linker { 41 class Arm64RelativePatcherTest; 42 } // namespace linker 43 44 namespace arm64 { 45 46 class CodeGeneratorARM64; 47 48 // Use a local definition to prevent copying mistakes. 49 static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize); 50 51 // These constants are used as an approximate margin when emission of veneer and literal pools 52 // must be blocked. 53 static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize; 54 static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes; 55 56 static const vixl::aarch64::Register kParameterCoreRegisters[] = { 57 vixl::aarch64::x1, 58 vixl::aarch64::x2, 59 vixl::aarch64::x3, 60 vixl::aarch64::x4, 61 vixl::aarch64::x5, 62 vixl::aarch64::x6, 63 vixl::aarch64::x7 64 }; 65 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); 66 static const vixl::aarch64::FPRegister kParameterFPRegisters[] = { 67 vixl::aarch64::d0, 68 vixl::aarch64::d1, 69 vixl::aarch64::d2, 70 vixl::aarch64::d3, 71 vixl::aarch64::d4, 72 vixl::aarch64::d5, 73 vixl::aarch64::d6, 74 vixl::aarch64::d7 75 }; 76 static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters); 77 78 // Thread Register. 79 const vixl::aarch64::Register tr = vixl::aarch64::x19; 80 // Marking Register. 81 const vixl::aarch64::Register mr = vixl::aarch64::x20; 82 // Method register on invoke. 83 static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0; 84 const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0, 85 vixl::aarch64::ip1); 86 const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31); 87 88 const vixl::aarch64::CPURegList runtime_reserved_core_registers = 89 vixl::aarch64::CPURegList( 90 tr, 91 // Reserve X20 as Marking Register when emitting Baker read barriers. 92 ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) ? mr : vixl::aarch64::NoCPUReg), 93 vixl::aarch64::lr); 94 95 // Some instructions have special requirements for a temporary, for example 96 // LoadClass/kBssEntry and LoadString/kBssEntry for Baker read barrier require 97 // temp that's not an R0 (to avoid an extra move) and Baker read barrier field 98 // loads with large offsets need a fixed register to limit the number of link-time 99 // thunks we generate. For these and similar cases, we want to reserve a specific 100 // register that's neither callee-save nor an argument register. We choose x15. 101 inline Location FixedTempLocation() { 102 return Location::RegisterLocation(vixl::aarch64::x15.GetCode()); 103 } 104 105 // Callee-save registers AAPCS64, without x19 (Thread Register) (nor 106 // x20 (Marking Register) when emitting Baker read barriers). 107 const vixl::aarch64::CPURegList callee_saved_core_registers( 108 vixl::aarch64::CPURegister::kRegister, 109 vixl::aarch64::kXRegSize, 110 ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) 111 ? vixl::aarch64::x21.GetCode() 112 : vixl::aarch64::x20.GetCode()), 113 vixl::aarch64::x30.GetCode()); 114 const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kFPRegister, 115 vixl::aarch64::kDRegSize, 116 vixl::aarch64::d8.GetCode(), 117 vixl::aarch64::d15.GetCode()); 118 Location ARM64ReturnLocation(DataType::Type return_type); 119 120 class SlowPathCodeARM64 : public SlowPathCode { 121 public: 122 explicit SlowPathCodeARM64(HInstruction* instruction) 123 : SlowPathCode(instruction), entry_label_(), exit_label_() {} 124 125 vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; } 126 vixl::aarch64::Label* GetExitLabel() { return &exit_label_; } 127 128 void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override; 129 void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override; 130 131 private: 132 vixl::aarch64::Label entry_label_; 133 vixl::aarch64::Label exit_label_; 134 135 DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64); 136 }; 137 138 class JumpTableARM64 : public DeletableArenaObject<kArenaAllocSwitchTable> { 139 public: 140 explicit JumpTableARM64(HPackedSwitch* switch_instr) 141 : switch_instr_(switch_instr), table_start_() {} 142 143 vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; } 144 145 void EmitTable(CodeGeneratorARM64* codegen); 146 147 private: 148 HPackedSwitch* const switch_instr_; 149 vixl::aarch64::Label table_start_; 150 151 DISALLOW_COPY_AND_ASSIGN(JumpTableARM64); 152 }; 153 154 static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] = 155 { vixl::aarch64::x0, 156 vixl::aarch64::x1, 157 vixl::aarch64::x2, 158 vixl::aarch64::x3, 159 vixl::aarch64::x4, 160 vixl::aarch64::x5, 161 vixl::aarch64::x6, 162 vixl::aarch64::x7 }; 163 static constexpr size_t kRuntimeParameterCoreRegistersLength = 164 arraysize(kRuntimeParameterCoreRegisters); 165 static const vixl::aarch64::FPRegister kRuntimeParameterFpuRegisters[] = 166 { vixl::aarch64::d0, 167 vixl::aarch64::d1, 168 vixl::aarch64::d2, 169 vixl::aarch64::d3, 170 vixl::aarch64::d4, 171 vixl::aarch64::d5, 172 vixl::aarch64::d6, 173 vixl::aarch64::d7 }; 174 static constexpr size_t kRuntimeParameterFpuRegistersLength = 175 arraysize(kRuntimeParameterCoreRegisters); 176 177 class InvokeRuntimeCallingConvention : public CallingConvention<vixl::aarch64::Register, 178 vixl::aarch64::FPRegister> { 179 public: 180 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); 181 182 InvokeRuntimeCallingConvention() 183 : CallingConvention(kRuntimeParameterCoreRegisters, 184 kRuntimeParameterCoreRegistersLength, 185 kRuntimeParameterFpuRegisters, 186 kRuntimeParameterFpuRegistersLength, 187 kArm64PointerSize) {} 188 189 Location GetReturnLocation(DataType::Type return_type); 190 191 private: 192 DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention); 193 }; 194 195 class InvokeDexCallingConvention : public CallingConvention<vixl::aarch64::Register, 196 vixl::aarch64::FPRegister> { 197 public: 198 InvokeDexCallingConvention() 199 : CallingConvention(kParameterCoreRegisters, 200 kParameterCoreRegistersLength, 201 kParameterFPRegisters, 202 kParameterFPRegistersLength, 203 kArm64PointerSize) {} 204 205 Location GetReturnLocation(DataType::Type return_type) const { 206 return ARM64ReturnLocation(return_type); 207 } 208 209 210 private: 211 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention); 212 }; 213 214 class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor { 215 public: 216 InvokeDexCallingConventionVisitorARM64() {} 217 virtual ~InvokeDexCallingConventionVisitorARM64() {} 218 219 Location GetNextLocation(DataType::Type type) override; 220 Location GetReturnLocation(DataType::Type return_type) const override { 221 return calling_convention.GetReturnLocation(return_type); 222 } 223 Location GetMethodLocation() const override; 224 225 private: 226 InvokeDexCallingConvention calling_convention; 227 228 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64); 229 }; 230 231 class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention { 232 public: 233 FieldAccessCallingConventionARM64() {} 234 235 Location GetObjectLocation() const override { 236 return helpers::LocationFrom(vixl::aarch64::x1); 237 } 238 Location GetFieldIndexLocation() const override { 239 return helpers::LocationFrom(vixl::aarch64::x0); 240 } 241 Location GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED) const override { 242 return helpers::LocationFrom(vixl::aarch64::x0); 243 } 244 Location GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED, 245 bool is_instance) const override { 246 return is_instance 247 ? helpers::LocationFrom(vixl::aarch64::x2) 248 : helpers::LocationFrom(vixl::aarch64::x1); 249 } 250 Location GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED) const override { 251 return helpers::LocationFrom(vixl::aarch64::d0); 252 } 253 254 private: 255 DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64); 256 }; 257 258 class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { 259 public: 260 InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen); 261 262 #define DECLARE_VISIT_INSTRUCTION(name, super) \ 263 void Visit##name(H##name* instr) override; 264 265 FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION) 266 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION) 267 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) 268 269 #undef DECLARE_VISIT_INSTRUCTION 270 271 void VisitInstruction(HInstruction* instruction) override { 272 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() 273 << " (id " << instruction->GetId() << ")"; 274 } 275 276 Arm64Assembler* GetAssembler() const { return assembler_; } 277 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); } 278 279 private: 280 void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path, 281 vixl::aarch64::Register class_reg); 282 void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, 283 vixl::aarch64::Register temp); 284 void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor); 285 void HandleBinaryOp(HBinaryOperation* instr); 286 287 void HandleFieldSet(HInstruction* instruction, 288 const FieldInfo& field_info, 289 bool value_can_be_null); 290 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); 291 void HandleCondition(HCondition* instruction); 292 293 // Generate a heap reference load using one register `out`: 294 // 295 // out <- *(out + offset) 296 // 297 // while honoring heap poisoning and/or read barriers (if any). 298 // 299 // Location `maybe_temp` is used when generating a read barrier and 300 // shall be a register in that case; it may be an invalid location 301 // otherwise. 302 void GenerateReferenceLoadOneRegister(HInstruction* instruction, 303 Location out, 304 uint32_t offset, 305 Location maybe_temp, 306 ReadBarrierOption read_barrier_option); 307 // Generate a heap reference load using two different registers 308 // `out` and `obj`: 309 // 310 // out <- *(obj + offset) 311 // 312 // while honoring heap poisoning and/or read barriers (if any). 313 // 314 // Location `maybe_temp` is used when generating a Baker's (fast 315 // path) read barrier and shall be a register in that case; it may 316 // be an invalid location otherwise. 317 void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, 318 Location out, 319 Location obj, 320 uint32_t offset, 321 Location maybe_temp, 322 ReadBarrierOption read_barrier_option); 323 324 // Generate a floating-point comparison. 325 void GenerateFcmp(HInstruction* instruction); 326 327 void HandleShift(HBinaryOperation* instr); 328 void GenerateTestAndBranch(HInstruction* instruction, 329 size_t condition_input_index, 330 vixl::aarch64::Label* true_target, 331 vixl::aarch64::Label* false_target); 332 void DivRemOneOrMinusOne(HBinaryOperation* instruction); 333 void DivRemByPowerOfTwo(HBinaryOperation* instruction); 334 void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction); 335 void GenerateIntDiv(HDiv* instruction); 336 void GenerateIntDivForConstDenom(HDiv *instruction); 337 void GenerateIntDivForPower2Denom(HDiv *instruction); 338 void GenerateIntRem(HRem* instruction); 339 void GenerateIntRemForConstDenom(HRem *instruction); 340 void GenerateIntRemForPower2Denom(HRem *instruction); 341 void HandleGoto(HInstruction* got, HBasicBlock* successor); 342 343 vixl::aarch64::MemOperand VecAddress( 344 HVecMemoryOperation* instruction, 345 // This function may acquire a scratch register. 346 vixl::aarch64::UseScratchRegisterScope* temps_scope, 347 size_t size, 348 bool is_string_char_at, 349 /*out*/ vixl::aarch64::Register* scratch); 350 351 Arm64Assembler* const assembler_; 352 CodeGeneratorARM64* const codegen_; 353 354 DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64); 355 }; 356 357 class LocationsBuilderARM64 : public HGraphVisitor { 358 public: 359 LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen) 360 : HGraphVisitor(graph), codegen_(codegen) {} 361 362 #define DECLARE_VISIT_INSTRUCTION(name, super) \ 363 void Visit##name(H##name* instr) override; 364 365 FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION) 366 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION) 367 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) 368 369 #undef DECLARE_VISIT_INSTRUCTION 370 371 void VisitInstruction(HInstruction* instruction) override { 372 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() 373 << " (id " << instruction->GetId() << ")"; 374 } 375 376 private: 377 void HandleBinaryOp(HBinaryOperation* instr); 378 void HandleFieldSet(HInstruction* instruction); 379 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); 380 void HandleInvoke(HInvoke* instr); 381 void HandleCondition(HCondition* instruction); 382 void HandleShift(HBinaryOperation* instr); 383 384 CodeGeneratorARM64* const codegen_; 385 InvokeDexCallingConventionVisitorARM64 parameter_visitor_; 386 387 DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64); 388 }; 389 390 class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap { 391 public: 392 ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen) 393 : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {} 394 395 protected: 396 void PrepareForEmitNativeCode() override; 397 void FinishEmitNativeCode() override; 398 Location AllocateScratchLocationFor(Location::Kind kind) override; 399 void FreeScratchLocation(Location loc) override; 400 void EmitMove(size_t index) override; 401 402 private: 403 Arm64Assembler* GetAssembler() const; 404 vixl::aarch64::MacroAssembler* GetVIXLAssembler() const { 405 return GetAssembler()->GetVIXLAssembler(); 406 } 407 408 CodeGeneratorARM64* const codegen_; 409 vixl::aarch64::UseScratchRegisterScope vixl_temps_; 410 411 DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64); 412 }; 413 414 class CodeGeneratorARM64 : public CodeGenerator { 415 public: 416 CodeGeneratorARM64(HGraph* graph, 417 const CompilerOptions& compiler_options, 418 OptimizingCompilerStats* stats = nullptr); 419 virtual ~CodeGeneratorARM64() {} 420 421 void GenerateFrameEntry() override; 422 void GenerateFrameExit() override; 423 424 vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const; 425 vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const; 426 427 void Bind(HBasicBlock* block) override; 428 429 vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) { 430 block = FirstNonEmptyBlock(block); 431 return &(block_labels_[block->GetBlockId()]); 432 } 433 434 size_t GetWordSize() const override { 435 return kArm64WordSize; 436 } 437 438 size_t GetFloatingPointSpillSlotSize() const override { 439 return GetGraph()->HasSIMD() 440 ? 2 * kArm64WordSize // 16 bytes == 2 arm64 words for each spill 441 : 1 * kArm64WordSize; // 8 bytes == 1 arm64 words for each spill 442 } 443 444 uintptr_t GetAddressOf(HBasicBlock* block) override { 445 vixl::aarch64::Label* block_entry_label = GetLabelOf(block); 446 DCHECK(block_entry_label->IsBound()); 447 return block_entry_label->GetLocation(); 448 } 449 450 HGraphVisitor* GetLocationBuilder() override { return &location_builder_; } 451 HGraphVisitor* GetInstructionVisitor() override { return &instruction_visitor_; } 452 Arm64Assembler* GetAssembler() override { return &assembler_; } 453 const Arm64Assembler& GetAssembler() const override { return assembler_; } 454 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); } 455 456 // Emit a write barrier. 457 void MarkGCCard(vixl::aarch64::Register object, 458 vixl::aarch64::Register value, 459 bool value_can_be_null); 460 461 void GenerateMemoryBarrier(MemBarrierKind kind); 462 463 // Register allocation. 464 465 void SetupBlockedRegisters() const override; 466 467 size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override; 468 size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override; 469 size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override; 470 size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override; 471 472 // The number of registers that can be allocated. The register allocator may 473 // decide to reserve and not use a few of them. 474 // We do not consider registers sp, xzr, wzr. They are either not allocatable 475 // (xzr, wzr), or make for poor allocatable registers (sp alignment 476 // requirements, etc.). This also facilitates our task as all other registers 477 // can easily be mapped via to or from their type and index or code. 478 static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1; 479 static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfFPRegisters; 480 static constexpr int kNumberOfAllocatableRegisterPairs = 0; 481 482 void DumpCoreRegister(std::ostream& stream, int reg) const override; 483 void DumpFloatingPointRegister(std::ostream& stream, int reg) const override; 484 485 InstructionSet GetInstructionSet() const override { 486 return InstructionSet::kArm64; 487 } 488 489 const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const; 490 491 void Initialize() override { 492 block_labels_.resize(GetGraph()->GetBlocks().size()); 493 } 494 495 // We want to use the STP and LDP instructions to spill and restore registers for slow paths. 496 // These instructions can only encode offsets that are multiples of the register size accessed. 497 uint32_t GetPreferredSlotsAlignment() const override { return vixl::aarch64::kXRegSizeInBytes; } 498 499 JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) { 500 jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARM64(switch_instr)); 501 return jump_tables_.back().get(); 502 } 503 504 void Finalize(CodeAllocator* allocator) override; 505 506 // Code generation helpers. 507 void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant); 508 void MoveConstant(Location destination, int32_t value) override; 509 void MoveLocation(Location dst, Location src, DataType::Type dst_type) override; 510 void AddLocationAsTemp(Location location, LocationSummary* locations) override; 511 512 void Load(DataType::Type type, 513 vixl::aarch64::CPURegister dst, 514 const vixl::aarch64::MemOperand& src); 515 void Store(DataType::Type type, 516 vixl::aarch64::CPURegister src, 517 const vixl::aarch64::MemOperand& dst); 518 void LoadAcquire(HInstruction* instruction, 519 vixl::aarch64::CPURegister dst, 520 const vixl::aarch64::MemOperand& src, 521 bool needs_null_check); 522 void StoreRelease(HInstruction* instruction, 523 DataType::Type type, 524 vixl::aarch64::CPURegister src, 525 const vixl::aarch64::MemOperand& dst, 526 bool needs_null_check); 527 528 // Generate code to invoke a runtime entry point. 529 void InvokeRuntime(QuickEntrypointEnum entrypoint, 530 HInstruction* instruction, 531 uint32_t dex_pc, 532 SlowPathCode* slow_path = nullptr) override; 533 534 // Generate code to invoke a runtime entry point, but do not record 535 // PC-related information in a stack map. 536 void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 537 HInstruction* instruction, 538 SlowPathCode* slow_path); 539 540 ParallelMoveResolverARM64* GetMoveResolver() override { return &move_resolver_; } 541 542 bool NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED) const override { 543 return false; 544 } 545 546 // Check if the desired_string_load_kind is supported. If it is, return it, 547 // otherwise return a fall-back kind that should be used instead. 548 HLoadString::LoadKind GetSupportedLoadStringKind( 549 HLoadString::LoadKind desired_string_load_kind) override; 550 551 // Check if the desired_class_load_kind is supported. If it is, return it, 552 // otherwise return a fall-back kind that should be used instead. 553 HLoadClass::LoadKind GetSupportedLoadClassKind( 554 HLoadClass::LoadKind desired_class_load_kind) override; 555 556 // Check if the desired_dispatch_info is supported. If it is, return it, 557 // otherwise return a fall-back info that should be used instead. 558 HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( 559 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 560 ArtMethod* method) override; 561 562 void GenerateStaticOrDirectCall( 563 HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override; 564 void GenerateVirtualCall( 565 HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override; 566 567 void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, 568 DataType::Type type ATTRIBUTE_UNUSED) override { 569 UNIMPLEMENTED(FATAL); 570 } 571 572 // Add a new boot image intrinsic patch for an instruction and return the label 573 // to be bound before the instruction. The instruction will be either the 574 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing 575 // to the associated ADRP patch label). 576 vixl::aarch64::Label* NewBootImageIntrinsicPatch(uint32_t intrinsic_data, 577 vixl::aarch64::Label* adrp_label = nullptr); 578 579 // Add a new boot image relocation patch for an instruction and return the label 580 // to be bound before the instruction. The instruction will be either the 581 // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing 582 // to the associated ADRP patch label). 583 vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset, 584 vixl::aarch64::Label* adrp_label = nullptr); 585 586 // Add a new boot image method patch for an instruction and return the label 587 // to be bound before the instruction. The instruction will be either the 588 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing 589 // to the associated ADRP patch label). 590 vixl::aarch64::Label* NewBootImageMethodPatch(MethodReference target_method, 591 vixl::aarch64::Label* adrp_label = nullptr); 592 593 // Add a new .bss entry method patch for an instruction and return 594 // the label to be bound before the instruction. The instruction will be 595 // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` 596 // pointing to the associated ADRP patch label). 597 vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method, 598 vixl::aarch64::Label* adrp_label = nullptr); 599 600 // Add a new boot image type patch for an instruction and return the label 601 // to be bound before the instruction. The instruction will be either the 602 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing 603 // to the associated ADRP patch label). 604 vixl::aarch64::Label* NewBootImageTypePatch(const DexFile& dex_file, 605 dex::TypeIndex type_index, 606 vixl::aarch64::Label* adrp_label = nullptr); 607 608 // Add a new .bss entry type patch for an instruction and return the label 609 // to be bound before the instruction. The instruction will be either the 610 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing 611 // to the associated ADRP patch label). 612 vixl::aarch64::Label* NewBssEntryTypePatch(const DexFile& dex_file, 613 dex::TypeIndex type_index, 614 vixl::aarch64::Label* adrp_label = nullptr); 615 616 // Add a new boot image string patch for an instruction and return the label 617 // to be bound before the instruction. The instruction will be either the 618 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing 619 // to the associated ADRP patch label). 620 vixl::aarch64::Label* NewBootImageStringPatch(const DexFile& dex_file, 621 dex::StringIndex string_index, 622 vixl::aarch64::Label* adrp_label = nullptr); 623 624 // Add a new .bss entry string patch for an instruction and return the label 625 // to be bound before the instruction. The instruction will be either the 626 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing 627 // to the associated ADRP patch label). 628 vixl::aarch64::Label* NewStringBssEntryPatch(const DexFile& dex_file, 629 dex::StringIndex string_index, 630 vixl::aarch64::Label* adrp_label = nullptr); 631 632 // Emit the CBNZ instruction for baker read barrier and record 633 // the associated patch for AOT or slow path for JIT. 634 void EmitBakerReadBarrierCbnz(uint32_t custom_data); 635 636 vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address); 637 vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file, 638 dex::StringIndex string_index, 639 Handle<mirror::String> handle); 640 vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file, 641 dex::TypeIndex string_index, 642 Handle<mirror::Class> handle); 643 644 void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg); 645 void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label, 646 vixl::aarch64::Register out, 647 vixl::aarch64::Register base); 648 void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label, 649 vixl::aarch64::Register out, 650 vixl::aarch64::Register base); 651 652 void LoadBootImageAddress(vixl::aarch64::Register reg, uint32_t boot_image_reference); 653 void AllocateInstanceForIntrinsic(HInvokeStaticOrDirect* invoke, uint32_t boot_image_offset); 654 655 void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override; 656 bool NeedsThunkCode(const linker::LinkerPatch& patch) const override; 657 void EmitThunkCode(const linker::LinkerPatch& patch, 658 /*out*/ ArenaVector<uint8_t>* code, 659 /*out*/ std::string* debug_name) override; 660 661 void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override; 662 663 // Generate a GC root reference load: 664 // 665 // root <- *(obj + offset) 666 // 667 // while honoring read barriers based on read_barrier_option. 668 void GenerateGcRootFieldLoad(HInstruction* instruction, 669 Location root, 670 vixl::aarch64::Register obj, 671 uint32_t offset, 672 vixl::aarch64::Label* fixup_label, 673 ReadBarrierOption read_barrier_option); 674 // Generate MOV for the `old_value` in UnsafeCASObject and mark it with Baker read barrier. 675 void GenerateUnsafeCasOldValueMovWithBakerReadBarrier(vixl::aarch64::Register marked, 676 vixl::aarch64::Register old_value); 677 // Fast path implementation of ReadBarrier::Barrier for a heap 678 // reference field load when Baker's read barriers are used. 679 // Overload suitable for Unsafe.getObject/-Volatile() intrinsic. 680 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 681 Location ref, 682 vixl::aarch64::Register obj, 683 const vixl::aarch64::MemOperand& src, 684 bool needs_null_check, 685 bool use_load_acquire); 686 // Fast path implementation of ReadBarrier::Barrier for a heap 687 // reference field load when Baker's read barriers are used. 688 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 689 Location ref, 690 vixl::aarch64::Register obj, 691 uint32_t offset, 692 Location maybe_temp, 693 bool needs_null_check, 694 bool use_load_acquire); 695 // Fast path implementation of ReadBarrier::Barrier for a heap 696 // reference array load when Baker's read barriers are used. 697 void GenerateArrayLoadWithBakerReadBarrier(HArrayGet* instruction, 698 Location ref, 699 vixl::aarch64::Register obj, 700 uint32_t data_offset, 701 Location index, 702 bool needs_null_check); 703 704 // Emit code checking the status of the Marking Register, and 705 // aborting the program if MR does not match the value stored in the 706 // art::Thread object. Code is only emitted in debug mode and if 707 // CompilerOptions::EmitRunTimeChecksInDebugMode returns true. 708 // 709 // Argument `code` is used to identify the different occurrences of 710 // MaybeGenerateMarkingRegisterCheck in the code generator, and is 711 // passed to the BRK instruction. 712 // 713 // If `temp_loc` is a valid location, it is expected to be a 714 // register and will be used as a temporary to generate code; 715 // otherwise, a temporary will be fetched from the core register 716 // scratch pool. 717 virtual void MaybeGenerateMarkingRegisterCheck(int code, 718 Location temp_loc = Location::NoLocation()); 719 720 // Generate a read barrier for a heap reference within `instruction` 721 // using a slow path. 722 // 723 // A read barrier for an object reference read from the heap is 724 // implemented as a call to the artReadBarrierSlow runtime entry 725 // point, which is passed the values in locations `ref`, `obj`, and 726 // `offset`: 727 // 728 // mirror::Object* artReadBarrierSlow(mirror::Object* ref, 729 // mirror::Object* obj, 730 // uint32_t offset); 731 // 732 // The `out` location contains the value returned by 733 // artReadBarrierSlow. 734 // 735 // When `index` is provided (i.e. for array accesses), the offset 736 // value passed to artReadBarrierSlow is adjusted to take `index` 737 // into account. 738 void GenerateReadBarrierSlow(HInstruction* instruction, 739 Location out, 740 Location ref, 741 Location obj, 742 uint32_t offset, 743 Location index = Location::NoLocation()); 744 745 // If read barriers are enabled, generate a read barrier for a heap 746 // reference using a slow path. If heap poisoning is enabled, also 747 // unpoison the reference in `out`. 748 void MaybeGenerateReadBarrierSlow(HInstruction* instruction, 749 Location out, 750 Location ref, 751 Location obj, 752 uint32_t offset, 753 Location index = Location::NoLocation()); 754 755 // Generate a read barrier for a GC root within `instruction` using 756 // a slow path. 757 // 758 // A read barrier for an object reference GC root is implemented as 759 // a call to the artReadBarrierForRootSlow runtime entry point, 760 // which is passed the value in location `root`: 761 // 762 // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root); 763 // 764 // The `out` location contains the value returned by 765 // artReadBarrierForRootSlow. 766 void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); 767 768 void GenerateNop() override; 769 770 void GenerateImplicitNullCheck(HNullCheck* instruction) override; 771 void GenerateExplicitNullCheck(HNullCheck* instruction) override; 772 773 private: 774 // Encoding of thunk type and data for link-time generated thunks for Baker read barriers. 775 776 enum class BakerReadBarrierKind : uint8_t { 777 kField, // Field get or array get with constant offset (i.e. constant index). 778 kAcquire, // Volatile field get. 779 kArray, // Array get with index in register. 780 kGcRoot, // GC root load. 781 kLast = kGcRoot 782 }; 783 784 static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u; 785 786 static constexpr size_t kBitsForBakerReadBarrierKind = 787 MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); 788 static constexpr size_t kBakerReadBarrierBitsForRegister = 789 MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg); 790 using BakerReadBarrierKindField = 791 BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; 792 using BakerReadBarrierFirstRegField = 793 BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>; 794 using BakerReadBarrierSecondRegField = 795 BitField<uint32_t, 796 kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister, 797 kBakerReadBarrierBitsForRegister>; 798 799 static void CheckValidReg(uint32_t reg) { 800 DCHECK(reg < vixl::aarch64::lr.GetCode() && 801 reg != vixl::aarch64::ip0.GetCode() && 802 reg != vixl::aarch64::ip1.GetCode()) << reg; 803 } 804 805 static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { 806 CheckValidReg(base_reg); 807 CheckValidReg(holder_reg); 808 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | 809 BakerReadBarrierFirstRegField::Encode(base_reg) | 810 BakerReadBarrierSecondRegField::Encode(holder_reg); 811 } 812 813 static inline uint32_t EncodeBakerReadBarrierAcquireData(uint32_t base_reg, uint32_t holder_reg) { 814 CheckValidReg(base_reg); 815 CheckValidReg(holder_reg); 816 DCHECK_NE(base_reg, holder_reg); 817 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kAcquire) | 818 BakerReadBarrierFirstRegField::Encode(base_reg) | 819 BakerReadBarrierSecondRegField::Encode(holder_reg); 820 } 821 822 static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { 823 CheckValidReg(base_reg); 824 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | 825 BakerReadBarrierFirstRegField::Encode(base_reg) | 826 BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg); 827 } 828 829 static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { 830 CheckValidReg(root_reg); 831 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | 832 BakerReadBarrierFirstRegField::Encode(root_reg) | 833 BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg); 834 } 835 836 void CompileBakerReadBarrierThunk(Arm64Assembler& assembler, 837 uint32_t encoded_data, 838 /*out*/ std::string* debug_name); 839 840 using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>; 841 using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>; 842 using StringToLiteralMap = ArenaSafeMap<StringReference, 843 vixl::aarch64::Literal<uint32_t>*, 844 StringReferenceValueComparator>; 845 using TypeToLiteralMap = ArenaSafeMap<TypeReference, 846 vixl::aarch64::Literal<uint32_t>*, 847 TypeReferenceValueComparator>; 848 849 vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value); 850 vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value); 851 852 // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types, 853 // whether through .data.bimg.rel.ro, .bss, or directly in the boot image. 854 struct PcRelativePatchInfo : PatchInfo<vixl::aarch64::Label> { 855 PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx) 856 : PatchInfo<vixl::aarch64::Label>(dex_file, off_or_idx), pc_insn_label() { } 857 858 vixl::aarch64::Label* pc_insn_label; 859 }; 860 861 struct BakerReadBarrierPatchInfo { 862 explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { } 863 864 vixl::aarch64::Label label; 865 uint32_t custom_data; 866 }; 867 868 vixl::aarch64::Label* NewPcRelativePatch(const DexFile* dex_file, 869 uint32_t offset_or_index, 870 vixl::aarch64::Label* adrp_label, 871 ArenaDeque<PcRelativePatchInfo>* patches); 872 873 void EmitJumpTables(); 874 875 template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 876 static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos, 877 ArenaVector<linker::LinkerPatch>* linker_patches); 878 879 // Labels for each block that will be compiled. 880 // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory. 881 ArenaDeque<vixl::aarch64::Label> block_labels_; // Indexed by block id. 882 vixl::aarch64::Label frame_entry_label_; 883 ArenaVector<std::unique_ptr<JumpTableARM64>> jump_tables_; 884 885 LocationsBuilderARM64 location_builder_; 886 InstructionCodeGeneratorARM64 instruction_visitor_; 887 ParallelMoveResolverARM64 move_resolver_; 888 Arm64Assembler assembler_; 889 890 // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. 891 Uint32ToLiteralMap uint32_literals_; 892 // Deduplication map for 64-bit literals, used for non-patchable method address or method code. 893 Uint64ToLiteralMap uint64_literals_; 894 // PC-relative method patch info for kBootImageLinkTimePcRelative/BootImageRelRo. 895 // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). 896 ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_; 897 // PC-relative method patch info for kBssEntry. 898 ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; 899 // PC-relative type patch info for kBootImageLinkTimePcRelative. 900 ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_; 901 // PC-relative type patch info for kBssEntry. 902 ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; 903 // PC-relative String patch info for kBootImageLinkTimePcRelative. 904 ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_; 905 // PC-relative String patch info for kBssEntry. 906 ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_; 907 // PC-relative patch info for IntrinsicObjects. 908 ArenaDeque<PcRelativePatchInfo> boot_image_intrinsic_patches_; 909 // Baker read barrier patch info. 910 ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_; 911 912 // Patches for string literals in JIT compiled code. 913 StringToLiteralMap jit_string_patches_; 914 // Patches for class literals in JIT compiled code. 915 TypeToLiteralMap jit_class_patches_; 916 917 // Baker read barrier slow paths, mapping custom data (uint32_t) to label. 918 // Wrap the label to work around vixl::aarch64::Label being non-copyable 919 // and non-moveable and as such unusable in ArenaSafeMap<>. 920 struct LabelWrapper { 921 LabelWrapper(const LabelWrapper& src) 922 : label() { 923 DCHECK(!src.label.IsLinked() && !src.label.IsBound()); 924 } 925 LabelWrapper() = default; 926 vixl::aarch64::Label label; 927 }; 928 ArenaSafeMap<uint32_t, LabelWrapper> jit_baker_read_barrier_slow_paths_; 929 930 friend class linker::Arm64RelativePatcherTest; 931 DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); 932 }; 933 934 inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const { 935 return codegen_->GetAssembler(); 936 } 937 938 } // namespace arm64 939 } // namespace art 940 941 #endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ 942
