1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "code_generator_arm_vixl.h" 18 19 #include "arch/arm/asm_support_arm.h" 20 #include "arch/arm/instruction_set_features_arm.h" 21 #include "art_method.h" 22 #include "base/bit_utils.h" 23 #include "base/bit_utils_iterator.h" 24 #include "code_generator_utils.h" 25 #include "common_arm.h" 26 #include "compiled_method.h" 27 #include "entrypoints/quick/quick_entrypoints.h" 28 #include "gc/accounting/card_table.h" 29 #include "intrinsics_arm_vixl.h" 30 #include "linker/arm/relative_patcher_thumb2.h" 31 #include "mirror/array-inl.h" 32 #include "mirror/class-inl.h" 33 #include "thread.h" 34 #include "utils/arm/assembler_arm_vixl.h" 35 #include "utils/arm/managed_register_arm.h" 36 #include "utils/assembler.h" 37 #include "utils/stack_checks.h" 38 39 namespace art { 40 namespace arm { 41 42 namespace vixl32 = vixl::aarch32; 43 using namespace vixl32; // NOLINT(build/namespaces) 44 45 using helpers::DRegisterFrom; 46 using helpers::DWARFReg; 47 using helpers::HighDRegisterFrom; 48 using helpers::HighRegisterFrom; 49 using helpers::InputDRegisterAt; 50 using helpers::InputOperandAt; 51 using helpers::InputRegister; 52 using helpers::InputRegisterAt; 53 using helpers::InputSRegisterAt; 54 using helpers::InputVRegister; 55 using helpers::InputVRegisterAt; 56 using helpers::Int32ConstantFrom; 57 using helpers::Int64ConstantFrom; 58 using helpers::LocationFrom; 59 using helpers::LowRegisterFrom; 60 using helpers::LowSRegisterFrom; 61 using helpers::OperandFrom; 62 using helpers::OutputRegister; 63 using helpers::OutputSRegister; 64 using helpers::OutputVRegister; 65 using helpers::RegisterFrom; 66 using helpers::SRegisterFrom; 67 using helpers::Uint64ConstantFrom; 68 69 using vixl::ExactAssemblyScope; 70 using vixl::CodeBufferCheckScope; 71 72 using RegisterList = vixl32::RegisterList; 73 74 static bool ExpectedPairLayout(Location location) { 75 // We expected this for both core and fpu register pairs. 76 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 77 } 78 // Use a local definition to prevent copying mistakes. 79 static constexpr size_t kArmWordSize = static_cast<size_t>(kArmPointerSize); 80 static constexpr size_t kArmBitsPerWord = kArmWordSize * kBitsPerByte; 81 static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 82 83 // Reference load (except object array loads) is using LDR Rt, [Rn, #offset] which can handle 84 // offset < 4KiB. For offsets >= 4KiB, the load shall be emitted as two or more instructions. 85 // For the Baker read barrier implementation using link-generated thunks we need to split 86 // the offset explicitly. 87 constexpr uint32_t kReferenceLoadMinFarOffset = 4 * KB; 88 89 // Flags controlling the use of link-time generated thunks for Baker read barriers. 90 constexpr bool kBakerReadBarrierLinkTimeThunksEnableForFields = true; 91 constexpr bool kBakerReadBarrierLinkTimeThunksEnableForArrays = true; 92 constexpr bool kBakerReadBarrierLinkTimeThunksEnableForGcRoots = true; 93 94 // The reserved entrypoint register for link-time generated thunks. 95 const vixl32::Register kBakerCcEntrypointRegister = r4; 96 97 #ifdef __ 98 #error "ARM Codegen VIXL macro-assembler macro already defined." 99 #endif 100 101 // NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 102 #define __ down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()-> // NOLINT 103 #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 104 105 // Marker that code is yet to be, and must, be implemented. 106 #define TODO_VIXL32(level) LOG(level) << __PRETTY_FUNCTION__ << " unimplemented " 107 108 static inline void ExcludeIPAndBakerCcEntrypointRegister(UseScratchRegisterScope* temps, 109 HInstruction* instruction) { 110 DCHECK(temps->IsAvailable(ip)); 111 temps->Exclude(ip); 112 DCHECK(!temps->IsAvailable(kBakerCcEntrypointRegister)); 113 DCHECK_EQ(kBakerCcEntrypointRegister.GetCode(), 114 linker::Thumb2RelativePatcher::kBakerCcEntrypointRegister); 115 DCHECK_NE(instruction->GetLocations()->GetTempCount(), 0u); 116 DCHECK(RegisterFrom(instruction->GetLocations()->GetTemp( 117 instruction->GetLocations()->GetTempCount() - 1u)).Is(kBakerCcEntrypointRegister)); 118 } 119 120 static inline void EmitPlaceholderBne(CodeGeneratorARMVIXL* codegen, vixl32::Label* patch_label) { 121 ExactAssemblyScope eas(codegen->GetVIXLAssembler(), kMaxInstructionSizeInBytes); 122 __ bind(patch_label); 123 vixl32::Label placeholder_label; 124 __ b(ne, EncodingSize(Wide), &placeholder_label); // Placeholder, patched at link-time. 125 __ bind(&placeholder_label); 126 } 127 128 static inline bool CanEmitNarrowLdr(vixl32::Register rt, vixl32::Register rn, uint32_t offset) { 129 return rt.IsLow() && rn.IsLow() && offset < 32u; 130 } 131 132 class EmitAdrCode { 133 public: 134 EmitAdrCode(ArmVIXLMacroAssembler* assembler, vixl32::Register rd, vixl32::Label* label) 135 : assembler_(assembler), rd_(rd), label_(label) { 136 ExactAssemblyScope aas(assembler, kMaxInstructionSizeInBytes); 137 adr_location_ = assembler->GetCursorOffset(); 138 assembler->adr(EncodingSize(Wide), rd, label); 139 } 140 141 ~EmitAdrCode() { 142 DCHECK(label_->IsBound()); 143 // The ADR emitted by the assembler does not set the Thumb mode bit we need. 144 // TODO: Maybe extend VIXL to allow ADR for return address? 145 uint8_t* raw_adr = assembler_->GetBuffer()->GetOffsetAddress<uint8_t*>(adr_location_); 146 // Expecting ADR encoding T3 with `(offset & 1) == 0`. 147 DCHECK_EQ(raw_adr[1] & 0xfbu, 0xf2u); // Check bits 24-31, except 26. 148 DCHECK_EQ(raw_adr[0] & 0xffu, 0x0fu); // Check bits 16-23. 149 DCHECK_EQ(raw_adr[3] & 0x8fu, rd_.GetCode()); // Check bits 8-11 and 15. 150 DCHECK_EQ(raw_adr[2] & 0x01u, 0x00u); // Check bit 0, i.e. the `offset & 1`. 151 // Add the Thumb mode bit. 152 raw_adr[2] |= 0x01u; 153 } 154 155 private: 156 ArmVIXLMacroAssembler* const assembler_; 157 vixl32::Register rd_; 158 vixl32::Label* const label_; 159 int32_t adr_location_; 160 }; 161 162 // SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 163 // for each live D registers they treat two corresponding S registers as live ones. 164 // 165 // Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 166 // from a list of contiguous S registers a list of contiguous D registers (processing first/last 167 // S registers corner cases) and save/restore this new list treating them as D registers. 168 // - decreasing code size 169 // - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 170 // restored and then used in regular non SlowPath code as D register. 171 // 172 // For the following example (v means the S register is live): 173 // D names: | D0 | D1 | D2 | D4 | ... 174 // S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 175 // Live? | | v | v | v | v | v | v | | ... 176 // 177 // S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 178 // as D registers. 179 // 180 // TODO(VIXL): All this code should be unnecessary once the VIXL AArch32 backend provides helpers 181 // for lists of floating-point registers. 182 static size_t SaveContiguousSRegisterList(size_t first, 183 size_t last, 184 CodeGenerator* codegen, 185 size_t stack_offset) { 186 static_assert(kSRegSizeInBytes == kArmWordSize, "Broken assumption on reg/word sizes."); 187 static_assert(kDRegSizeInBytes == 2 * kArmWordSize, "Broken assumption on reg/word sizes."); 188 DCHECK_LE(first, last); 189 if ((first == last) && (first == 0)) { 190 __ Vstr(vixl32::SRegister(first), MemOperand(sp, stack_offset)); 191 return stack_offset + kSRegSizeInBytes; 192 } 193 if (first % 2 == 1) { 194 __ Vstr(vixl32::SRegister(first++), MemOperand(sp, stack_offset)); 195 stack_offset += kSRegSizeInBytes; 196 } 197 198 bool save_last = false; 199 if (last % 2 == 0) { 200 save_last = true; 201 --last; 202 } 203 204 if (first < last) { 205 vixl32::DRegister d_reg = vixl32::DRegister(first / 2); 206 DCHECK_EQ((last - first + 1) % 2, 0u); 207 size_t number_of_d_regs = (last - first + 1) / 2; 208 209 if (number_of_d_regs == 1) { 210 __ Vstr(d_reg, MemOperand(sp, stack_offset)); 211 } else if (number_of_d_regs > 1) { 212 UseScratchRegisterScope temps(down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); 213 vixl32::Register base = sp; 214 if (stack_offset != 0) { 215 base = temps.Acquire(); 216 __ Add(base, sp, Operand::From(stack_offset)); 217 } 218 __ Vstm(F64, base, NO_WRITE_BACK, DRegisterList(d_reg, number_of_d_regs)); 219 } 220 stack_offset += number_of_d_regs * kDRegSizeInBytes; 221 } 222 223 if (save_last) { 224 __ Vstr(vixl32::SRegister(last + 1), MemOperand(sp, stack_offset)); 225 stack_offset += kSRegSizeInBytes; 226 } 227 228 return stack_offset; 229 } 230 231 static size_t RestoreContiguousSRegisterList(size_t first, 232 size_t last, 233 CodeGenerator* codegen, 234 size_t stack_offset) { 235 static_assert(kSRegSizeInBytes == kArmWordSize, "Broken assumption on reg/word sizes."); 236 static_assert(kDRegSizeInBytes == 2 * kArmWordSize, "Broken assumption on reg/word sizes."); 237 DCHECK_LE(first, last); 238 if ((first == last) && (first == 0)) { 239 __ Vldr(vixl32::SRegister(first), MemOperand(sp, stack_offset)); 240 return stack_offset + kSRegSizeInBytes; 241 } 242 if (first % 2 == 1) { 243 __ Vldr(vixl32::SRegister(first++), MemOperand(sp, stack_offset)); 244 stack_offset += kSRegSizeInBytes; 245 } 246 247 bool restore_last = false; 248 if (last % 2 == 0) { 249 restore_last = true; 250 --last; 251 } 252 253 if (first < last) { 254 vixl32::DRegister d_reg = vixl32::DRegister(first / 2); 255 DCHECK_EQ((last - first + 1) % 2, 0u); 256 size_t number_of_d_regs = (last - first + 1) / 2; 257 if (number_of_d_regs == 1) { 258 __ Vldr(d_reg, MemOperand(sp, stack_offset)); 259 } else if (number_of_d_regs > 1) { 260 UseScratchRegisterScope temps(down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); 261 vixl32::Register base = sp; 262 if (stack_offset != 0) { 263 base = temps.Acquire(); 264 __ Add(base, sp, Operand::From(stack_offset)); 265 } 266 __ Vldm(F64, base, NO_WRITE_BACK, DRegisterList(d_reg, number_of_d_regs)); 267 } 268 stack_offset += number_of_d_regs * kDRegSizeInBytes; 269 } 270 271 if (restore_last) { 272 __ Vldr(vixl32::SRegister(last + 1), MemOperand(sp, stack_offset)); 273 stack_offset += kSRegSizeInBytes; 274 } 275 276 return stack_offset; 277 } 278 279 void SlowPathCodeARMVIXL::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 280 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 281 size_t orig_offset = stack_offset; 282 283 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 284 for (uint32_t i : LowToHighBits(core_spills)) { 285 // If the register holds an object, update the stack mask. 286 if (locations->RegisterContainsObject(i)) { 287 locations->SetStackBit(stack_offset / kVRegSize); 288 } 289 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 290 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 291 saved_core_stack_offsets_[i] = stack_offset; 292 stack_offset += kArmWordSize; 293 } 294 295 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 296 arm_codegen->GetAssembler()->StoreRegisterList(core_spills, orig_offset); 297 298 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 299 orig_offset = stack_offset; 300 for (uint32_t i : LowToHighBits(fp_spills)) { 301 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 302 saved_fpu_stack_offsets_[i] = stack_offset; 303 stack_offset += kArmWordSize; 304 } 305 306 stack_offset = orig_offset; 307 while (fp_spills != 0u) { 308 uint32_t begin = CTZ(fp_spills); 309 uint32_t tmp = fp_spills + (1u << begin); 310 fp_spills &= tmp; // Clear the contiguous range of 1s. 311 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 312 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 313 } 314 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 315 } 316 317 void SlowPathCodeARMVIXL::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 318 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 319 size_t orig_offset = stack_offset; 320 321 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 322 for (uint32_t i : LowToHighBits(core_spills)) { 323 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 324 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 325 stack_offset += kArmWordSize; 326 } 327 328 // TODO(VIXL): Check the coherency of stack_offset after this with a test. 329 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 330 arm_codegen->GetAssembler()->LoadRegisterList(core_spills, orig_offset); 331 332 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 333 while (fp_spills != 0u) { 334 uint32_t begin = CTZ(fp_spills); 335 uint32_t tmp = fp_spills + (1u << begin); 336 fp_spills &= tmp; // Clear the contiguous range of 1s. 337 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 338 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 339 } 340 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 341 } 342 343 class NullCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL { 344 public: 345 explicit NullCheckSlowPathARMVIXL(HNullCheck* instruction) : SlowPathCodeARMVIXL(instruction) {} 346 347 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 348 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 349 __ Bind(GetEntryLabel()); 350 if (instruction_->CanThrowIntoCatchBlock()) { 351 // Live registers will be restored in the catch block if caught. 352 SaveLiveRegisters(codegen, instruction_->GetLocations()); 353 } 354 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 355 instruction_, 356 instruction_->GetDexPc(), 357 this); 358 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 359 } 360 361 bool IsFatal() const OVERRIDE { return true; } 362 363 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARMVIXL"; } 364 365 private: 366 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARMVIXL); 367 }; 368 369 class DivZeroCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL { 370 public: 371 explicit DivZeroCheckSlowPathARMVIXL(HDivZeroCheck* instruction) 372 : SlowPathCodeARMVIXL(instruction) {} 373 374 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 375 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 376 __ Bind(GetEntryLabel()); 377 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 378 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 379 } 380 381 bool IsFatal() const OVERRIDE { return true; } 382 383 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARMVIXL"; } 384 385 private: 386 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARMVIXL); 387 }; 388 389 class SuspendCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL { 390 public: 391 SuspendCheckSlowPathARMVIXL(HSuspendCheck* instruction, HBasicBlock* successor) 392 : SlowPathCodeARMVIXL(instruction), successor_(successor) {} 393 394 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 395 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 396 __ Bind(GetEntryLabel()); 397 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 398 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 399 if (successor_ == nullptr) { 400 __ B(GetReturnLabel()); 401 } else { 402 __ B(arm_codegen->GetLabelOf(successor_)); 403 } 404 } 405 406 vixl32::Label* GetReturnLabel() { 407 DCHECK(successor_ == nullptr); 408 return &return_label_; 409 } 410 411 HBasicBlock* GetSuccessor() const { 412 return successor_; 413 } 414 415 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARMVIXL"; } 416 417 private: 418 // If not null, the block to branch to after the suspend check. 419 HBasicBlock* const successor_; 420 421 // If `successor_` is null, the label to branch to after the suspend check. 422 vixl32::Label return_label_; 423 424 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARMVIXL); 425 }; 426 427 class BoundsCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL { 428 public: 429 explicit BoundsCheckSlowPathARMVIXL(HBoundsCheck* instruction) 430 : SlowPathCodeARMVIXL(instruction) {} 431 432 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 433 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 434 LocationSummary* locations = instruction_->GetLocations(); 435 436 __ Bind(GetEntryLabel()); 437 if (instruction_->CanThrowIntoCatchBlock()) { 438 // Live registers will be restored in the catch block if caught. 439 SaveLiveRegisters(codegen, instruction_->GetLocations()); 440 } 441 // We're moving two locations to locations that could overlap, so we need a parallel 442 // move resolver. 443 InvokeRuntimeCallingConventionARMVIXL calling_convention; 444 codegen->EmitParallelMoves( 445 locations->InAt(0), 446 LocationFrom(calling_convention.GetRegisterAt(0)), 447 Primitive::kPrimInt, 448 locations->InAt(1), 449 LocationFrom(calling_convention.GetRegisterAt(1)), 450 Primitive::kPrimInt); 451 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 452 ? kQuickThrowStringBounds 453 : kQuickThrowArrayBounds; 454 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 455 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 456 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 457 } 458 459 bool IsFatal() const OVERRIDE { return true; } 460 461 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARMVIXL"; } 462 463 private: 464 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARMVIXL); 465 }; 466 467 class LoadClassSlowPathARMVIXL : public SlowPathCodeARMVIXL { 468 public: 469 LoadClassSlowPathARMVIXL(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) 470 : SlowPathCodeARMVIXL(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { 471 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 472 } 473 474 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 475 LocationSummary* locations = instruction_->GetLocations(); 476 Location out = locations->Out(); 477 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 478 479 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 480 __ Bind(GetEntryLabel()); 481 SaveLiveRegisters(codegen, locations); 482 483 InvokeRuntimeCallingConventionARMVIXL calling_convention; 484 // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. 485 DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); 486 bool is_load_class_bss_entry = 487 (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); 488 vixl32::Register entry_address; 489 if (is_load_class_bss_entry && call_saves_everything_except_r0) { 490 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 491 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 492 // the kSaveEverything call. 493 bool temp_is_r0 = temp.Is(calling_convention.GetRegisterAt(0)); 494 entry_address = temp_is_r0 ? RegisterFrom(out) : temp; 495 DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); 496 if (temp_is_r0) { 497 __ Mov(entry_address, temp); 498 } 499 } 500 dex::TypeIndex type_index = cls_->GetTypeIndex(); 501 __ Mov(calling_convention.GetRegisterAt(0), type_index.index_); 502 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 503 : kQuickInitializeType; 504 arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); 505 if (do_clinit_) { 506 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 507 } else { 508 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 509 } 510 511 // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. 512 if (is_load_class_bss_entry) { 513 if (call_saves_everything_except_r0) { 514 // The class entry address was preserved in `entry_address` thanks to kSaveEverything. 515 __ Str(r0, MemOperand(entry_address)); 516 } else { 517 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 518 UseScratchRegisterScope temps( 519 down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); 520 vixl32::Register temp = temps.Acquire(); 521 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = 522 arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); 523 arm_codegen->EmitMovwMovtPlaceholder(labels, temp); 524 __ Str(r0, MemOperand(temp)); 525 } 526 } 527 // Move the class to the desired location. 528 if (out.IsValid()) { 529 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 530 arm_codegen->Move32(locations->Out(), LocationFrom(r0)); 531 } 532 RestoreLiveRegisters(codegen, locations); 533 __ B(GetExitLabel()); 534 } 535 536 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARMVIXL"; } 537 538 private: 539 // The class this slow path will load. 540 HLoadClass* const cls_; 541 542 // The dex PC of `at_`. 543 const uint32_t dex_pc_; 544 545 // Whether to initialize the class. 546 const bool do_clinit_; 547 548 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARMVIXL); 549 }; 550 551 class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { 552 public: 553 explicit LoadStringSlowPathARMVIXL(HLoadString* instruction) 554 : SlowPathCodeARMVIXL(instruction) {} 555 556 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 557 DCHECK(instruction_->IsLoadString()); 558 DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); 559 LocationSummary* locations = instruction_->GetLocations(); 560 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 561 HLoadString* load = instruction_->AsLoadString(); 562 const dex::StringIndex string_index = load->GetStringIndex(); 563 vixl32::Register out = OutputRegister(load); 564 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 565 566 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 567 __ Bind(GetEntryLabel()); 568 SaveLiveRegisters(codegen, locations); 569 570 InvokeRuntimeCallingConventionARMVIXL calling_convention; 571 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 572 // the kSaveEverything call. 573 vixl32::Register entry_address; 574 if (call_saves_everything_except_r0) { 575 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 576 bool temp_is_r0 = (temp.Is(calling_convention.GetRegisterAt(0))); 577 entry_address = temp_is_r0 ? out : temp; 578 DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); 579 if (temp_is_r0) { 580 __ Mov(entry_address, temp); 581 } 582 } 583 584 __ Mov(calling_convention.GetRegisterAt(0), string_index.index_); 585 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 586 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 587 588 // Store the resolved String to the .bss entry. 589 if (call_saves_everything_except_r0) { 590 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 591 __ Str(r0, MemOperand(entry_address)); 592 } else { 593 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 594 UseScratchRegisterScope temps( 595 down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); 596 vixl32::Register temp = temps.Acquire(); 597 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = 598 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 599 arm_codegen->EmitMovwMovtPlaceholder(labels, temp); 600 __ Str(r0, MemOperand(temp)); 601 } 602 603 arm_codegen->Move32(locations->Out(), LocationFrom(r0)); 604 RestoreLiveRegisters(codegen, locations); 605 606 __ B(GetExitLabel()); 607 } 608 609 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARMVIXL"; } 610 611 private: 612 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARMVIXL); 613 }; 614 615 class TypeCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL { 616 public: 617 TypeCheckSlowPathARMVIXL(HInstruction* instruction, bool is_fatal) 618 : SlowPathCodeARMVIXL(instruction), is_fatal_(is_fatal) {} 619 620 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 621 LocationSummary* locations = instruction_->GetLocations(); 622 DCHECK(instruction_->IsCheckCast() 623 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 624 625 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 626 __ Bind(GetEntryLabel()); 627 628 if (!is_fatal_) { 629 SaveLiveRegisters(codegen, locations); 630 } 631 632 // We're moving two locations to locations that could overlap, so we need a parallel 633 // move resolver. 634 InvokeRuntimeCallingConventionARMVIXL calling_convention; 635 636 codegen->EmitParallelMoves(locations->InAt(0), 637 LocationFrom(calling_convention.GetRegisterAt(0)), 638 Primitive::kPrimNot, 639 locations->InAt(1), 640 LocationFrom(calling_convention.GetRegisterAt(1)), 641 Primitive::kPrimNot); 642 if (instruction_->IsInstanceOf()) { 643 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 644 instruction_, 645 instruction_->GetDexPc(), 646 this); 647 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 648 arm_codegen->Move32(locations->Out(), LocationFrom(r0)); 649 } else { 650 DCHECK(instruction_->IsCheckCast()); 651 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 652 instruction_, 653 instruction_->GetDexPc(), 654 this); 655 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 656 } 657 658 if (!is_fatal_) { 659 RestoreLiveRegisters(codegen, locations); 660 __ B(GetExitLabel()); 661 } 662 } 663 664 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARMVIXL"; } 665 666 bool IsFatal() const OVERRIDE { return is_fatal_; } 667 668 private: 669 const bool is_fatal_; 670 671 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARMVIXL); 672 }; 673 674 class DeoptimizationSlowPathARMVIXL : public SlowPathCodeARMVIXL { 675 public: 676 explicit DeoptimizationSlowPathARMVIXL(HDeoptimize* instruction) 677 : SlowPathCodeARMVIXL(instruction) {} 678 679 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 680 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 681 __ Bind(GetEntryLabel()); 682 LocationSummary* locations = instruction_->GetLocations(); 683 SaveLiveRegisters(codegen, locations); 684 InvokeRuntimeCallingConventionARMVIXL calling_convention; 685 __ Mov(calling_convention.GetRegisterAt(0), 686 static_cast<uint32_t>(instruction_->AsDeoptimize()->GetDeoptimizationKind())); 687 688 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 689 CheckEntrypointTypes<kQuickDeoptimize, void, DeoptimizationKind>(); 690 } 691 692 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARMVIXL"; } 693 694 private: 695 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARMVIXL); 696 }; 697 698 class ArraySetSlowPathARMVIXL : public SlowPathCodeARMVIXL { 699 public: 700 explicit ArraySetSlowPathARMVIXL(HInstruction* instruction) : SlowPathCodeARMVIXL(instruction) {} 701 702 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 703 LocationSummary* locations = instruction_->GetLocations(); 704 __ Bind(GetEntryLabel()); 705 SaveLiveRegisters(codegen, locations); 706 707 InvokeRuntimeCallingConventionARMVIXL calling_convention; 708 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 709 parallel_move.AddMove( 710 locations->InAt(0), 711 LocationFrom(calling_convention.GetRegisterAt(0)), 712 Primitive::kPrimNot, 713 nullptr); 714 parallel_move.AddMove( 715 locations->InAt(1), 716 LocationFrom(calling_convention.GetRegisterAt(1)), 717 Primitive::kPrimInt, 718 nullptr); 719 parallel_move.AddMove( 720 locations->InAt(2), 721 LocationFrom(calling_convention.GetRegisterAt(2)), 722 Primitive::kPrimNot, 723 nullptr); 724 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 725 726 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 727 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 728 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 729 RestoreLiveRegisters(codegen, locations); 730 __ B(GetExitLabel()); 731 } 732 733 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARMVIXL"; } 734 735 private: 736 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARMVIXL); 737 }; 738 739 // Abstract base class for read barrier slow paths marking a reference 740 // `ref`. 741 // 742 // Argument `entrypoint` must be a register location holding the read 743 // barrier marking runtime entry point to be invoked or an empty 744 // location; in the latter case, the read barrier marking runtime 745 // entry point will be loaded by the slow path code itself. 746 class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { 747 protected: 748 ReadBarrierMarkSlowPathBaseARMVIXL(HInstruction* instruction, Location ref, Location entrypoint) 749 : SlowPathCodeARMVIXL(instruction), ref_(ref), entrypoint_(entrypoint) { 750 DCHECK(kEmitCompilerReadBarrier); 751 } 752 753 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARMVIXL"; } 754 755 // Generate assembly code calling the read barrier marking runtime 756 // entry point (ReadBarrierMarkRegX). 757 void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) { 758 vixl32::Register ref_reg = RegisterFrom(ref_); 759 760 // No need to save live registers; it's taken care of by the 761 // entrypoint. Also, there is no need to update the stack mask, 762 // as this runtime call will not trigger a garbage collection. 763 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 764 DCHECK(!ref_reg.Is(sp)); 765 DCHECK(!ref_reg.Is(lr)); 766 DCHECK(!ref_reg.Is(pc)); 767 // IP is used internally by the ReadBarrierMarkRegX entry point 768 // as a temporary, it cannot be the entry point's input/output. 769 DCHECK(!ref_reg.Is(ip)); 770 DCHECK(ref_reg.IsRegister()) << ref_reg; 771 // "Compact" slow path, saving two moves. 772 // 773 // Instead of using the standard runtime calling convention (input 774 // and output in R0): 775 // 776 // R0 <- ref 777 // R0 <- ReadBarrierMark(R0) 778 // ref <- R0 779 // 780 // we just use rX (the register containing `ref`) as input and output 781 // of a dedicated entrypoint: 782 // 783 // rX <- ReadBarrierMarkRegX(rX) 784 // 785 if (entrypoint_.IsValid()) { 786 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 787 __ Blx(RegisterFrom(entrypoint_)); 788 } else { 789 // Entrypoint is not already loaded, load from the thread. 790 int32_t entry_point_offset = 791 Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); 792 // This runtime call does not require a stack map. 793 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 794 } 795 } 796 797 // The location (register) of the marked object reference. 798 const Location ref_; 799 800 // The location of the entrypoint if already loaded. 801 const Location entrypoint_; 802 803 private: 804 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARMVIXL); 805 }; 806 807 // Slow path marking an object reference `ref` during a read 808 // barrier. The field `obj.field` in the object `obj` holding this 809 // reference does not get updated by this slow path after marking. 810 // 811 // This means that after the execution of this slow path, `ref` will 812 // always be up-to-date, but `obj.field` may not; i.e., after the 813 // flip, `ref` will be a to-space reference, but `obj.field` will 814 // probably still be a from-space reference (unless it gets updated by 815 // another thread, or if another thread installed another object 816 // reference (different from `ref`) in `obj.field`). 817 // 818 // Argument `entrypoint` must be a register location holding the read 819 // barrier marking runtime entry point to be invoked or an empty 820 // location; in the latter case, the read barrier marking runtime 821 // entry point will be loaded by the slow path code itself. 822 class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { 823 public: 824 ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction, 825 Location ref, 826 Location entrypoint = Location::NoLocation()) 827 : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint) { 828 DCHECK(kEmitCompilerReadBarrier); 829 } 830 831 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; } 832 833 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 834 LocationSummary* locations = instruction_->GetLocations(); 835 DCHECK(locations->CanCall()); 836 DCHECK(ref_.IsRegister()) << ref_; 837 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg(); 838 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 839 << "Unexpected instruction in read barrier marking slow path: " 840 << instruction_->DebugName(); 841 842 __ Bind(GetEntryLabel()); 843 GenerateReadBarrierMarkRuntimeCall(codegen); 844 __ B(GetExitLabel()); 845 } 846 847 private: 848 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL); 849 }; 850 851 // Slow path loading `obj`'s lock word, loading a reference from 852 // object `*(obj + offset + (index << scale_factor))` into `ref`, and 853 // marking `ref` if `obj` is gray according to the lock word (Baker 854 // read barrier). The field `obj.field` in the object `obj` holding 855 // this reference does not get updated by this slow path after marking 856 // (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL 857 // below for that). 858 // 859 // This means that after the execution of this slow path, `ref` will 860 // always be up-to-date, but `obj.field` may not; i.e., after the 861 // flip, `ref` will be a to-space reference, but `obj.field` will 862 // probably still be a from-space reference (unless it gets updated by 863 // another thread, or if another thread installed another object 864 // reference (different from `ref`) in `obj.field`). 865 // 866 // Argument `entrypoint` must be a register location holding the read 867 // barrier marking runtime entry point to be invoked or an empty 868 // location; in the latter case, the read barrier marking runtime 869 // entry point will be loaded by the slow path code itself. 870 class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { 871 public: 872 LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(HInstruction* instruction, 873 Location ref, 874 vixl32::Register obj, 875 uint32_t offset, 876 Location index, 877 ScaleFactor scale_factor, 878 bool needs_null_check, 879 vixl32::Register temp, 880 Location entrypoint = Location::NoLocation()) 881 : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint), 882 obj_(obj), 883 offset_(offset), 884 index_(index), 885 scale_factor_(scale_factor), 886 needs_null_check_(needs_null_check), 887 temp_(temp) { 888 DCHECK(kEmitCompilerReadBarrier); 889 DCHECK(kUseBakerReadBarrier); 890 } 891 892 const char* GetDescription() const OVERRIDE { 893 return "LoadReferenceWithBakerReadBarrierSlowPathARMVIXL"; 894 } 895 896 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 897 LocationSummary* locations = instruction_->GetLocations(); 898 vixl32::Register ref_reg = RegisterFrom(ref_); 899 DCHECK(locations->CanCall()); 900 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg; 901 DCHECK(instruction_->IsInstanceFieldGet() || 902 instruction_->IsStaticFieldGet() || 903 instruction_->IsArrayGet() || 904 instruction_->IsArraySet() || 905 instruction_->IsInstanceOf() || 906 instruction_->IsCheckCast() || 907 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 908 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 909 << "Unexpected instruction in read barrier marking slow path: " 910 << instruction_->DebugName(); 911 // The read barrier instrumentation of object ArrayGet 912 // instructions does not support the HIntermediateAddress 913 // instruction. 914 DCHECK(!(instruction_->IsArrayGet() && 915 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 916 917 // Temporary register `temp_`, used to store the lock word, must 918 // not be IP, as we may use it to emit the reference load (in the 919 // call to GenerateRawReferenceLoad below), and we need the lock 920 // word to still be in `temp_` after the reference load. 921 DCHECK(!temp_.Is(ip)); 922 923 __ Bind(GetEntryLabel()); 924 925 // When using MaybeGenerateReadBarrierSlow, the read barrier call is 926 // inserted after the original load. However, in fast path based 927 // Baker's read barriers, we need to perform the load of 928 // mirror::Object::monitor_ *before* the original reference load. 929 // This load-load ordering is required by the read barrier. 930 // The slow path (for Baker's algorithm) should look like: 931 // 932 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 933 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 934 // HeapReference<mirror::Object> ref = *src; // Original reference load. 935 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 936 // if (is_gray) { 937 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 938 // } 939 // 940 // Note: the original implementation in ReadBarrier::Barrier is 941 // slightly more complex as it performs additional checks that we do 942 // not do here for performance reasons. 943 944 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 945 946 // /* int32_t */ monitor = obj->monitor_ 947 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 948 arm_codegen->GetAssembler()->LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset); 949 if (needs_null_check_) { 950 codegen->MaybeRecordImplicitNullCheck(instruction_); 951 } 952 // /* LockWord */ lock_word = LockWord(monitor) 953 static_assert(sizeof(LockWord) == sizeof(int32_t), 954 "art::LockWord and int32_t have different sizes."); 955 956 // Introduce a dependency on the lock_word including the rb_state, 957 // which shall prevent load-load reordering without using 958 // a memory barrier (which would be more expensive). 959 // `obj` is unchanged by this operation, but its value now depends 960 // on `temp`. 961 __ Add(obj_, obj_, Operand(temp_, ShiftType::LSR, 32)); 962 963 // The actual reference load. 964 // A possible implicit null check has already been handled above. 965 arm_codegen->GenerateRawReferenceLoad( 966 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 967 968 // Mark the object `ref` when `obj` is gray. 969 // 970 // if (rb_state == ReadBarrier::GrayState()) 971 // ref = ReadBarrier::Mark(ref); 972 // 973 // Given the numeric representation, it's enough to check the low bit of the 974 // rb_state. We do that by shifting the bit out of the lock word with LSRS 975 // which can be a 16-bit instruction unlike the TST immediate. 976 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 977 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 978 __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1); 979 __ B(cc, GetExitLabel()); // Carry flag is the last bit shifted out by LSRS. 980 GenerateReadBarrierMarkRuntimeCall(codegen); 981 982 __ B(GetExitLabel()); 983 } 984 985 private: 986 // The register containing the object holding the marked object reference field. 987 vixl32::Register obj_; 988 // The offset, index and scale factor to access the reference in `obj_`. 989 uint32_t offset_; 990 Location index_; 991 ScaleFactor scale_factor_; 992 // Is a null check required? 993 bool needs_null_check_; 994 // A temporary register used to hold the lock word of `obj_`. 995 vixl32::Register temp_; 996 997 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARMVIXL); 998 }; 999 1000 // Slow path loading `obj`'s lock word, loading a reference from 1001 // object `*(obj + offset + (index << scale_factor))` into `ref`, and 1002 // marking `ref` if `obj` is gray according to the lock word (Baker 1003 // read barrier). If needed, this slow path also atomically updates 1004 // the field `obj.field` in the object `obj` holding this reference 1005 // after marking (contrary to 1006 // LoadReferenceWithBakerReadBarrierSlowPathARMVIXL above, which never 1007 // tries to update `obj.field`). 1008 // 1009 // This means that after the execution of this slow path, both `ref` 1010 // and `obj.field` will be up-to-date; i.e., after the flip, both will 1011 // hold the same to-space reference (unless another thread installed 1012 // another object reference (different from `ref`) in `obj.field`). 1013 // 1014 // Argument `entrypoint` must be a register location holding the read 1015 // barrier marking runtime entry point to be invoked or an empty 1016 // location; in the latter case, the read barrier marking runtime 1017 // entry point will be loaded by the slow path code itself. 1018 class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL 1019 : public ReadBarrierMarkSlowPathBaseARMVIXL { 1020 public: 1021 LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL( 1022 HInstruction* instruction, 1023 Location ref, 1024 vixl32::Register obj, 1025 uint32_t offset, 1026 Location index, 1027 ScaleFactor scale_factor, 1028 bool needs_null_check, 1029 vixl32::Register temp1, 1030 vixl32::Register temp2, 1031 Location entrypoint = Location::NoLocation()) 1032 : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint), 1033 obj_(obj), 1034 offset_(offset), 1035 index_(index), 1036 scale_factor_(scale_factor), 1037 needs_null_check_(needs_null_check), 1038 temp1_(temp1), 1039 temp2_(temp2) { 1040 DCHECK(kEmitCompilerReadBarrier); 1041 DCHECK(kUseBakerReadBarrier); 1042 } 1043 1044 const char* GetDescription() const OVERRIDE { 1045 return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL"; 1046 } 1047 1048 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1049 LocationSummary* locations = instruction_->GetLocations(); 1050 vixl32::Register ref_reg = RegisterFrom(ref_); 1051 DCHECK(locations->CanCall()); 1052 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg; 1053 DCHECK_NE(ref_.reg(), LocationFrom(temp1_).reg()); 1054 1055 // This slow path is only used by the UnsafeCASObject intrinsic at the moment. 1056 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 1057 << "Unexpected instruction in read barrier marking and field updating slow path: " 1058 << instruction_->DebugName(); 1059 DCHECK(instruction_->GetLocations()->Intrinsified()); 1060 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 1061 DCHECK_EQ(offset_, 0u); 1062 DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1); 1063 Location field_offset = index_; 1064 DCHECK(field_offset.IsRegisterPair()) << field_offset; 1065 1066 // Temporary register `temp1_`, used to store the lock word, must 1067 // not be IP, as we may use it to emit the reference load (in the 1068 // call to GenerateRawReferenceLoad below), and we need the lock 1069 // word to still be in `temp1_` after the reference load. 1070 DCHECK(!temp1_.Is(ip)); 1071 1072 __ Bind(GetEntryLabel()); 1073 1074 // The implementation is similar to LoadReferenceWithBakerReadBarrierSlowPathARMVIXL's: 1075 // 1076 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 1077 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 1078 // HeapReference<mirror::Object> ref = *src; // Original reference load. 1079 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 1080 // if (is_gray) { 1081 // old_ref = ref; 1082 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 1083 // compareAndSwapObject(obj, field_offset, old_ref, ref); 1084 // } 1085 1086 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 1087 1088 // /* int32_t */ monitor = obj->monitor_ 1089 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 1090 arm_codegen->GetAssembler()->LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset); 1091 if (needs_null_check_) { 1092 codegen->MaybeRecordImplicitNullCheck(instruction_); 1093 } 1094 // /* LockWord */ lock_word = LockWord(monitor) 1095 static_assert(sizeof(LockWord) == sizeof(int32_t), 1096 "art::LockWord and int32_t have different sizes."); 1097 1098 // Introduce a dependency on the lock_word including the rb_state, 1099 // which shall prevent load-load reordering without using 1100 // a memory barrier (which would be more expensive). 1101 // `obj` is unchanged by this operation, but its value now depends 1102 // on `temp`. 1103 __ Add(obj_, obj_, Operand(temp1_, ShiftType::LSR, 32)); 1104 1105 // The actual reference load. 1106 // A possible implicit null check has already been handled above. 1107 arm_codegen->GenerateRawReferenceLoad( 1108 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 1109 1110 // Mark the object `ref` when `obj` is gray. 1111 // 1112 // if (rb_state == ReadBarrier::GrayState()) 1113 // ref = ReadBarrier::Mark(ref); 1114 // 1115 // Given the numeric representation, it's enough to check the low bit of the 1116 // rb_state. We do that by shifting the bit out of the lock word with LSRS 1117 // which can be a 16-bit instruction unlike the TST immediate. 1118 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 1119 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 1120 __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1); 1121 __ B(cc, GetExitLabel()); // Carry flag is the last bit shifted out by LSRS. 1122 1123 // Save the old value of the reference before marking it. 1124 // Note that we cannot use IP to save the old reference, as IP is 1125 // used internally by the ReadBarrierMarkRegX entry point, and we 1126 // need the old reference after the call to that entry point. 1127 DCHECK(!temp1_.Is(ip)); 1128 __ Mov(temp1_, ref_reg); 1129 1130 GenerateReadBarrierMarkRuntimeCall(codegen); 1131 1132 // If the new reference is different from the old reference, 1133 // update the field in the holder (`*(obj_ + field_offset)`). 1134 // 1135 // Note that this field could also hold a different object, if 1136 // another thread had concurrently changed it. In that case, the 1137 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 1138 // (CAS) operation below would abort the CAS, leaving the field 1139 // as-is. 1140 __ Cmp(temp1_, ref_reg); 1141 __ B(eq, GetExitLabel()); 1142 1143 // Update the the holder's field atomically. This may fail if 1144 // mutator updates before us, but it's OK. This is achieved 1145 // using a strong compare-and-set (CAS) operation with relaxed 1146 // memory synchronization ordering, where the expected value is 1147 // the old reference and the desired value is the new reference. 1148 1149 UseScratchRegisterScope temps(arm_codegen->GetVIXLAssembler()); 1150 // Convenience aliases. 1151 vixl32::Register base = obj_; 1152 // The UnsafeCASObject intrinsic uses a register pair as field 1153 // offset ("long offset"), of which only the low part contains 1154 // data. 1155 vixl32::Register offset = LowRegisterFrom(field_offset); 1156 vixl32::Register expected = temp1_; 1157 vixl32::Register value = ref_reg; 1158 vixl32::Register tmp_ptr = temps.Acquire(); // Pointer to actual memory. 1159 vixl32::Register tmp = temp2_; // Value in memory. 1160 1161 __ Add(tmp_ptr, base, offset); 1162 1163 if (kPoisonHeapReferences) { 1164 arm_codegen->GetAssembler()->PoisonHeapReference(expected); 1165 if (value.Is(expected)) { 1166 // Do not poison `value`, as it is the same register as 1167 // `expected`, which has just been poisoned. 1168 } else { 1169 arm_codegen->GetAssembler()->PoisonHeapReference(value); 1170 } 1171 } 1172 1173 // do { 1174 // tmp = [r_ptr] - expected; 1175 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 1176 1177 vixl32::Label loop_head, exit_loop; 1178 __ Bind(&loop_head); 1179 1180 __ Ldrex(tmp, MemOperand(tmp_ptr)); 1181 1182 __ Subs(tmp, tmp, expected); 1183 1184 { 1185 ExactAssemblyScope aas(arm_codegen->GetVIXLAssembler(), 1186 2 * kMaxInstructionSizeInBytes, 1187 CodeBufferCheckScope::kMaximumSize); 1188 1189 __ it(ne); 1190 __ clrex(ne); 1191 } 1192 1193 __ B(ne, &exit_loop, /* far_target */ false); 1194 1195 __ Strex(tmp, value, MemOperand(tmp_ptr)); 1196 __ Cmp(tmp, 1); 1197 __ B(eq, &loop_head, /* far_target */ false); 1198 1199 __ Bind(&exit_loop); 1200 1201 if (kPoisonHeapReferences) { 1202 arm_codegen->GetAssembler()->UnpoisonHeapReference(expected); 1203 if (value.Is(expected)) { 1204 // Do not unpoison `value`, as it is the same register as 1205 // `expected`, which has just been unpoisoned. 1206 } else { 1207 arm_codegen->GetAssembler()->UnpoisonHeapReference(value); 1208 } 1209 } 1210 1211 __ B(GetExitLabel()); 1212 } 1213 1214 private: 1215 // The register containing the object holding the marked object reference field. 1216 const vixl32::Register obj_; 1217 // The offset, index and scale factor to access the reference in `obj_`. 1218 uint32_t offset_; 1219 Location index_; 1220 ScaleFactor scale_factor_; 1221 // Is a null check required? 1222 bool needs_null_check_; 1223 // A temporary register used to hold the lock word of `obj_`; and 1224 // also to hold the original reference value, when the reference is 1225 // marked. 1226 const vixl32::Register temp1_; 1227 // A temporary register used in the implementation of the CAS, to 1228 // update the object's reference field. 1229 const vixl32::Register temp2_; 1230 1231 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL); 1232 }; 1233 1234 // Slow path generating a read barrier for a heap reference. 1235 class ReadBarrierForHeapReferenceSlowPathARMVIXL : public SlowPathCodeARMVIXL { 1236 public: 1237 ReadBarrierForHeapReferenceSlowPathARMVIXL(HInstruction* instruction, 1238 Location out, 1239 Location ref, 1240 Location obj, 1241 uint32_t offset, 1242 Location index) 1243 : SlowPathCodeARMVIXL(instruction), 1244 out_(out), 1245 ref_(ref), 1246 obj_(obj), 1247 offset_(offset), 1248 index_(index) { 1249 DCHECK(kEmitCompilerReadBarrier); 1250 // If `obj` is equal to `out` or `ref`, it means the initial object 1251 // has been overwritten by (or after) the heap object reference load 1252 // to be instrumented, e.g.: 1253 // 1254 // __ LoadFromOffset(kLoadWord, out, out, offset); 1255 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 1256 // 1257 // In that case, we have lost the information about the original 1258 // object, and the emitted read barrier cannot work properly. 1259 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 1260 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 1261 } 1262 1263 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1264 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 1265 LocationSummary* locations = instruction_->GetLocations(); 1266 vixl32::Register reg_out = RegisterFrom(out_); 1267 DCHECK(locations->CanCall()); 1268 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.GetCode())); 1269 DCHECK(instruction_->IsInstanceFieldGet() || 1270 instruction_->IsStaticFieldGet() || 1271 instruction_->IsArrayGet() || 1272 instruction_->IsInstanceOf() || 1273 instruction_->IsCheckCast() || 1274 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 1275 << "Unexpected instruction in read barrier for heap reference slow path: " 1276 << instruction_->DebugName(); 1277 // The read barrier instrumentation of object ArrayGet 1278 // instructions does not support the HIntermediateAddress 1279 // instruction. 1280 DCHECK(!(instruction_->IsArrayGet() && 1281 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 1282 1283 __ Bind(GetEntryLabel()); 1284 SaveLiveRegisters(codegen, locations); 1285 1286 // We may have to change the index's value, but as `index_` is a 1287 // constant member (like other "inputs" of this slow path), 1288 // introduce a copy of it, `index`. 1289 Location index = index_; 1290 if (index_.IsValid()) { 1291 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 1292 if (instruction_->IsArrayGet()) { 1293 // Compute the actual memory offset and store it in `index`. 1294 vixl32::Register index_reg = RegisterFrom(index_); 1295 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg.GetCode())); 1296 if (codegen->IsCoreCalleeSaveRegister(index_reg.GetCode())) { 1297 // We are about to change the value of `index_reg` (see the 1298 // calls to art::arm::ArmVIXLMacroAssembler::Lsl and 1299 // art::arm::ArmVIXLMacroAssembler::Add below), but it has 1300 // not been saved by the previous call to 1301 // art::SlowPathCode::SaveLiveRegisters, as it is a 1302 // callee-save register -- 1303 // art::SlowPathCode::SaveLiveRegisters does not consider 1304 // callee-save registers, as it has been designed with the 1305 // assumption that callee-save registers are supposed to be 1306 // handled by the called function. So, as a callee-save 1307 // register, `index_reg` _would_ eventually be saved onto 1308 // the stack, but it would be too late: we would have 1309 // changed its value earlier. Therefore, we manually save 1310 // it here into another freely available register, 1311 // `free_reg`, chosen of course among the caller-save 1312 // registers (as a callee-save `free_reg` register would 1313 // exhibit the same problem). 1314 // 1315 // Note we could have requested a temporary register from 1316 // the register allocator instead; but we prefer not to, as 1317 // this is a slow path, and we know we can find a 1318 // caller-save register that is available. 1319 vixl32::Register free_reg = FindAvailableCallerSaveRegister(codegen); 1320 __ Mov(free_reg, index_reg); 1321 index_reg = free_reg; 1322 index = LocationFrom(index_reg); 1323 } else { 1324 // The initial register stored in `index_` has already been 1325 // saved in the call to art::SlowPathCode::SaveLiveRegisters 1326 // (as it is not a callee-save register), so we can freely 1327 // use it. 1328 } 1329 // Shifting the index value contained in `index_reg` by the scale 1330 // factor (2) cannot overflow in practice, as the runtime is 1331 // unable to allocate object arrays with a size larger than 1332 // 2^26 - 1 (that is, 2^28 - 4 bytes). 1333 __ Lsl(index_reg, index_reg, TIMES_4); 1334 static_assert( 1335 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 1336 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 1337 __ Add(index_reg, index_reg, offset_); 1338 } else { 1339 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 1340 // intrinsics, `index_` is not shifted by a scale factor of 2 1341 // (as in the case of ArrayGet), as it is actually an offset 1342 // to an object field within an object. 1343 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 1344 DCHECK(instruction_->GetLocations()->Intrinsified()); 1345 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 1346 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 1347 << instruction_->AsInvoke()->GetIntrinsic(); 1348 DCHECK_EQ(offset_, 0U); 1349 DCHECK(index_.IsRegisterPair()); 1350 // UnsafeGet's offset location is a register pair, the low 1351 // part contains the correct offset. 1352 index = index_.ToLow(); 1353 } 1354 } 1355 1356 // We're moving two or three locations to locations that could 1357 // overlap, so we need a parallel move resolver. 1358 InvokeRuntimeCallingConventionARMVIXL calling_convention; 1359 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1360 parallel_move.AddMove(ref_, 1361 LocationFrom(calling_convention.GetRegisterAt(0)), 1362 Primitive::kPrimNot, 1363 nullptr); 1364 parallel_move.AddMove(obj_, 1365 LocationFrom(calling_convention.GetRegisterAt(1)), 1366 Primitive::kPrimNot, 1367 nullptr); 1368 if (index.IsValid()) { 1369 parallel_move.AddMove(index, 1370 LocationFrom(calling_convention.GetRegisterAt(2)), 1371 Primitive::kPrimInt, 1372 nullptr); 1373 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1374 } else { 1375 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1376 __ Mov(calling_convention.GetRegisterAt(2), offset_); 1377 } 1378 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1379 CheckEntrypointTypes< 1380 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1381 arm_codegen->Move32(out_, LocationFrom(r0)); 1382 1383 RestoreLiveRegisters(codegen, locations); 1384 __ B(GetExitLabel()); 1385 } 1386 1387 const char* GetDescription() const OVERRIDE { 1388 return "ReadBarrierForHeapReferenceSlowPathARMVIXL"; 1389 } 1390 1391 private: 1392 vixl32::Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1393 uint32_t ref = RegisterFrom(ref_).GetCode(); 1394 uint32_t obj = RegisterFrom(obj_).GetCode(); 1395 for (uint32_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1396 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1397 return vixl32::Register(i); 1398 } 1399 } 1400 // We shall never fail to find a free caller-save register, as 1401 // there are more than two core caller-save registers on ARM 1402 // (meaning it is possible to find one which is different from 1403 // `ref` and `obj`). 1404 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1405 LOG(FATAL) << "Could not find a free caller-save register"; 1406 UNREACHABLE(); 1407 } 1408 1409 const Location out_; 1410 const Location ref_; 1411 const Location obj_; 1412 const uint32_t offset_; 1413 // An additional location containing an index to an array. 1414 // Only used for HArrayGet and the UnsafeGetObject & 1415 // UnsafeGetObjectVolatile intrinsics. 1416 const Location index_; 1417 1418 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARMVIXL); 1419 }; 1420 1421 // Slow path generating a read barrier for a GC root. 1422 class ReadBarrierForRootSlowPathARMVIXL : public SlowPathCodeARMVIXL { 1423 public: 1424 ReadBarrierForRootSlowPathARMVIXL(HInstruction* instruction, Location out, Location root) 1425 : SlowPathCodeARMVIXL(instruction), out_(out), root_(root) { 1426 DCHECK(kEmitCompilerReadBarrier); 1427 } 1428 1429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1430 LocationSummary* locations = instruction_->GetLocations(); 1431 vixl32::Register reg_out = RegisterFrom(out_); 1432 DCHECK(locations->CanCall()); 1433 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.GetCode())); 1434 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1435 << "Unexpected instruction in read barrier for GC root slow path: " 1436 << instruction_->DebugName(); 1437 1438 __ Bind(GetEntryLabel()); 1439 SaveLiveRegisters(codegen, locations); 1440 1441 InvokeRuntimeCallingConventionARMVIXL calling_convention; 1442 CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); 1443 arm_codegen->Move32(LocationFrom(calling_convention.GetRegisterAt(0)), root_); 1444 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1445 instruction_, 1446 instruction_->GetDexPc(), 1447 this); 1448 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1449 arm_codegen->Move32(out_, LocationFrom(r0)); 1450 1451 RestoreLiveRegisters(codegen, locations); 1452 __ B(GetExitLabel()); 1453 } 1454 1455 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARMVIXL"; } 1456 1457 private: 1458 const Location out_; 1459 const Location root_; 1460 1461 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARMVIXL); 1462 }; 1463 1464 inline vixl32::Condition ARMCondition(IfCondition cond) { 1465 switch (cond) { 1466 case kCondEQ: return eq; 1467 case kCondNE: return ne; 1468 case kCondLT: return lt; 1469 case kCondLE: return le; 1470 case kCondGT: return gt; 1471 case kCondGE: return ge; 1472 case kCondB: return lo; 1473 case kCondBE: return ls; 1474 case kCondA: return hi; 1475 case kCondAE: return hs; 1476 } 1477 LOG(FATAL) << "Unreachable"; 1478 UNREACHABLE(); 1479 } 1480 1481 // Maps signed condition to unsigned condition. 1482 inline vixl32::Condition ARMUnsignedCondition(IfCondition cond) { 1483 switch (cond) { 1484 case kCondEQ: return eq; 1485 case kCondNE: return ne; 1486 // Signed to unsigned. 1487 case kCondLT: return lo; 1488 case kCondLE: return ls; 1489 case kCondGT: return hi; 1490 case kCondGE: return hs; 1491 // Unsigned remain unchanged. 1492 case kCondB: return lo; 1493 case kCondBE: return ls; 1494 case kCondA: return hi; 1495 case kCondAE: return hs; 1496 } 1497 LOG(FATAL) << "Unreachable"; 1498 UNREACHABLE(); 1499 } 1500 1501 inline vixl32::Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1502 // The ARM condition codes can express all the necessary branches, see the 1503 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1504 // There is no dex instruction or HIR that would need the missing conditions 1505 // "equal or unordered" or "not equal". 1506 switch (cond) { 1507 case kCondEQ: return eq; 1508 case kCondNE: return ne /* unordered */; 1509 case kCondLT: return gt_bias ? cc : lt /* unordered */; 1510 case kCondLE: return gt_bias ? ls : le /* unordered */; 1511 case kCondGT: return gt_bias ? hi /* unordered */ : gt; 1512 case kCondGE: return gt_bias ? cs /* unordered */ : ge; 1513 default: 1514 LOG(FATAL) << "UNREACHABLE"; 1515 UNREACHABLE(); 1516 } 1517 } 1518 1519 inline ShiftType ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { 1520 switch (op_kind) { 1521 case HDataProcWithShifterOp::kASR: return ShiftType::ASR; 1522 case HDataProcWithShifterOp::kLSL: return ShiftType::LSL; 1523 case HDataProcWithShifterOp::kLSR: return ShiftType::LSR; 1524 default: 1525 LOG(FATAL) << "Unexpected op kind " << op_kind; 1526 UNREACHABLE(); 1527 } 1528 } 1529 1530 void CodeGeneratorARMVIXL::DumpCoreRegister(std::ostream& stream, int reg) const { 1531 stream << vixl32::Register(reg); 1532 } 1533 1534 void CodeGeneratorARMVIXL::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1535 stream << vixl32::SRegister(reg); 1536 } 1537 1538 static uint32_t ComputeSRegisterListMask(const SRegisterList& regs) { 1539 uint32_t mask = 0; 1540 for (uint32_t i = regs.GetFirstSRegister().GetCode(); 1541 i <= regs.GetLastSRegister().GetCode(); 1542 ++i) { 1543 mask |= (1 << i); 1544 } 1545 return mask; 1546 } 1547 1548 // Saves the register in the stack. Returns the size taken on stack. 1549 size_t CodeGeneratorARMVIXL::SaveCoreRegister(size_t stack_index ATTRIBUTE_UNUSED, 1550 uint32_t reg_id ATTRIBUTE_UNUSED) { 1551 TODO_VIXL32(FATAL); 1552 return 0; 1553 } 1554 1555 // Restores the register from the stack. Returns the size taken on stack. 1556 size_t CodeGeneratorARMVIXL::RestoreCoreRegister(size_t stack_index ATTRIBUTE_UNUSED, 1557 uint32_t reg_id ATTRIBUTE_UNUSED) { 1558 TODO_VIXL32(FATAL); 1559 return 0; 1560 } 1561 1562 size_t CodeGeneratorARMVIXL::SaveFloatingPointRegister(size_t stack_index ATTRIBUTE_UNUSED, 1563 uint32_t reg_id ATTRIBUTE_UNUSED) { 1564 TODO_VIXL32(FATAL); 1565 return 0; 1566 } 1567 1568 size_t CodeGeneratorARMVIXL::RestoreFloatingPointRegister(size_t stack_index ATTRIBUTE_UNUSED, 1569 uint32_t reg_id ATTRIBUTE_UNUSED) { 1570 TODO_VIXL32(FATAL); 1571 return 0; 1572 } 1573 1574 static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, 1575 vixl32::Register out, 1576 vixl32::Register first, 1577 const Operand& second, 1578 CodeGeneratorARMVIXL* codegen) { 1579 if (second.IsImmediate() && second.GetImmediate() == 0) { 1580 const Operand in = kind == HInstruction::kAnd 1581 ? Operand(0) 1582 : Operand(first); 1583 1584 __ Mov(out, in); 1585 } else { 1586 switch (kind) { 1587 case HInstruction::kAdd: 1588 __ Add(out, first, second); 1589 break; 1590 case HInstruction::kAnd: 1591 __ And(out, first, second); 1592 break; 1593 case HInstruction::kOr: 1594 __ Orr(out, first, second); 1595 break; 1596 case HInstruction::kSub: 1597 __ Sub(out, first, second); 1598 break; 1599 case HInstruction::kXor: 1600 __ Eor(out, first, second); 1601 break; 1602 default: 1603 LOG(FATAL) << "Unexpected instruction kind: " << kind; 1604 UNREACHABLE(); 1605 } 1606 } 1607 } 1608 1609 static void GenerateDataProc(HInstruction::InstructionKind kind, 1610 const Location& out, 1611 const Location& first, 1612 const Operand& second_lo, 1613 const Operand& second_hi, 1614 CodeGeneratorARMVIXL* codegen) { 1615 const vixl32::Register first_hi = HighRegisterFrom(first); 1616 const vixl32::Register first_lo = LowRegisterFrom(first); 1617 const vixl32::Register out_hi = HighRegisterFrom(out); 1618 const vixl32::Register out_lo = LowRegisterFrom(out); 1619 1620 if (kind == HInstruction::kAdd) { 1621 __ Adds(out_lo, first_lo, second_lo); 1622 __ Adc(out_hi, first_hi, second_hi); 1623 } else if (kind == HInstruction::kSub) { 1624 __ Subs(out_lo, first_lo, second_lo); 1625 __ Sbc(out_hi, first_hi, second_hi); 1626 } else { 1627 GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); 1628 GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); 1629 } 1630 } 1631 1632 static Operand GetShifterOperand(vixl32::Register rm, ShiftType shift, uint32_t shift_imm) { 1633 return shift_imm == 0 ? Operand(rm) : Operand(rm, shift, shift_imm); 1634 } 1635 1636 static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, 1637 CodeGeneratorARMVIXL* codegen) { 1638 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 1639 DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); 1640 1641 const LocationSummary* const locations = instruction->GetLocations(); 1642 const uint32_t shift_value = instruction->GetShiftAmount(); 1643 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1644 const Location first = locations->InAt(0); 1645 const Location second = locations->InAt(1); 1646 const Location out = locations->Out(); 1647 const vixl32::Register first_hi = HighRegisterFrom(first); 1648 const vixl32::Register first_lo = LowRegisterFrom(first); 1649 const vixl32::Register out_hi = HighRegisterFrom(out); 1650 const vixl32::Register out_lo = LowRegisterFrom(out); 1651 const vixl32::Register second_hi = HighRegisterFrom(second); 1652 const vixl32::Register second_lo = LowRegisterFrom(second); 1653 const ShiftType shift = ShiftFromOpKind(instruction->GetOpKind()); 1654 1655 if (shift_value >= 32) { 1656 if (shift == ShiftType::LSL) { 1657 GenerateDataProcInstruction(kind, 1658 out_hi, 1659 first_hi, 1660 Operand(second_lo, ShiftType::LSL, shift_value - 32), 1661 codegen); 1662 GenerateDataProcInstruction(kind, out_lo, first_lo, 0, codegen); 1663 } else if (shift == ShiftType::ASR) { 1664 GenerateDataProc(kind, 1665 out, 1666 first, 1667 GetShifterOperand(second_hi, ShiftType::ASR, shift_value - 32), 1668 Operand(second_hi, ShiftType::ASR, 31), 1669 codegen); 1670 } else { 1671 DCHECK_EQ(shift, ShiftType::LSR); 1672 GenerateDataProc(kind, 1673 out, 1674 first, 1675 GetShifterOperand(second_hi, ShiftType::LSR, shift_value - 32), 1676 0, 1677 codegen); 1678 } 1679 } else { 1680 DCHECK_GT(shift_value, 1U); 1681 DCHECK_LT(shift_value, 32U); 1682 1683 UseScratchRegisterScope temps(codegen->GetVIXLAssembler()); 1684 1685 if (shift == ShiftType::LSL) { 1686 // We are not doing this for HInstruction::kAdd because the output will require 1687 // Location::kOutputOverlap; not applicable to other cases. 1688 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1689 GenerateDataProcInstruction(kind, 1690 out_hi, 1691 first_hi, 1692 Operand(second_hi, ShiftType::LSL, shift_value), 1693 codegen); 1694 GenerateDataProcInstruction(kind, 1695 out_hi, 1696 out_hi, 1697 Operand(second_lo, ShiftType::LSR, 32 - shift_value), 1698 codegen); 1699 GenerateDataProcInstruction(kind, 1700 out_lo, 1701 first_lo, 1702 Operand(second_lo, ShiftType::LSL, shift_value), 1703 codegen); 1704 } else { 1705 const vixl32::Register temp = temps.Acquire(); 1706 1707 __ Lsl(temp, second_hi, shift_value); 1708 __ Orr(temp, temp, Operand(second_lo, ShiftType::LSR, 32 - shift_value)); 1709 GenerateDataProc(kind, 1710 out, 1711 first, 1712 Operand(second_lo, ShiftType::LSL, shift_value), 1713 temp, 1714 codegen); 1715 } 1716 } else { 1717 DCHECK(shift == ShiftType::ASR || shift == ShiftType::LSR); 1718 1719 // We are not doing this for HInstruction::kAdd because the output will require 1720 // Location::kOutputOverlap; not applicable to other cases. 1721 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1722 GenerateDataProcInstruction(kind, 1723 out_lo, 1724 first_lo, 1725 Operand(second_lo, ShiftType::LSR, shift_value), 1726 codegen); 1727 GenerateDataProcInstruction(kind, 1728 out_lo, 1729 out_lo, 1730 Operand(second_hi, ShiftType::LSL, 32 - shift_value), 1731 codegen); 1732 GenerateDataProcInstruction(kind, 1733 out_hi, 1734 first_hi, 1735 Operand(second_hi, shift, shift_value), 1736 codegen); 1737 } else { 1738 const vixl32::Register temp = temps.Acquire(); 1739 1740 __ Lsr(temp, second_lo, shift_value); 1741 __ Orr(temp, temp, Operand(second_hi, ShiftType::LSL, 32 - shift_value)); 1742 GenerateDataProc(kind, 1743 out, 1744 first, 1745 temp, 1746 Operand(second_hi, shift, shift_value), 1747 codegen); 1748 } 1749 } 1750 } 1751 } 1752 1753 static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARMVIXL* codegen) { 1754 const Location rhs_loc = instruction->GetLocations()->InAt(1); 1755 if (rhs_loc.IsConstant()) { 1756 // 0.0 is the only immediate that can be encoded directly in 1757 // a VCMP instruction. 1758 // 1759 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1760 // specify that in a floating-point comparison, positive zero 1761 // and negative zero are considered equal, so we can use the 1762 // literal 0.0 for both cases here. 1763 // 1764 // Note however that some methods (Float.equal, Float.compare, 1765 // Float.compareTo, Double.equal, Double.compare, 1766 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1767 // StrictMath.min) consider 0.0 to be (strictly) greater than 1768 // -0.0. So if we ever translate calls to these methods into a 1769 // HCompare instruction, we must handle the -0.0 case with 1770 // care here. 1771 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1772 1773 const Primitive::Type type = instruction->InputAt(0)->GetType(); 1774 1775 if (type == Primitive::kPrimFloat) { 1776 __ Vcmp(F32, InputSRegisterAt(instruction, 0), 0.0); 1777 } else { 1778 DCHECK_EQ(type, Primitive::kPrimDouble); 1779 __ Vcmp(F64, InputDRegisterAt(instruction, 0), 0.0); 1780 } 1781 } else { 1782 __ Vcmp(InputVRegisterAt(instruction, 0), InputVRegisterAt(instruction, 1)); 1783 } 1784 } 1785 1786 static int64_t AdjustConstantForCondition(int64_t value, 1787 IfCondition* condition, 1788 IfCondition* opposite) { 1789 if (value == 1) { 1790 if (*condition == kCondB) { 1791 value = 0; 1792 *condition = kCondEQ; 1793 *opposite = kCondNE; 1794 } else if (*condition == kCondAE) { 1795 value = 0; 1796 *condition = kCondNE; 1797 *opposite = kCondEQ; 1798 } 1799 } else if (value == -1) { 1800 if (*condition == kCondGT) { 1801 value = 0; 1802 *condition = kCondGE; 1803 *opposite = kCondLT; 1804 } else if (*condition == kCondLE) { 1805 value = 0; 1806 *condition = kCondLT; 1807 *opposite = kCondGE; 1808 } 1809 } 1810 1811 return value; 1812 } 1813 1814 static std::pair<vixl32::Condition, vixl32::Condition> GenerateLongTestConstant( 1815 HCondition* condition, 1816 bool invert, 1817 CodeGeneratorARMVIXL* codegen) { 1818 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1819 1820 const LocationSummary* const locations = condition->GetLocations(); 1821 IfCondition cond = condition->GetCondition(); 1822 IfCondition opposite = condition->GetOppositeCondition(); 1823 1824 if (invert) { 1825 std::swap(cond, opposite); 1826 } 1827 1828 std::pair<vixl32::Condition, vixl32::Condition> ret(eq, ne); 1829 const Location left = locations->InAt(0); 1830 const Location right = locations->InAt(1); 1831 1832 DCHECK(right.IsConstant()); 1833 1834 const vixl32::Register left_high = HighRegisterFrom(left); 1835 const vixl32::Register left_low = LowRegisterFrom(left); 1836 int64_t value = AdjustConstantForCondition(Int64ConstantFrom(right), &cond, &opposite); 1837 UseScratchRegisterScope temps(codegen->GetVIXLAssembler()); 1838 1839 // Comparisons against 0 are common enough to deserve special attention. 1840 if (value == 0) { 1841 switch (cond) { 1842 case kCondNE: 1843 // x > 0 iff x != 0 when the comparison is unsigned. 1844 case kCondA: 1845 ret = std::make_pair(ne, eq); 1846 FALLTHROUGH_INTENDED; 1847 case kCondEQ: 1848 // x <= 0 iff x == 0 when the comparison is unsigned. 1849 case kCondBE: 1850 __ Orrs(temps.Acquire(), left_low, left_high); 1851 return ret; 1852 case kCondLT: 1853 case kCondGE: 1854 __ Cmp(left_high, 0); 1855 return std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 1856 // Trivially true or false. 1857 case kCondB: 1858 ret = std::make_pair(ne, eq); 1859 FALLTHROUGH_INTENDED; 1860 case kCondAE: 1861 __ Cmp(left_low, left_low); 1862 return ret; 1863 default: 1864 break; 1865 } 1866 } 1867 1868 switch (cond) { 1869 case kCondEQ: 1870 case kCondNE: 1871 case kCondB: 1872 case kCondBE: 1873 case kCondA: 1874 case kCondAE: { 1875 __ Cmp(left_high, High32Bits(value)); 1876 1877 // We use the scope because of the IT block that follows. 1878 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 1879 2 * vixl32::k16BitT32InstructionSizeInBytes, 1880 CodeBufferCheckScope::kExactSize); 1881 1882 __ it(eq); 1883 __ cmp(eq, left_low, Low32Bits(value)); 1884 ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); 1885 break; 1886 } 1887 case kCondLE: 1888 case kCondGT: 1889 // Trivially true or false. 1890 if (value == std::numeric_limits<int64_t>::max()) { 1891 __ Cmp(left_low, left_low); 1892 ret = cond == kCondLE ? std::make_pair(eq, ne) : std::make_pair(ne, eq); 1893 break; 1894 } 1895 1896 if (cond == kCondLE) { 1897 DCHECK_EQ(opposite, kCondGT); 1898 cond = kCondLT; 1899 opposite = kCondGE; 1900 } else { 1901 DCHECK_EQ(cond, kCondGT); 1902 DCHECK_EQ(opposite, kCondLE); 1903 cond = kCondGE; 1904 opposite = kCondLT; 1905 } 1906 1907 value++; 1908 FALLTHROUGH_INTENDED; 1909 case kCondGE: 1910 case kCondLT: { 1911 __ Cmp(left_low, Low32Bits(value)); 1912 __ Sbcs(temps.Acquire(), left_high, High32Bits(value)); 1913 ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 1914 break; 1915 } 1916 default: 1917 LOG(FATAL) << "Unreachable"; 1918 UNREACHABLE(); 1919 } 1920 1921 return ret; 1922 } 1923 1924 static std::pair<vixl32::Condition, vixl32::Condition> GenerateLongTest( 1925 HCondition* condition, 1926 bool invert, 1927 CodeGeneratorARMVIXL* codegen) { 1928 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1929 1930 const LocationSummary* const locations = condition->GetLocations(); 1931 IfCondition cond = condition->GetCondition(); 1932 IfCondition opposite = condition->GetOppositeCondition(); 1933 1934 if (invert) { 1935 std::swap(cond, opposite); 1936 } 1937 1938 std::pair<vixl32::Condition, vixl32::Condition> ret(eq, ne); 1939 Location left = locations->InAt(0); 1940 Location right = locations->InAt(1); 1941 1942 DCHECK(right.IsRegisterPair()); 1943 1944 switch (cond) { 1945 case kCondEQ: 1946 case kCondNE: 1947 case kCondB: 1948 case kCondBE: 1949 case kCondA: 1950 case kCondAE: { 1951 __ Cmp(HighRegisterFrom(left), HighRegisterFrom(right)); 1952 1953 // We use the scope because of the IT block that follows. 1954 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 1955 2 * vixl32::k16BitT32InstructionSizeInBytes, 1956 CodeBufferCheckScope::kExactSize); 1957 1958 __ it(eq); 1959 __ cmp(eq, LowRegisterFrom(left), LowRegisterFrom(right)); 1960 ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); 1961 break; 1962 } 1963 case kCondLE: 1964 case kCondGT: 1965 if (cond == kCondLE) { 1966 DCHECK_EQ(opposite, kCondGT); 1967 cond = kCondGE; 1968 opposite = kCondLT; 1969 } else { 1970 DCHECK_EQ(cond, kCondGT); 1971 DCHECK_EQ(opposite, kCondLE); 1972 cond = kCondLT; 1973 opposite = kCondGE; 1974 } 1975 1976 std::swap(left, right); 1977 FALLTHROUGH_INTENDED; 1978 case kCondGE: 1979 case kCondLT: { 1980 UseScratchRegisterScope temps(codegen->GetVIXLAssembler()); 1981 1982 __ Cmp(LowRegisterFrom(left), LowRegisterFrom(right)); 1983 __ Sbcs(temps.Acquire(), HighRegisterFrom(left), HighRegisterFrom(right)); 1984 ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 1985 break; 1986 } 1987 default: 1988 LOG(FATAL) << "Unreachable"; 1989 UNREACHABLE(); 1990 } 1991 1992 return ret; 1993 } 1994 1995 static std::pair<vixl32::Condition, vixl32::Condition> GenerateTest(HCondition* condition, 1996 bool invert, 1997 CodeGeneratorARMVIXL* codegen) { 1998 const Primitive::Type type = condition->GetLeft()->GetType(); 1999 IfCondition cond = condition->GetCondition(); 2000 IfCondition opposite = condition->GetOppositeCondition(); 2001 std::pair<vixl32::Condition, vixl32::Condition> ret(eq, ne); 2002 2003 if (invert) { 2004 std::swap(cond, opposite); 2005 } 2006 2007 if (type == Primitive::kPrimLong) { 2008 ret = condition->GetLocations()->InAt(1).IsConstant() 2009 ? GenerateLongTestConstant(condition, invert, codegen) 2010 : GenerateLongTest(condition, invert, codegen); 2011 } else if (Primitive::IsFloatingPointType(type)) { 2012 GenerateVcmp(condition, codegen); 2013 __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); 2014 ret = std::make_pair(ARMFPCondition(cond, condition->IsGtBias()), 2015 ARMFPCondition(opposite, condition->IsGtBias())); 2016 } else { 2017 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 2018 __ Cmp(InputRegisterAt(condition, 0), InputOperandAt(condition, 1)); 2019 ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 2020 } 2021 2022 return ret; 2023 } 2024 2025 static bool CanGenerateTest(HCondition* condition, ArmVIXLAssembler* assembler) { 2026 if (condition->GetLeft()->GetType() == Primitive::kPrimLong) { 2027 const LocationSummary* const locations = condition->GetLocations(); 2028 2029 if (locations->InAt(1).IsConstant()) { 2030 IfCondition c = condition->GetCondition(); 2031 IfCondition opposite = condition->GetOppositeCondition(); 2032 const int64_t value = 2033 AdjustConstantForCondition(Int64ConstantFrom(locations->InAt(1)), &c, &opposite); 2034 2035 if (c < kCondLT || c > kCondGE) { 2036 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 2037 // we check that the least significant half of the first input to be compared 2038 // is in a low register (the other half is read outside an IT block), and 2039 // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP 2040 // encoding can be used; 0 is always handled, no matter what registers are 2041 // used by the first input. 2042 if (value != 0 && 2043 (!LowRegisterFrom(locations->InAt(0)).IsLow() || !IsUint<8>(Low32Bits(value)))) { 2044 return false; 2045 } 2046 // TODO(VIXL): The rest of the checks are there to keep the backend in sync with 2047 // the previous one, but are not strictly necessary. 2048 } else if (c == kCondLE || c == kCondGT) { 2049 if (value < std::numeric_limits<int64_t>::max() && 2050 !assembler->ShifterOperandCanHold(SBC, High32Bits(value + 1), kCcSet)) { 2051 return false; 2052 } 2053 } else if (!assembler->ShifterOperandCanHold(SBC, High32Bits(value), kCcSet)) { 2054 return false; 2055 } 2056 } 2057 } 2058 2059 return true; 2060 } 2061 2062 static void GenerateConditionGeneric(HCondition* cond, CodeGeneratorARMVIXL* codegen) { 2063 DCHECK(CanGenerateTest(cond, codegen->GetAssembler())); 2064 2065 const vixl32::Register out = OutputRegister(cond); 2066 const auto condition = GenerateTest(cond, false, codegen); 2067 2068 __ Mov(LeaveFlags, out, 0); 2069 2070 if (out.IsLow()) { 2071 // We use the scope because of the IT block that follows. 2072 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 2073 2 * vixl32::k16BitT32InstructionSizeInBytes, 2074 CodeBufferCheckScope::kExactSize); 2075 2076 __ it(condition.first); 2077 __ mov(condition.first, out, 1); 2078 } else { 2079 vixl32::Label done_label; 2080 vixl32::Label* const final_label = codegen->GetFinalLabel(cond, &done_label); 2081 2082 __ B(condition.second, final_label, /* far_target */ false); 2083 __ Mov(out, 1); 2084 2085 if (done_label.IsReferenced()) { 2086 __ Bind(&done_label); 2087 } 2088 } 2089 } 2090 2091 static void GenerateEqualLong(HCondition* cond, CodeGeneratorARMVIXL* codegen) { 2092 DCHECK_EQ(cond->GetLeft()->GetType(), Primitive::kPrimLong); 2093 2094 const LocationSummary* const locations = cond->GetLocations(); 2095 IfCondition condition = cond->GetCondition(); 2096 const vixl32::Register out = OutputRegister(cond); 2097 const Location left = locations->InAt(0); 2098 const Location right = locations->InAt(1); 2099 vixl32::Register left_high = HighRegisterFrom(left); 2100 vixl32::Register left_low = LowRegisterFrom(left); 2101 vixl32::Register temp; 2102 UseScratchRegisterScope temps(codegen->GetVIXLAssembler()); 2103 2104 if (right.IsConstant()) { 2105 IfCondition opposite = cond->GetOppositeCondition(); 2106 const int64_t value = AdjustConstantForCondition(Int64ConstantFrom(right), 2107 &condition, 2108 &opposite); 2109 Operand right_high = High32Bits(value); 2110 Operand right_low = Low32Bits(value); 2111 2112 // The output uses Location::kNoOutputOverlap. 2113 if (out.Is(left_high)) { 2114 std::swap(left_low, left_high); 2115 std::swap(right_low, right_high); 2116 } 2117 2118 __ Sub(out, left_low, right_low); 2119 temp = temps.Acquire(); 2120 __ Sub(temp, left_high, right_high); 2121 } else { 2122 DCHECK(right.IsRegisterPair()); 2123 temp = temps.Acquire(); 2124 __ Sub(temp, left_high, HighRegisterFrom(right)); 2125 __ Sub(out, left_low, LowRegisterFrom(right)); 2126 } 2127 2128 // Need to check after calling AdjustConstantForCondition(). 2129 DCHECK(condition == kCondEQ || condition == kCondNE) << condition; 2130 2131 if (condition == kCondNE && out.IsLow()) { 2132 __ Orrs(out, out, temp); 2133 2134 // We use the scope because of the IT block that follows. 2135 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 2136 2 * vixl32::k16BitT32InstructionSizeInBytes, 2137 CodeBufferCheckScope::kExactSize); 2138 2139 __ it(ne); 2140 __ mov(ne, out, 1); 2141 } else { 2142 __ Orr(out, out, temp); 2143 codegen->GenerateConditionWithZero(condition, out, out, temp); 2144 } 2145 } 2146 2147 static void GenerateLongComparesAndJumps(HCondition* cond, 2148 vixl32::Label* true_label, 2149 vixl32::Label* false_label, 2150 CodeGeneratorARMVIXL* codegen, 2151 bool is_far_target = true) { 2152 LocationSummary* locations = cond->GetLocations(); 2153 Location left = locations->InAt(0); 2154 Location right = locations->InAt(1); 2155 IfCondition if_cond = cond->GetCondition(); 2156 2157 vixl32::Register left_high = HighRegisterFrom(left); 2158 vixl32::Register left_low = LowRegisterFrom(left); 2159 IfCondition true_high_cond = if_cond; 2160 IfCondition false_high_cond = cond->GetOppositeCondition(); 2161 vixl32::Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 2162 2163 // Set the conditions for the test, remembering that == needs to be 2164 // decided using the low words. 2165 switch (if_cond) { 2166 case kCondEQ: 2167 case kCondNE: 2168 // Nothing to do. 2169 break; 2170 case kCondLT: 2171 false_high_cond = kCondGT; 2172 break; 2173 case kCondLE: 2174 true_high_cond = kCondLT; 2175 break; 2176 case kCondGT: 2177 false_high_cond = kCondLT; 2178 break; 2179 case kCondGE: 2180 true_high_cond = kCondGT; 2181 break; 2182 case kCondB: 2183 false_high_cond = kCondA; 2184 break; 2185 case kCondBE: 2186 true_high_cond = kCondB; 2187 break; 2188 case kCondA: 2189 false_high_cond = kCondB; 2190 break; 2191 case kCondAE: 2192 true_high_cond = kCondA; 2193 break; 2194 } 2195 if (right.IsConstant()) { 2196 int64_t value = Int64ConstantFrom(right); 2197 int32_t val_low = Low32Bits(value); 2198 int32_t val_high = High32Bits(value); 2199 2200 __ Cmp(left_high, val_high); 2201 if (if_cond == kCondNE) { 2202 __ B(ARMCondition(true_high_cond), true_label, is_far_target); 2203 } else if (if_cond == kCondEQ) { 2204 __ B(ARMCondition(false_high_cond), false_label, is_far_target); 2205 } else { 2206 __ B(ARMCondition(true_high_cond), true_label, is_far_target); 2207 __ B(ARMCondition(false_high_cond), false_label, is_far_target); 2208 } 2209 // Must be equal high, so compare the lows. 2210 __ Cmp(left_low, val_low); 2211 } else { 2212 vixl32::Register right_high = HighRegisterFrom(right); 2213 vixl32::Register right_low = LowRegisterFrom(right); 2214 2215 __ Cmp(left_high, right_high); 2216 if (if_cond == kCondNE) { 2217 __ B(ARMCondition(true_high_cond), true_label, is_far_target); 2218 } else if (if_cond == kCondEQ) { 2219 __ B(ARMCondition(false_high_cond), false_label, is_far_target); 2220 } else { 2221 __ B(ARMCondition(true_high_cond), true_label, is_far_target); 2222 __ B(ARMCondition(false_high_cond), false_label, is_far_target); 2223 } 2224 // Must be equal high, so compare the lows. 2225 __ Cmp(left_low, right_low); 2226 } 2227 // The last comparison might be unsigned. 2228 // TODO: optimize cases where this is always true/false 2229 __ B(final_condition, true_label, is_far_target); 2230 } 2231 2232 static void GenerateConditionLong(HCondition* cond, CodeGeneratorARMVIXL* codegen) { 2233 DCHECK_EQ(cond->GetLeft()->GetType(), Primitive::kPrimLong); 2234 2235 const LocationSummary* const locations = cond->GetLocations(); 2236 IfCondition condition = cond->GetCondition(); 2237 const vixl32::Register out = OutputRegister(cond); 2238 const Location left = locations->InAt(0); 2239 const Location right = locations->InAt(1); 2240 2241 if (right.IsConstant()) { 2242 IfCondition opposite = cond->GetOppositeCondition(); 2243 2244 // Comparisons against 0 are common enough to deserve special attention. 2245 if (AdjustConstantForCondition(Int64ConstantFrom(right), &condition, &opposite) == 0) { 2246 switch (condition) { 2247 case kCondNE: 2248 case kCondA: 2249 if (out.IsLow()) { 2250 // We only care if both input registers are 0 or not. 2251 __ Orrs(out, LowRegisterFrom(left), HighRegisterFrom(left)); 2252 2253 // We use the scope because of the IT block that follows. 2254 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 2255 2 * vixl32::k16BitT32InstructionSizeInBytes, 2256 CodeBufferCheckScope::kExactSize); 2257 2258 __ it(ne); 2259 __ mov(ne, out, 1); 2260 return; 2261 } 2262 2263 FALLTHROUGH_INTENDED; 2264 case kCondEQ: 2265 case kCondBE: 2266 // We only care if both input registers are 0 or not. 2267 __ Orr(out, LowRegisterFrom(left), HighRegisterFrom(left)); 2268 codegen->GenerateConditionWithZero(condition, out, out); 2269 return; 2270 case kCondLT: 2271 case kCondGE: 2272 // We only care about the sign bit. 2273 FALLTHROUGH_INTENDED; 2274 case kCondAE: 2275 case kCondB: 2276 codegen->GenerateConditionWithZero(condition, out, HighRegisterFrom(left)); 2277 return; 2278 case kCondLE: 2279 case kCondGT: 2280 default: 2281 break; 2282 } 2283 } 2284 } 2285 2286 if ((condition == kCondEQ || condition == kCondNE) && 2287 // If `out` is a low register, then the GenerateConditionGeneric() 2288 // function generates a shorter code sequence that is still branchless. 2289 (!out.IsLow() || !CanGenerateTest(cond, codegen->GetAssembler()))) { 2290 GenerateEqualLong(cond, codegen); 2291 return; 2292 } 2293 2294 if (CanGenerateTest(cond, codegen->GetAssembler())) { 2295 GenerateConditionGeneric(cond, codegen); 2296 return; 2297 } 2298 2299 // Convert the jumps into the result. 2300 vixl32::Label done_label; 2301 vixl32::Label* const final_label = codegen->GetFinalLabel(cond, &done_label); 2302 vixl32::Label true_label, false_label; 2303 2304 GenerateLongComparesAndJumps(cond, &true_label, &false_label, codegen, /* is_far_target */ false); 2305 2306 // False case: result = 0. 2307 __ Bind(&false_label); 2308 __ Mov(out, 0); 2309 __ B(final_label); 2310 2311 // True case: result = 1. 2312 __ Bind(&true_label); 2313 __ Mov(out, 1); 2314 2315 if (done_label.IsReferenced()) { 2316 __ Bind(&done_label); 2317 } 2318 } 2319 2320 static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, 2321 CodeGeneratorARMVIXL* codegen) { 2322 const Primitive::Type type = cond->GetLeft()->GetType(); 2323 2324 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 2325 2326 if (type == Primitive::kPrimLong) { 2327 GenerateConditionLong(cond, codegen); 2328 return; 2329 } 2330 2331 IfCondition condition = cond->GetCondition(); 2332 vixl32::Register in = InputRegisterAt(cond, 0); 2333 const vixl32::Register out = OutputRegister(cond); 2334 const Location right = cond->GetLocations()->InAt(1); 2335 int64_t value; 2336 2337 if (right.IsConstant()) { 2338 IfCondition opposite = cond->GetOppositeCondition(); 2339 2340 value = AdjustConstantForCondition(Int64ConstantFrom(right), &condition, &opposite); 2341 2342 // Comparisons against 0 are common enough to deserve special attention. 2343 if (value == 0) { 2344 switch (condition) { 2345 case kCondNE: 2346 case kCondA: 2347 if (out.IsLow() && out.Is(in)) { 2348 __ Cmp(out, 0); 2349 2350 // We use the scope because of the IT block that follows. 2351 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 2352 2 * vixl32::k16BitT32InstructionSizeInBytes, 2353 CodeBufferCheckScope::kExactSize); 2354 2355 __ it(ne); 2356 __ mov(ne, out, 1); 2357 return; 2358 } 2359 2360 FALLTHROUGH_INTENDED; 2361 case kCondEQ: 2362 case kCondBE: 2363 case kCondLT: 2364 case kCondGE: 2365 case kCondAE: 2366 case kCondB: 2367 codegen->GenerateConditionWithZero(condition, out, in); 2368 return; 2369 case kCondLE: 2370 case kCondGT: 2371 default: 2372 break; 2373 } 2374 } 2375 } 2376 2377 if (condition == kCondEQ || condition == kCondNE) { 2378 Operand operand(0); 2379 2380 if (right.IsConstant()) { 2381 operand = Operand::From(value); 2382 } else if (out.Is(RegisterFrom(right))) { 2383 // Avoid 32-bit instructions if possible. 2384 operand = InputOperandAt(cond, 0); 2385 in = RegisterFrom(right); 2386 } else { 2387 operand = InputOperandAt(cond, 1); 2388 } 2389 2390 if (condition == kCondNE && out.IsLow()) { 2391 __ Subs(out, in, operand); 2392 2393 // We use the scope because of the IT block that follows. 2394 ExactAssemblyScope guard(codegen->GetVIXLAssembler(), 2395 2 * vixl32::k16BitT32InstructionSizeInBytes, 2396 CodeBufferCheckScope::kExactSize); 2397 2398 __ it(ne); 2399 __ mov(ne, out, 1); 2400 } else { 2401 __ Sub(out, in, operand); 2402 codegen->GenerateConditionWithZero(condition, out, out); 2403 } 2404 2405 return; 2406 } 2407 2408 GenerateConditionGeneric(cond, codegen); 2409 } 2410 2411 static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) { 2412 const Primitive::Type type = constant->GetType(); 2413 bool ret = false; 2414 2415 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 2416 2417 if (type == Primitive::kPrimLong) { 2418 const uint64_t value = Uint64ConstantFrom(constant); 2419 2420 ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value)); 2421 } else { 2422 ret = IsUint<8>(Int32ConstantFrom(constant)); 2423 } 2424 2425 return ret; 2426 } 2427 2428 static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) { 2429 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 2430 2431 if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) { 2432 return Location::ConstantLocation(constant->AsConstant()); 2433 } 2434 2435 return Location::RequiresRegister(); 2436 } 2437 2438 static bool CanGenerateConditionalMove(const Location& out, const Location& src) { 2439 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 2440 // we check that we are not dealing with floating-point output (there is no 2441 // 16-bit VMOV encoding). 2442 if (!out.IsRegister() && !out.IsRegisterPair()) { 2443 return false; 2444 } 2445 2446 // For constants, we also check that the output is in one or two low registers, 2447 // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit 2448 // MOV encoding can be used. 2449 if (src.IsConstant()) { 2450 if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) { 2451 return false; 2452 } 2453 2454 if (out.IsRegister()) { 2455 if (!RegisterFrom(out).IsLow()) { 2456 return false; 2457 } 2458 } else { 2459 DCHECK(out.IsRegisterPair()); 2460 2461 if (!HighRegisterFrom(out).IsLow()) { 2462 return false; 2463 } 2464 } 2465 } 2466 2467 return true; 2468 } 2469 2470 #undef __ 2471 2472 vixl32::Label* CodeGeneratorARMVIXL::GetFinalLabel(HInstruction* instruction, 2473 vixl32::Label* final_label) { 2474 DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck()); 2475 DCHECK(!instruction->IsInvoke() || !instruction->GetLocations()->CanCall()); 2476 2477 const HBasicBlock* const block = instruction->GetBlock(); 2478 const HLoopInformation* const info = block->GetLoopInformation(); 2479 HInstruction* const next = instruction->GetNext(); 2480 2481 // Avoid a branch to a branch. 2482 if (next->IsGoto() && (info == nullptr || 2483 !info->IsBackEdge(*block) || 2484 !info->HasSuspendCheck())) { 2485 final_label = GetLabelOf(next->AsGoto()->GetSuccessor()); 2486 } 2487 2488 return final_label; 2489 } 2490 2491 CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph, 2492 const ArmInstructionSetFeatures& isa_features, 2493 const CompilerOptions& compiler_options, 2494 OptimizingCompilerStats* stats) 2495 : CodeGenerator(graph, 2496 kNumberOfCoreRegisters, 2497 kNumberOfSRegisters, 2498 kNumberOfRegisterPairs, 2499 kCoreCalleeSaves.GetList(), 2500 ComputeSRegisterListMask(kFpuCalleeSaves), 2501 compiler_options, 2502 stats), 2503 block_labels_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2504 jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2505 location_builder_(graph, this), 2506 instruction_visitor_(graph, this), 2507 move_resolver_(graph->GetArena(), this), 2508 assembler_(graph->GetArena()), 2509 isa_features_(isa_features), 2510 uint32_literals_(std::less<uint32_t>(), 2511 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2512 pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2513 method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2514 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2515 type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2516 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2517 baker_read_barrier_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2518 jit_string_patches_(StringReferenceValueComparator(), 2519 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 2520 jit_class_patches_(TypeReferenceValueComparator(), 2521 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 2522 // Always save the LR register to mimic Quick. 2523 AddAllocatedRegister(Location::RegisterLocation(LR)); 2524 // Give D30 and D31 as scratch register to VIXL. The register allocator only works on 2525 // S0-S31, which alias to D0-D15. 2526 GetVIXLAssembler()->GetScratchVRegisterList()->Combine(d31); 2527 GetVIXLAssembler()->GetScratchVRegisterList()->Combine(d30); 2528 } 2529 2530 void JumpTableARMVIXL::EmitTable(CodeGeneratorARMVIXL* codegen) { 2531 uint32_t num_entries = switch_instr_->GetNumEntries(); 2532 DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold); 2533 2534 // We are about to use the assembler to place literals directly. Make sure we have enough 2535 // underlying code buffer and we have generated a jump table of the right size, using 2536 // codegen->GetVIXLAssembler()->GetBuffer().Align(); 2537 ExactAssemblyScope aas(codegen->GetVIXLAssembler(), 2538 num_entries * sizeof(int32_t), 2539 CodeBufferCheckScope::kMaximumSize); 2540 // TODO(VIXL): Check that using lower case bind is fine here. 2541 codegen->GetVIXLAssembler()->bind(&table_start_); 2542 for (uint32_t i = 0; i < num_entries; i++) { 2543 codegen->GetVIXLAssembler()->place(bb_addresses_[i].get()); 2544 } 2545 } 2546 2547 void JumpTableARMVIXL::FixTable(CodeGeneratorARMVIXL* codegen) { 2548 uint32_t num_entries = switch_instr_->GetNumEntries(); 2549 DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold); 2550 2551 const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors(); 2552 for (uint32_t i = 0; i < num_entries; i++) { 2553 vixl32::Label* target_label = codegen->GetLabelOf(successors[i]); 2554 DCHECK(target_label->IsBound()); 2555 int32_t jump_offset = target_label->GetLocation() - table_start_.GetLocation(); 2556 // When doing BX to address we need to have lower bit set to 1 in T32. 2557 if (codegen->GetVIXLAssembler()->IsUsingT32()) { 2558 jump_offset++; 2559 } 2560 DCHECK_GT(jump_offset, std::numeric_limits<int32_t>::min()); 2561 DCHECK_LE(jump_offset, std::numeric_limits<int32_t>::max()); 2562 2563 bb_addresses_[i].get()->UpdateValue(jump_offset, codegen->GetVIXLAssembler()->GetBuffer()); 2564 } 2565 } 2566 2567 void CodeGeneratorARMVIXL::FixJumpTables() { 2568 for (auto&& jump_table : jump_tables_) { 2569 jump_table->FixTable(this); 2570 } 2571 } 2572 2573 #define __ reinterpret_cast<ArmVIXLAssembler*>(GetAssembler())->GetVIXLAssembler()-> // NOLINT 2574 2575 void CodeGeneratorARMVIXL::Finalize(CodeAllocator* allocator) { 2576 FixJumpTables(); 2577 GetAssembler()->FinalizeCode(); 2578 CodeGenerator::Finalize(allocator); 2579 } 2580 2581 void CodeGeneratorARMVIXL::SetupBlockedRegisters() const { 2582 // Stack register, LR and PC are always reserved. 2583 blocked_core_registers_[SP] = true; 2584 blocked_core_registers_[LR] = true; 2585 blocked_core_registers_[PC] = true; 2586 2587 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 2588 // Reserve marking register. 2589 blocked_core_registers_[MR] = true; 2590 } 2591 2592 // Reserve thread register. 2593 blocked_core_registers_[TR] = true; 2594 2595 // Reserve temp register. 2596 blocked_core_registers_[IP] = true; 2597 2598 if (GetGraph()->IsDebuggable()) { 2599 // Stubs do not save callee-save floating point registers. If the graph 2600 // is debuggable, we need to deal with these registers differently. For 2601 // now, just block them. 2602 for (uint32_t i = kFpuCalleeSaves.GetFirstSRegister().GetCode(); 2603 i <= kFpuCalleeSaves.GetLastSRegister().GetCode(); 2604 ++i) { 2605 blocked_fpu_registers_[i] = true; 2606 } 2607 } 2608 } 2609 2610 InstructionCodeGeneratorARMVIXL::InstructionCodeGeneratorARMVIXL(HGraph* graph, 2611 CodeGeneratorARMVIXL* codegen) 2612 : InstructionCodeGenerator(graph, codegen), 2613 assembler_(codegen->GetAssembler()), 2614 codegen_(codegen) {} 2615 2616 void CodeGeneratorARMVIXL::ComputeSpillMask() { 2617 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 2618 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 2619 // There is no easy instruction to restore just the PC on thumb2. We spill and 2620 // restore another arbitrary register. 2621 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister.GetCode()); 2622 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 2623 // We use vpush and vpop for saving and restoring floating point registers, which take 2624 // a SRegister and the number of registers to save/restore after that SRegister. We 2625 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 2626 // but in the range. 2627 if (fpu_spill_mask_ != 0) { 2628 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 2629 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 2630 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 2631 fpu_spill_mask_ |= (1 << i); 2632 } 2633 } 2634 } 2635 2636 void CodeGeneratorARMVIXL::GenerateFrameEntry() { 2637 bool skip_overflow_check = 2638 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 2639 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 2640 __ Bind(&frame_entry_label_); 2641 2642 if (HasEmptyFrame()) { 2643 return; 2644 } 2645 2646 if (!skip_overflow_check) { 2647 UseScratchRegisterScope temps(GetVIXLAssembler()); 2648 vixl32::Register temp = temps.Acquire(); 2649 __ Sub(temp, sp, Operand::From(GetStackOverflowReservedBytes(kArm))); 2650 // The load must immediately precede RecordPcInfo. 2651 ExactAssemblyScope aas(GetVIXLAssembler(), 2652 vixl32::kMaxInstructionSizeInBytes, 2653 CodeBufferCheckScope::kMaximumSize); 2654 __ ldr(temp, MemOperand(temp)); 2655 RecordPcInfo(nullptr, 0); 2656 } 2657 2658 __ Push(RegisterList(core_spill_mask_)); 2659 GetAssembler()->cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 2660 GetAssembler()->cfi().RelOffsetForMany(DWARFReg(kMethodRegister), 2661 0, 2662 core_spill_mask_, 2663 kArmWordSize); 2664 if (fpu_spill_mask_ != 0) { 2665 uint32_t first = LeastSignificantBit(fpu_spill_mask_); 2666 2667 // Check that list is contiguous. 2668 DCHECK_EQ(fpu_spill_mask_ >> CTZ(fpu_spill_mask_), ~0u >> (32 - POPCOUNT(fpu_spill_mask_))); 2669 2670 __ Vpush(SRegisterList(vixl32::SRegister(first), POPCOUNT(fpu_spill_mask_))); 2671 GetAssembler()->cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 2672 GetAssembler()->cfi().RelOffsetForMany(DWARFReg(s0), 0, fpu_spill_mask_, kArmWordSize); 2673 } 2674 2675 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2676 __ Sub(sp, sp, adjust); 2677 GetAssembler()->cfi().AdjustCFAOffset(adjust); 2678 2679 // Save the current method if we need it. Note that we do not 2680 // do this in HCurrentMethod, as the instruction might have been removed 2681 // in the SSA graph. 2682 if (RequiresCurrentMethod()) { 2683 GetAssembler()->StoreToOffset(kStoreWord, kMethodRegister, sp, 0); 2684 } 2685 2686 if (GetGraph()->HasShouldDeoptimizeFlag()) { 2687 UseScratchRegisterScope temps(GetVIXLAssembler()); 2688 vixl32::Register temp = temps.Acquire(); 2689 // Initialize should_deoptimize flag to 0. 2690 __ Mov(temp, 0); 2691 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, GetStackOffsetOfShouldDeoptimizeFlag()); 2692 } 2693 } 2694 2695 void CodeGeneratorARMVIXL::GenerateFrameExit() { 2696 if (HasEmptyFrame()) { 2697 __ Bx(lr); 2698 return; 2699 } 2700 GetAssembler()->cfi().RememberState(); 2701 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2702 __ Add(sp, sp, adjust); 2703 GetAssembler()->cfi().AdjustCFAOffset(-adjust); 2704 if (fpu_spill_mask_ != 0) { 2705 uint32_t first = LeastSignificantBit(fpu_spill_mask_); 2706 2707 // Check that list is contiguous. 2708 DCHECK_EQ(fpu_spill_mask_ >> CTZ(fpu_spill_mask_), ~0u >> (32 - POPCOUNT(fpu_spill_mask_))); 2709 2710 __ Vpop(SRegisterList(vixl32::SRegister(first), POPCOUNT(fpu_spill_mask_))); 2711 GetAssembler()->cfi().AdjustCFAOffset( 2712 -static_cast<int>(kArmWordSize) * POPCOUNT(fpu_spill_mask_)); 2713 GetAssembler()->cfi().RestoreMany(DWARFReg(vixl32::SRegister(0)), fpu_spill_mask_); 2714 } 2715 // Pop LR into PC to return. 2716 DCHECK_NE(core_spill_mask_ & (1 << kLrCode), 0U); 2717 uint32_t pop_mask = (core_spill_mask_ & (~(1 << kLrCode))) | 1 << kPcCode; 2718 __ Pop(RegisterList(pop_mask)); 2719 GetAssembler()->cfi().RestoreState(); 2720 GetAssembler()->cfi().DefCFAOffset(GetFrameSize()); 2721 } 2722 2723 void CodeGeneratorARMVIXL::Bind(HBasicBlock* block) { 2724 __ Bind(GetLabelOf(block)); 2725 } 2726 2727 Location InvokeDexCallingConventionVisitorARMVIXL::GetNextLocation(Primitive::Type type) { 2728 switch (type) { 2729 case Primitive::kPrimBoolean: 2730 case Primitive::kPrimByte: 2731 case Primitive::kPrimChar: 2732 case Primitive::kPrimShort: 2733 case Primitive::kPrimInt: 2734 case Primitive::kPrimNot: { 2735 uint32_t index = gp_index_++; 2736 uint32_t stack_index = stack_index_++; 2737 if (index < calling_convention.GetNumberOfRegisters()) { 2738 return LocationFrom(calling_convention.GetRegisterAt(index)); 2739 } else { 2740 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2741 } 2742 } 2743 2744 case Primitive::kPrimLong: { 2745 uint32_t index = gp_index_; 2746 uint32_t stack_index = stack_index_; 2747 gp_index_ += 2; 2748 stack_index_ += 2; 2749 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2750 if (calling_convention.GetRegisterAt(index).Is(r1)) { 2751 // Skip R1, and use R2_R3 instead. 2752 gp_index_++; 2753 index++; 2754 } 2755 } 2756 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2757 DCHECK_EQ(calling_convention.GetRegisterAt(index).GetCode() + 1, 2758 calling_convention.GetRegisterAt(index + 1).GetCode()); 2759 2760 return LocationFrom(calling_convention.GetRegisterAt(index), 2761 calling_convention.GetRegisterAt(index + 1)); 2762 } else { 2763 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2764 } 2765 } 2766 2767 case Primitive::kPrimFloat: { 2768 uint32_t stack_index = stack_index_++; 2769 if (float_index_ % 2 == 0) { 2770 float_index_ = std::max(double_index_, float_index_); 2771 } 2772 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 2773 return LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++)); 2774 } else { 2775 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2776 } 2777 } 2778 2779 case Primitive::kPrimDouble: { 2780 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 2781 uint32_t stack_index = stack_index_; 2782 stack_index_ += 2; 2783 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 2784 uint32_t index = double_index_; 2785 double_index_ += 2; 2786 Location result = LocationFrom( 2787 calling_convention.GetFpuRegisterAt(index), 2788 calling_convention.GetFpuRegisterAt(index + 1)); 2789 DCHECK(ExpectedPairLayout(result)); 2790 return result; 2791 } else { 2792 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2793 } 2794 } 2795 2796 case Primitive::kPrimVoid: 2797 LOG(FATAL) << "Unexpected parameter type " << type; 2798 break; 2799 } 2800 return Location::NoLocation(); 2801 } 2802 2803 Location InvokeDexCallingConventionVisitorARMVIXL::GetReturnLocation(Primitive::Type type) const { 2804 switch (type) { 2805 case Primitive::kPrimBoolean: 2806 case Primitive::kPrimByte: 2807 case Primitive::kPrimChar: 2808 case Primitive::kPrimShort: 2809 case Primitive::kPrimInt: 2810 case Primitive::kPrimNot: { 2811 return LocationFrom(r0); 2812 } 2813 2814 case Primitive::kPrimFloat: { 2815 return LocationFrom(s0); 2816 } 2817 2818 case Primitive::kPrimLong: { 2819 return LocationFrom(r0, r1); 2820 } 2821 2822 case Primitive::kPrimDouble: { 2823 return LocationFrom(s0, s1); 2824 } 2825 2826 case Primitive::kPrimVoid: 2827 return Location::NoLocation(); 2828 } 2829 2830 UNREACHABLE(); 2831 } 2832 2833 Location InvokeDexCallingConventionVisitorARMVIXL::GetMethodLocation() const { 2834 return LocationFrom(kMethodRegister); 2835 } 2836 2837 void CodeGeneratorARMVIXL::Move32(Location destination, Location source) { 2838 if (source.Equals(destination)) { 2839 return; 2840 } 2841 if (destination.IsRegister()) { 2842 if (source.IsRegister()) { 2843 __ Mov(RegisterFrom(destination), RegisterFrom(source)); 2844 } else if (source.IsFpuRegister()) { 2845 __ Vmov(RegisterFrom(destination), SRegisterFrom(source)); 2846 } else { 2847 GetAssembler()->LoadFromOffset(kLoadWord, 2848 RegisterFrom(destination), 2849 sp, 2850 source.GetStackIndex()); 2851 } 2852 } else if (destination.IsFpuRegister()) { 2853 if (source.IsRegister()) { 2854 __ Vmov(SRegisterFrom(destination), RegisterFrom(source)); 2855 } else if (source.IsFpuRegister()) { 2856 __ Vmov(SRegisterFrom(destination), SRegisterFrom(source)); 2857 } else { 2858 GetAssembler()->LoadSFromOffset(SRegisterFrom(destination), sp, source.GetStackIndex()); 2859 } 2860 } else { 2861 DCHECK(destination.IsStackSlot()) << destination; 2862 if (source.IsRegister()) { 2863 GetAssembler()->StoreToOffset(kStoreWord, 2864 RegisterFrom(source), 2865 sp, 2866 destination.GetStackIndex()); 2867 } else if (source.IsFpuRegister()) { 2868 GetAssembler()->StoreSToOffset(SRegisterFrom(source), sp, destination.GetStackIndex()); 2869 } else { 2870 DCHECK(source.IsStackSlot()) << source; 2871 UseScratchRegisterScope temps(GetVIXLAssembler()); 2872 vixl32::Register temp = temps.Acquire(); 2873 GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, source.GetStackIndex()); 2874 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); 2875 } 2876 } 2877 } 2878 2879 void CodeGeneratorARMVIXL::MoveConstant(Location location, int32_t value) { 2880 DCHECK(location.IsRegister()); 2881 __ Mov(RegisterFrom(location), value); 2882 } 2883 2884 void CodeGeneratorARMVIXL::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 2885 // TODO(VIXL): Maybe refactor to have the 'move' implementation here and use it in 2886 // `ParallelMoveResolverARMVIXL::EmitMove`, as is done in the `arm64` backend. 2887 HParallelMove move(GetGraph()->GetArena()); 2888 move.AddMove(src, dst, dst_type, nullptr); 2889 GetMoveResolver()->EmitNativeCode(&move); 2890 } 2891 2892 void CodeGeneratorARMVIXL::AddLocationAsTemp(Location location, LocationSummary* locations) { 2893 if (location.IsRegister()) { 2894 locations->AddTemp(location); 2895 } else if (location.IsRegisterPair()) { 2896 locations->AddTemp(LocationFrom(LowRegisterFrom(location))); 2897 locations->AddTemp(LocationFrom(HighRegisterFrom(location))); 2898 } else { 2899 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 2900 } 2901 } 2902 2903 void CodeGeneratorARMVIXL::InvokeRuntime(QuickEntrypointEnum entrypoint, 2904 HInstruction* instruction, 2905 uint32_t dex_pc, 2906 SlowPathCode* slow_path) { 2907 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 2908 __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value())); 2909 // Ensure the pc position is recorded immediately after the `blx` instruction. 2910 // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. 2911 ExactAssemblyScope aas(GetVIXLAssembler(), 2912 vixl32::k16BitT32InstructionSizeInBytes, 2913 CodeBufferCheckScope::kExactSize); 2914 __ blx(lr); 2915 if (EntrypointRequiresStackMap(entrypoint)) { 2916 RecordPcInfo(instruction, dex_pc, slow_path); 2917 } 2918 } 2919 2920 void CodeGeneratorARMVIXL::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 2921 HInstruction* instruction, 2922 SlowPathCode* slow_path) { 2923 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 2924 __ Ldr(lr, MemOperand(tr, entry_point_offset)); 2925 __ Blx(lr); 2926 } 2927 2928 void InstructionCodeGeneratorARMVIXL::HandleGoto(HInstruction* got, HBasicBlock* successor) { 2929 DCHECK(!successor->IsExitBlock()); 2930 HBasicBlock* block = got->GetBlock(); 2931 HInstruction* previous = got->GetPrevious(); 2932 HLoopInformation* info = block->GetLoopInformation(); 2933 2934 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 2935 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 2936 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 2937 return; 2938 } 2939 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 2940 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 2941 } 2942 if (!codegen_->GoesToNextBlock(block, successor)) { 2943 __ B(codegen_->GetLabelOf(successor)); 2944 } 2945 } 2946 2947 void LocationsBuilderARMVIXL::VisitGoto(HGoto* got) { 2948 got->SetLocations(nullptr); 2949 } 2950 2951 void InstructionCodeGeneratorARMVIXL::VisitGoto(HGoto* got) { 2952 HandleGoto(got, got->GetSuccessor()); 2953 } 2954 2955 void LocationsBuilderARMVIXL::VisitTryBoundary(HTryBoundary* try_boundary) { 2956 try_boundary->SetLocations(nullptr); 2957 } 2958 2959 void InstructionCodeGeneratorARMVIXL::VisitTryBoundary(HTryBoundary* try_boundary) { 2960 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 2961 if (!successor->IsExitBlock()) { 2962 HandleGoto(try_boundary, successor); 2963 } 2964 } 2965 2966 void LocationsBuilderARMVIXL::VisitExit(HExit* exit) { 2967 exit->SetLocations(nullptr); 2968 } 2969 2970 void InstructionCodeGeneratorARMVIXL::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 2971 } 2972 2973 void InstructionCodeGeneratorARMVIXL::GenerateCompareTestAndBranch(HCondition* condition, 2974 vixl32::Label* true_target_in, 2975 vixl32::Label* false_target_in, 2976 bool is_far_target) { 2977 if (CanGenerateTest(condition, codegen_->GetAssembler())) { 2978 vixl32::Label* non_fallthrough_target; 2979 bool invert; 2980 bool emit_both_branches; 2981 2982 if (true_target_in == nullptr) { 2983 // The true target is fallthrough. 2984 DCHECK(false_target_in != nullptr); 2985 non_fallthrough_target = false_target_in; 2986 invert = true; 2987 emit_both_branches = false; 2988 } else { 2989 non_fallthrough_target = true_target_in; 2990 invert = false; 2991 // Either the false target is fallthrough, or there is no fallthrough 2992 // and both branches must be emitted. 2993 emit_both_branches = (false_target_in != nullptr); 2994 } 2995 2996 const auto cond = GenerateTest(condition, invert, codegen_); 2997 2998 __ B(cond.first, non_fallthrough_target, is_far_target); 2999 3000 if (emit_both_branches) { 3001 // No target falls through, we need to branch. 3002 __ B(false_target_in); 3003 } 3004 3005 return; 3006 } 3007 3008 // Generated branching requires both targets to be explicit. If either of the 3009 // targets is nullptr (fallthrough) use and bind `fallthrough` instead. 3010 vixl32::Label fallthrough; 3011 vixl32::Label* true_target = (true_target_in == nullptr) ? &fallthrough : true_target_in; 3012 vixl32::Label* false_target = (false_target_in == nullptr) ? &fallthrough : false_target_in; 3013 3014 DCHECK_EQ(condition->InputAt(0)->GetType(), Primitive::kPrimLong); 3015 GenerateLongComparesAndJumps(condition, true_target, false_target, codegen_, is_far_target); 3016 3017 if (false_target != &fallthrough) { 3018 __ B(false_target); 3019 } 3020 3021 if (fallthrough.IsReferenced()) { 3022 __ Bind(&fallthrough); 3023 } 3024 } 3025 3026 void InstructionCodeGeneratorARMVIXL::GenerateTestAndBranch(HInstruction* instruction, 3027 size_t condition_input_index, 3028 vixl32::Label* true_target, 3029 vixl32::Label* false_target, 3030 bool far_target) { 3031 HInstruction* cond = instruction->InputAt(condition_input_index); 3032 3033 if (true_target == nullptr && false_target == nullptr) { 3034 // Nothing to do. The code always falls through. 3035 return; 3036 } else if (cond->IsIntConstant()) { 3037 // Constant condition, statically compared against "true" (integer value 1). 3038 if (cond->AsIntConstant()->IsTrue()) { 3039 if (true_target != nullptr) { 3040 __ B(true_target); 3041 } 3042 } else { 3043 DCHECK(cond->AsIntConstant()->IsFalse()) << Int32ConstantFrom(cond); 3044 if (false_target != nullptr) { 3045 __ B(false_target); 3046 } 3047 } 3048 return; 3049 } 3050 3051 // The following code generates these patterns: 3052 // (1) true_target == nullptr && false_target != nullptr 3053 // - opposite condition true => branch to false_target 3054 // (2) true_target != nullptr && false_target == nullptr 3055 // - condition true => branch to true_target 3056 // (3) true_target != nullptr && false_target != nullptr 3057 // - condition true => branch to true_target 3058 // - branch to false_target 3059 if (IsBooleanValueOrMaterializedCondition(cond)) { 3060 // Condition has been materialized, compare the output to 0. 3061 if (kIsDebugBuild) { 3062 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 3063 DCHECK(cond_val.IsRegister()); 3064 } 3065 if (true_target == nullptr) { 3066 __ CompareAndBranchIfZero(InputRegisterAt(instruction, condition_input_index), 3067 false_target, 3068 far_target); 3069 } else { 3070 __ CompareAndBranchIfNonZero(InputRegisterAt(instruction, condition_input_index), 3071 true_target, 3072 far_target); 3073 } 3074 } else { 3075 // Condition has not been materialized. Use its inputs as the comparison and 3076 // its condition as the branch condition. 3077 HCondition* condition = cond->AsCondition(); 3078 3079 // If this is a long or FP comparison that has been folded into 3080 // the HCondition, generate the comparison directly. 3081 Primitive::Type type = condition->InputAt(0)->GetType(); 3082 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 3083 GenerateCompareTestAndBranch(condition, true_target, false_target, far_target); 3084 return; 3085 } 3086 3087 vixl32::Label* non_fallthrough_target; 3088 vixl32::Condition arm_cond = vixl32::Condition::None(); 3089 const vixl32::Register left = InputRegisterAt(cond, 0); 3090 const Operand right = InputOperandAt(cond, 1); 3091 3092 if (true_target == nullptr) { 3093 arm_cond = ARMCondition(condition->GetOppositeCondition()); 3094 non_fallthrough_target = false_target; 3095 } else { 3096 arm_cond = ARMCondition(condition->GetCondition()); 3097 non_fallthrough_target = true_target; 3098 } 3099 3100 if (right.IsImmediate() && right.GetImmediate() == 0 && (arm_cond.Is(ne) || arm_cond.Is(eq))) { 3101 if (arm_cond.Is(eq)) { 3102 __ CompareAndBranchIfZero(left, non_fallthrough_target, far_target); 3103 } else { 3104 DCHECK(arm_cond.Is(ne)); 3105 __ CompareAndBranchIfNonZero(left, non_fallthrough_target, far_target); 3106 } 3107 } else { 3108 __ Cmp(left, right); 3109 __ B(arm_cond, non_fallthrough_target, far_target); 3110 } 3111 } 3112 3113 // If neither branch falls through (case 3), the conditional branch to `true_target` 3114 // was already emitted (case 2) and we need to emit a jump to `false_target`. 3115 if (true_target != nullptr && false_target != nullptr) { 3116 __ B(false_target); 3117 } 3118 } 3119 3120 void LocationsBuilderARMVIXL::VisitIf(HIf* if_instr) { 3121 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 3122 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 3123 locations->SetInAt(0, Location::RequiresRegister()); 3124 } 3125 } 3126 3127 void InstructionCodeGeneratorARMVIXL::VisitIf(HIf* if_instr) { 3128 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 3129 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 3130 vixl32::Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 3131 nullptr : codegen_->GetLabelOf(true_successor); 3132 vixl32::Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 3133 nullptr : codegen_->GetLabelOf(false_successor); 3134 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 3135 } 3136 3137 void LocationsBuilderARMVIXL::VisitDeoptimize(HDeoptimize* deoptimize) { 3138 LocationSummary* locations = new (GetGraph()->GetArena()) 3139 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 3140 InvokeRuntimeCallingConventionARMVIXL calling_convention; 3141 RegisterSet caller_saves = RegisterSet::Empty(); 3142 caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0))); 3143 locations->SetCustomSlowPathCallerSaves(caller_saves); 3144 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 3145 locations->SetInAt(0, Location::RequiresRegister()); 3146 } 3147 } 3148 3149 void InstructionCodeGeneratorARMVIXL::VisitDeoptimize(HDeoptimize* deoptimize) { 3150 SlowPathCodeARMVIXL* slow_path = 3151 deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARMVIXL>(deoptimize); 3152 GenerateTestAndBranch(deoptimize, 3153 /* condition_input_index */ 0, 3154 slow_path->GetEntryLabel(), 3155 /* false_target */ nullptr); 3156 } 3157 3158 void LocationsBuilderARMVIXL::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 3159 LocationSummary* locations = new (GetGraph()->GetArena()) 3160 LocationSummary(flag, LocationSummary::kNoCall); 3161 locations->SetOut(Location::RequiresRegister()); 3162 } 3163 3164 void InstructionCodeGeneratorARMVIXL::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 3165 GetAssembler()->LoadFromOffset(kLoadWord, 3166 OutputRegister(flag), 3167 sp, 3168 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 3169 } 3170 3171 void LocationsBuilderARMVIXL::VisitSelect(HSelect* select) { 3172 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 3173 const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType()); 3174 3175 if (is_floating_point) { 3176 locations->SetInAt(0, Location::RequiresFpuRegister()); 3177 locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue())); 3178 } else { 3179 locations->SetInAt(0, Location::RequiresRegister()); 3180 locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue())); 3181 } 3182 3183 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 3184 locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition())); 3185 // The code generator handles overlap with the values, but not with the condition. 3186 locations->SetOut(Location::SameAsFirstInput()); 3187 } else if (is_floating_point) { 3188 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3189 } else { 3190 if (!locations->InAt(1).IsConstant()) { 3191 locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue())); 3192 } 3193 3194 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3195 } 3196 } 3197 3198 void InstructionCodeGeneratorARMVIXL::VisitSelect(HSelect* select) { 3199 HInstruction* const condition = select->GetCondition(); 3200 const LocationSummary* const locations = select->GetLocations(); 3201 const Primitive::Type type = select->GetType(); 3202 const Location first = locations->InAt(0); 3203 const Location out = locations->Out(); 3204 const Location second = locations->InAt(1); 3205 Location src; 3206 3207 if (condition->IsIntConstant()) { 3208 if (condition->AsIntConstant()->IsFalse()) { 3209 src = first; 3210 } else { 3211 src = second; 3212 } 3213 3214 codegen_->MoveLocation(out, src, type); 3215 return; 3216 } 3217 3218 if (!Primitive::IsFloatingPointType(type) && 3219 (IsBooleanValueOrMaterializedCondition(condition) || 3220 CanGenerateTest(condition->AsCondition(), codegen_->GetAssembler()))) { 3221 bool invert = false; 3222 3223 if (out.Equals(second)) { 3224 src = first; 3225 invert = true; 3226 } else if (out.Equals(first)) { 3227 src = second; 3228 } else if (second.IsConstant()) { 3229 DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant())); 3230 src = second; 3231 } else if (first.IsConstant()) { 3232 DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant())); 3233 src = first; 3234 invert = true; 3235 } else { 3236 src = second; 3237 } 3238 3239 if (CanGenerateConditionalMove(out, src)) { 3240 if (!out.Equals(first) && !out.Equals(second)) { 3241 codegen_->MoveLocation(out, src.Equals(first) ? second : first, type); 3242 } 3243 3244 std::pair<vixl32::Condition, vixl32::Condition> cond(eq, ne); 3245 3246 if (IsBooleanValueOrMaterializedCondition(condition)) { 3247 __ Cmp(InputRegisterAt(select, 2), 0); 3248 cond = invert ? std::make_pair(eq, ne) : std::make_pair(ne, eq); 3249 } else { 3250 cond = GenerateTest(condition->AsCondition(), invert, codegen_); 3251 } 3252 3253 const size_t instr_count = out.IsRegisterPair() ? 4 : 2; 3254 // We use the scope because of the IT block that follows. 3255 ExactAssemblyScope guard(GetVIXLAssembler(), 3256 instr_count * vixl32::k16BitT32InstructionSizeInBytes, 3257 CodeBufferCheckScope::kExactSize); 3258 3259 if (out.IsRegister()) { 3260 __ it(cond.first); 3261 __ mov(cond.first, RegisterFrom(out), OperandFrom(src, type)); 3262 } else { 3263 DCHECK(out.IsRegisterPair()); 3264 3265 Operand operand_high(0); 3266 Operand operand_low(0); 3267 3268 if (src.IsConstant()) { 3269 const int64_t value = Int64ConstantFrom(src); 3270 3271 operand_high = High32Bits(value); 3272 operand_low = Low32Bits(value); 3273 } else { 3274 DCHECK(src.IsRegisterPair()); 3275 operand_high = HighRegisterFrom(src); 3276 operand_low = LowRegisterFrom(src); 3277 } 3278 3279 __ it(cond.first); 3280 __ mov(cond.first, LowRegisterFrom(out), operand_low); 3281 __ it(cond.first); 3282 __ mov(cond.first, HighRegisterFrom(out), operand_high); 3283 } 3284 3285 return; 3286 } 3287 } 3288 3289 vixl32::Label* false_target = nullptr; 3290 vixl32::Label* true_target = nullptr; 3291 vixl32::Label select_end; 3292 vixl32::Label* const target = codegen_->GetFinalLabel(select, &select_end); 3293 3294 if (out.Equals(second)) { 3295 true_target = target; 3296 src = first; 3297 } else { 3298 false_target = target; 3299 src = second; 3300 3301 if (!out.Equals(first)) { 3302 codegen_->MoveLocation(out, first, type); 3303 } 3304 } 3305 3306 GenerateTestAndBranch(select, 2, true_target, false_target, /* far_target */ false); 3307 codegen_->MoveLocation(out, src, type); 3308 3309 if (select_end.IsReferenced()) { 3310 __ Bind(&select_end); 3311 } 3312 } 3313 3314 void LocationsBuilderARMVIXL::VisitNativeDebugInfo(HNativeDebugInfo* info) { 3315 new (GetGraph()->GetArena()) LocationSummary(info); 3316 } 3317 3318 void InstructionCodeGeneratorARMVIXL::VisitNativeDebugInfo(HNativeDebugInfo*) { 3319 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 3320 } 3321 3322 void CodeGeneratorARMVIXL::GenerateNop() { 3323 __ Nop(); 3324 } 3325 3326 // `temp` is an extra temporary register that is used for some conditions; 3327 // callers may not specify it, in which case the method will use a scratch 3328 // register instead. 3329 void CodeGeneratorARMVIXL::GenerateConditionWithZero(IfCondition condition, 3330 vixl32::Register out, 3331 vixl32::Register in, 3332 vixl32::Register temp) { 3333 switch (condition) { 3334 case kCondEQ: 3335 // x <= 0 iff x == 0 when the comparison is unsigned. 3336 case kCondBE: 3337 if (!temp.IsValid() || (out.IsLow() && !out.Is(in))) { 3338 temp = out; 3339 } 3340 3341 // Avoid 32-bit instructions if possible; note that `in` and `temp` must be 3342 // different as well. 3343 if (in.IsLow() && temp.IsLow() && !in.Is(temp)) { 3344 // temp = - in; only 0 sets the carry flag. 3345 __ Rsbs(temp, in, 0); 3346 3347 if (out.Is(in)) { 3348 std::swap(in, temp); 3349 } 3350 3351 // out = - in + in + carry = carry 3352 __ Adc(out, temp, in); 3353 } else { 3354 // If `in` is 0, then it has 32 leading zeros, and less than that otherwise. 3355 __ Clz(out, in); 3356 // Any number less than 32 logically shifted right by 5 bits results in 0; 3357 // the same operation on 32 yields 1. 3358 __ Lsr(out, out, 5); 3359 } 3360 3361 break; 3362 case kCondNE: 3363 // x > 0 iff x != 0 when the comparison is unsigned. 3364 case kCondA: { 3365 UseScratchRegisterScope temps(GetVIXLAssembler()); 3366 3367 if (out.Is(in)) { 3368 if (!temp.IsValid() || in.Is(temp)) { 3369 temp = temps.Acquire(); 3370 } 3371 } else if (!temp.IsValid() || !temp.IsLow()) { 3372 temp = out; 3373 } 3374 3375 // temp = in - 1; only 0 does not set the carry flag. 3376 __ Subs(temp, in, 1); 3377 // out = in + ~temp + carry = in + (-(in - 1) - 1) + carry = in - in + 1 - 1 + carry = carry 3378 __ Sbc(out, in, temp); 3379 break; 3380 } 3381 case kCondGE: 3382 __ Mvn(out, in); 3383 in = out; 3384 FALLTHROUGH_INTENDED; 3385 case kCondLT: 3386 // We only care about the sign bit. 3387 __ Lsr(out, in, 31); 3388 break; 3389 case kCondAE: 3390 // Trivially true. 3391 __ Mov(out, 1); 3392 break; 3393 case kCondB: 3394 // Trivially false. 3395 __ Mov(out, 0); 3396 break; 3397 default: 3398 LOG(FATAL) << "Unexpected condition " << condition; 3399 UNREACHABLE(); 3400 } 3401 } 3402 3403 void LocationsBuilderARMVIXL::HandleCondition(HCondition* cond) { 3404 LocationSummary* locations = 3405 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 3406 // Handle the long/FP comparisons made in instruction simplification. 3407 switch (cond->InputAt(0)->GetType()) { 3408 case Primitive::kPrimLong: 3409 locations->SetInAt(0, Location::RequiresRegister()); 3410 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 3411 if (!cond->IsEmittedAtUseSite()) { 3412 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3413 } 3414 break; 3415 3416 case Primitive::kPrimFloat: 3417 case Primitive::kPrimDouble: 3418 locations->SetInAt(0, Location::RequiresFpuRegister()); 3419 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 3420 if (!cond->IsEmittedAtUseSite()) { 3421 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3422 } 3423 break; 3424 3425 default: 3426 locations->SetInAt(0, Location::RequiresRegister()); 3427 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 3428 if (!cond->IsEmittedAtUseSite()) { 3429 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3430 } 3431 } 3432 } 3433 3434 void InstructionCodeGeneratorARMVIXL::HandleCondition(HCondition* cond) { 3435 if (cond->IsEmittedAtUseSite()) { 3436 return; 3437 } 3438 3439 const Primitive::Type type = cond->GetLeft()->GetType(); 3440 3441 if (Primitive::IsFloatingPointType(type)) { 3442 GenerateConditionGeneric(cond, codegen_); 3443 return; 3444 } 3445 3446 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 3447 3448 const IfCondition condition = cond->GetCondition(); 3449 3450 // A condition with only one boolean input, or two boolean inputs without being equality or 3451 // inequality results from transformations done by the instruction simplifier, and is handled 3452 // as a regular condition with integral inputs. 3453 if (type == Primitive::kPrimBoolean && 3454 cond->GetRight()->GetType() == Primitive::kPrimBoolean && 3455 (condition == kCondEQ || condition == kCondNE)) { 3456 vixl32::Register left = InputRegisterAt(cond, 0); 3457 const vixl32::Register out = OutputRegister(cond); 3458 const Location right_loc = cond->GetLocations()->InAt(1); 3459 3460 // The constant case is handled by the instruction simplifier. 3461 DCHECK(!right_loc.IsConstant()); 3462 3463 vixl32::Register right = RegisterFrom(right_loc); 3464 3465 // Avoid 32-bit instructions if possible. 3466 if (out.Is(right)) { 3467 std::swap(left, right); 3468 } 3469 3470 __ Eor(out, left, right); 3471 3472 if (condition == kCondEQ) { 3473 __ Eor(out, out, 1); 3474 } 3475 3476 return; 3477 } 3478 3479 GenerateConditionIntegralOrNonPrimitive(cond, codegen_); 3480 } 3481 3482 void LocationsBuilderARMVIXL::VisitEqual(HEqual* comp) { 3483 HandleCondition(comp); 3484 } 3485 3486 void InstructionCodeGeneratorARMVIXL::VisitEqual(HEqual* comp) { 3487 HandleCondition(comp); 3488 } 3489 3490 void LocationsBuilderARMVIXL::VisitNotEqual(HNotEqual* comp) { 3491 HandleCondition(comp); 3492 } 3493 3494 void InstructionCodeGeneratorARMVIXL::VisitNotEqual(HNotEqual* comp) { 3495 HandleCondition(comp); 3496 } 3497 3498 void LocationsBuilderARMVIXL::VisitLessThan(HLessThan* comp) { 3499 HandleCondition(comp); 3500 } 3501 3502 void InstructionCodeGeneratorARMVIXL::VisitLessThan(HLessThan* comp) { 3503 HandleCondition(comp); 3504 } 3505 3506 void LocationsBuilderARMVIXL::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 3507 HandleCondition(comp); 3508 } 3509 3510 void InstructionCodeGeneratorARMVIXL::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 3511 HandleCondition(comp); 3512 } 3513 3514 void LocationsBuilderARMVIXL::VisitGreaterThan(HGreaterThan* comp) { 3515 HandleCondition(comp); 3516 } 3517 3518 void InstructionCodeGeneratorARMVIXL::VisitGreaterThan(HGreaterThan* comp) { 3519 HandleCondition(comp); 3520 } 3521 3522 void LocationsBuilderARMVIXL::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 3523 HandleCondition(comp); 3524 } 3525 3526 void InstructionCodeGeneratorARMVIXL::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 3527 HandleCondition(comp); 3528 } 3529 3530 void LocationsBuilderARMVIXL::VisitBelow(HBelow* comp) { 3531 HandleCondition(comp); 3532 } 3533 3534 void InstructionCodeGeneratorARMVIXL::VisitBelow(HBelow* comp) { 3535 HandleCondition(comp); 3536 } 3537 3538 void LocationsBuilderARMVIXL::VisitBelowOrEqual(HBelowOrEqual* comp) { 3539 HandleCondition(comp); 3540 } 3541 3542 void InstructionCodeGeneratorARMVIXL::VisitBelowOrEqual(HBelowOrEqual* comp) { 3543 HandleCondition(comp); 3544 } 3545 3546 void LocationsBuilderARMVIXL::VisitAbove(HAbove* comp) { 3547 HandleCondition(comp); 3548 } 3549 3550 void InstructionCodeGeneratorARMVIXL::VisitAbove(HAbove* comp) { 3551 HandleCondition(comp); 3552 } 3553 3554 void LocationsBuilderARMVIXL::VisitAboveOrEqual(HAboveOrEqual* comp) { 3555 HandleCondition(comp); 3556 } 3557 3558 void InstructionCodeGeneratorARMVIXL::VisitAboveOrEqual(HAboveOrEqual* comp) { 3559 HandleCondition(comp); 3560 } 3561 3562 void LocationsBuilderARMVIXL::VisitIntConstant(HIntConstant* constant) { 3563 LocationSummary* locations = 3564 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3565 locations->SetOut(Location::ConstantLocation(constant)); 3566 } 3567 3568 void InstructionCodeGeneratorARMVIXL::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 3569 // Will be generated at use site. 3570 } 3571 3572 void LocationsBuilderARMVIXL::VisitNullConstant(HNullConstant* constant) { 3573 LocationSummary* locations = 3574 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3575 locations->SetOut(Location::ConstantLocation(constant)); 3576 } 3577 3578 void InstructionCodeGeneratorARMVIXL::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 3579 // Will be generated at use site. 3580 } 3581 3582 void LocationsBuilderARMVIXL::VisitLongConstant(HLongConstant* constant) { 3583 LocationSummary* locations = 3584 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3585 locations->SetOut(Location::ConstantLocation(constant)); 3586 } 3587 3588 void InstructionCodeGeneratorARMVIXL::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 3589 // Will be generated at use site. 3590 } 3591 3592 void LocationsBuilderARMVIXL::VisitFloatConstant(HFloatConstant* constant) { 3593 LocationSummary* locations = 3594 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3595 locations->SetOut(Location::ConstantLocation(constant)); 3596 } 3597 3598 void InstructionCodeGeneratorARMVIXL::VisitFloatConstant( 3599 HFloatConstant* constant ATTRIBUTE_UNUSED) { 3600 // Will be generated at use site. 3601 } 3602 3603 void LocationsBuilderARMVIXL::VisitDoubleConstant(HDoubleConstant* constant) { 3604 LocationSummary* locations = 3605 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3606 locations->SetOut(Location::ConstantLocation(constant)); 3607 } 3608 3609 void InstructionCodeGeneratorARMVIXL::VisitDoubleConstant( 3610 HDoubleConstant* constant ATTRIBUTE_UNUSED) { 3611 // Will be generated at use site. 3612 } 3613 3614 void LocationsBuilderARMVIXL::VisitConstructorFence(HConstructorFence* constructor_fence) { 3615 constructor_fence->SetLocations(nullptr); 3616 } 3617 3618 void InstructionCodeGeneratorARMVIXL::VisitConstructorFence( 3619 HConstructorFence* constructor_fence ATTRIBUTE_UNUSED) { 3620 codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); 3621 } 3622 3623 void LocationsBuilderARMVIXL::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3624 memory_barrier->SetLocations(nullptr); 3625 } 3626 3627 void InstructionCodeGeneratorARMVIXL::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3628 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 3629 } 3630 3631 void LocationsBuilderARMVIXL::VisitReturnVoid(HReturnVoid* ret) { 3632 ret->SetLocations(nullptr); 3633 } 3634 3635 void InstructionCodeGeneratorARMVIXL::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 3636 codegen_->GenerateFrameExit(); 3637 } 3638 3639 void LocationsBuilderARMVIXL::VisitReturn(HReturn* ret) { 3640 LocationSummary* locations = 3641 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 3642 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 3643 } 3644 3645 void InstructionCodeGeneratorARMVIXL::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 3646 codegen_->GenerateFrameExit(); 3647 } 3648 3649 void LocationsBuilderARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3650 // The trampoline uses the same calling convention as dex calling conventions, 3651 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 3652 // the method_idx. 3653 HandleInvoke(invoke); 3654 } 3655 3656 void InstructionCodeGeneratorARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3657 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 3658 } 3659 3660 void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3661 // Explicit clinit checks triggered by static invokes must have been pruned by 3662 // art::PrepareForRegisterAllocation. 3663 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3664 3665 IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_); 3666 if (intrinsic.TryDispatch(invoke)) { 3667 return; 3668 } 3669 3670 HandleInvoke(invoke); 3671 } 3672 3673 static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARMVIXL* codegen) { 3674 if (invoke->GetLocations()->Intrinsified()) { 3675 IntrinsicCodeGeneratorARMVIXL intrinsic(codegen); 3676 intrinsic.Dispatch(invoke); 3677 return true; 3678 } 3679 return false; 3680 } 3681 3682 void InstructionCodeGeneratorARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3683 // Explicit clinit checks triggered by static invokes must have been pruned by 3684 // art::PrepareForRegisterAllocation. 3685 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3686 3687 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3688 return; 3689 } 3690 3691 LocationSummary* locations = invoke->GetLocations(); 3692 codegen_->GenerateStaticOrDirectCall( 3693 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 3694 } 3695 3696 void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) { 3697 InvokeDexCallingConventionVisitorARMVIXL calling_convention_visitor; 3698 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 3699 } 3700 3701 void LocationsBuilderARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3702 IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_); 3703 if (intrinsic.TryDispatch(invoke)) { 3704 return; 3705 } 3706 3707 HandleInvoke(invoke); 3708 } 3709 3710 void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3711 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3712 return; 3713 } 3714 3715 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 3716 DCHECK(!codegen_->IsLeafMethod()); 3717 } 3718 3719 void LocationsBuilderARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) { 3720 HandleInvoke(invoke); 3721 // Add the hidden argument. 3722 invoke->GetLocations()->AddTemp(LocationFrom(r12)); 3723 } 3724 3725 void InstructionCodeGeneratorARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) { 3726 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 3727 LocationSummary* locations = invoke->GetLocations(); 3728 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 3729 vixl32::Register hidden_reg = RegisterFrom(locations->GetTemp(1)); 3730 Location receiver = locations->InAt(0); 3731 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 3732 3733 DCHECK(!receiver.IsStackSlot()); 3734 3735 // Ensure the pc position is recorded immediately after the `ldr` instruction. 3736 { 3737 ExactAssemblyScope aas(GetVIXLAssembler(), 3738 vixl32::kMaxInstructionSizeInBytes, 3739 CodeBufferCheckScope::kMaximumSize); 3740 // /* HeapReference<Class> */ temp = receiver->klass_ 3741 __ ldr(temp, MemOperand(RegisterFrom(receiver), class_offset)); 3742 codegen_->MaybeRecordImplicitNullCheck(invoke); 3743 } 3744 // Instead of simply (possibly) unpoisoning `temp` here, we should 3745 // emit a read barrier for the previous class reference load. 3746 // However this is not required in practice, as this is an 3747 // intermediate/temporary reference and because the current 3748 // concurrent copying collector keeps the from-space memory 3749 // intact/accessible until the end of the marking phase (the 3750 // concurrent copying collector may not in the future). 3751 GetAssembler()->MaybeUnpoisonHeapReference(temp); 3752 GetAssembler()->LoadFromOffset(kLoadWord, 3753 temp, 3754 temp, 3755 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 3756 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 3757 invoke->GetImtIndex(), kArmPointerSize)); 3758 // temp = temp->GetImtEntryAt(method_offset); 3759 GetAssembler()->LoadFromOffset(kLoadWord, temp, temp, method_offset); 3760 uint32_t entry_point = 3761 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 3762 // LR = temp->GetEntryPoint(); 3763 GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, entry_point); 3764 3765 // Set the hidden (in r12) argument. It is done here, right before a BLX to prevent other 3766 // instruction from clobbering it as they might use r12 as a scratch register. 3767 DCHECK(hidden_reg.Is(r12)); 3768 3769 { 3770 // The VIXL macro assembler may clobber any of the scratch registers that are available to it, 3771 // so it checks if the application is using them (by passing them to the macro assembler 3772 // methods). The following application of UseScratchRegisterScope corrects VIXL's notion of 3773 // what is available, and is the opposite of the standard usage: Instead of requesting a 3774 // temporary location, it imposes an external constraint (i.e. a specific register is reserved 3775 // for the hidden argument). Note that this works even if VIXL needs a scratch register itself 3776 // (to materialize the constant), since the destination register becomes available for such use 3777 // internally for the duration of the macro instruction. 3778 UseScratchRegisterScope temps(GetVIXLAssembler()); 3779 temps.Exclude(hidden_reg); 3780 __ Mov(hidden_reg, invoke->GetDexMethodIndex()); 3781 } 3782 { 3783 // Ensure the pc position is recorded immediately after the `blx` instruction. 3784 // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. 3785 ExactAssemblyScope aas(GetVIXLAssembler(), 3786 vixl32::k16BitT32InstructionSizeInBytes, 3787 CodeBufferCheckScope::kExactSize); 3788 // LR(); 3789 __ blx(lr); 3790 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3791 DCHECK(!codegen_->IsLeafMethod()); 3792 } 3793 } 3794 3795 void LocationsBuilderARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3796 HandleInvoke(invoke); 3797 } 3798 3799 void InstructionCodeGeneratorARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3800 codegen_->GenerateInvokePolymorphicCall(invoke); 3801 } 3802 3803 void LocationsBuilderARMVIXL::VisitNeg(HNeg* neg) { 3804 LocationSummary* locations = 3805 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 3806 switch (neg->GetResultType()) { 3807 case Primitive::kPrimInt: { 3808 locations->SetInAt(0, Location::RequiresRegister()); 3809 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3810 break; 3811 } 3812 case Primitive::kPrimLong: { 3813 locations->SetInAt(0, Location::RequiresRegister()); 3814 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3815 break; 3816 } 3817 3818 case Primitive::kPrimFloat: 3819 case Primitive::kPrimDouble: 3820 locations->SetInAt(0, Location::RequiresFpuRegister()); 3821 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3822 break; 3823 3824 default: 3825 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3826 } 3827 } 3828 3829 void InstructionCodeGeneratorARMVIXL::VisitNeg(HNeg* neg) { 3830 LocationSummary* locations = neg->GetLocations(); 3831 Location out = locations->Out(); 3832 Location in = locations->InAt(0); 3833 switch (neg->GetResultType()) { 3834 case Primitive::kPrimInt: 3835 __ Rsb(OutputRegister(neg), InputRegisterAt(neg, 0), 0); 3836 break; 3837 3838 case Primitive::kPrimLong: 3839 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 3840 __ Rsbs(LowRegisterFrom(out), LowRegisterFrom(in), 0); 3841 // We cannot emit an RSC (Reverse Subtract with Carry) 3842 // instruction here, as it does not exist in the Thumb-2 3843 // instruction set. We use the following approach 3844 // using SBC and SUB instead. 3845 // 3846 // out.hi = -C 3847 __ Sbc(HighRegisterFrom(out), HighRegisterFrom(out), HighRegisterFrom(out)); 3848 // out.hi = out.hi - in.hi 3849 __ Sub(HighRegisterFrom(out), HighRegisterFrom(out), HighRegisterFrom(in)); 3850 break; 3851 3852 case Primitive::kPrimFloat: 3853 case Primitive::kPrimDouble: 3854 __ Vneg(OutputVRegister(neg), InputVRegister(neg)); 3855 break; 3856 3857 default: 3858 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3859 } 3860 } 3861 3862 void LocationsBuilderARMVIXL::VisitTypeConversion(HTypeConversion* conversion) { 3863 Primitive::Type result_type = conversion->GetResultType(); 3864 Primitive::Type input_type = conversion->GetInputType(); 3865 DCHECK_NE(result_type, input_type); 3866 3867 // The float-to-long, double-to-long and long-to-float type conversions 3868 // rely on a call to the runtime. 3869 LocationSummary::CallKind call_kind = 3870 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 3871 && result_type == Primitive::kPrimLong) 3872 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 3873 ? LocationSummary::kCallOnMainOnly 3874 : LocationSummary::kNoCall; 3875 LocationSummary* locations = 3876 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 3877 3878 // The Java language does not allow treating boolean as an integral type but 3879 // our bit representation makes it safe. 3880 3881 switch (result_type) { 3882 case Primitive::kPrimByte: 3883 switch (input_type) { 3884 case Primitive::kPrimLong: 3885 // Type conversion from long to byte is a result of code transformations. 3886 case Primitive::kPrimBoolean: 3887 // Boolean input is a result of code transformations. 3888 case Primitive::kPrimShort: 3889 case Primitive::kPrimInt: 3890 case Primitive::kPrimChar: 3891 // Processing a Dex `int-to-byte' instruction. 3892 locations->SetInAt(0, Location::RequiresRegister()); 3893 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3894 break; 3895 3896 default: 3897 LOG(FATAL) << "Unexpected type conversion from " << input_type 3898 << " to " << result_type; 3899 } 3900 break; 3901 3902 case Primitive::kPrimShort: 3903 switch (input_type) { 3904 case Primitive::kPrimLong: 3905 // Type conversion from long to short is a result of code transformations. 3906 case Primitive::kPrimBoolean: 3907 // Boolean input is a result of code transformations. 3908 case Primitive::kPrimByte: 3909 case Primitive::kPrimInt: 3910 case Primitive::kPrimChar: 3911 // Processing a Dex `int-to-short' instruction. 3912 locations->SetInAt(0, Location::RequiresRegister()); 3913 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3914 break; 3915 3916 default: 3917 LOG(FATAL) << "Unexpected type conversion from " << input_type 3918 << " to " << result_type; 3919 } 3920 break; 3921 3922 case Primitive::kPrimInt: 3923 switch (input_type) { 3924 case Primitive::kPrimLong: 3925 // Processing a Dex `long-to-int' instruction. 3926 locations->SetInAt(0, Location::Any()); 3927 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3928 break; 3929 3930 case Primitive::kPrimFloat: 3931 // Processing a Dex `float-to-int' instruction. 3932 locations->SetInAt(0, Location::RequiresFpuRegister()); 3933 locations->SetOut(Location::RequiresRegister()); 3934 locations->AddTemp(Location::RequiresFpuRegister()); 3935 break; 3936 3937 case Primitive::kPrimDouble: 3938 // Processing a Dex `double-to-int' instruction. 3939 locations->SetInAt(0, Location::RequiresFpuRegister()); 3940 locations->SetOut(Location::RequiresRegister()); 3941 locations->AddTemp(Location::RequiresFpuRegister()); 3942 break; 3943 3944 default: 3945 LOG(FATAL) << "Unexpected type conversion from " << input_type 3946 << " to " << result_type; 3947 } 3948 break; 3949 3950 case Primitive::kPrimLong: 3951 switch (input_type) { 3952 case Primitive::kPrimBoolean: 3953 // Boolean input is a result of code transformations. 3954 case Primitive::kPrimByte: 3955 case Primitive::kPrimShort: 3956 case Primitive::kPrimInt: 3957 case Primitive::kPrimChar: 3958 // Processing a Dex `int-to-long' instruction. 3959 locations->SetInAt(0, Location::RequiresRegister()); 3960 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3961 break; 3962 3963 case Primitive::kPrimFloat: { 3964 // Processing a Dex `float-to-long' instruction. 3965 InvokeRuntimeCallingConventionARMVIXL calling_convention; 3966 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0))); 3967 locations->SetOut(LocationFrom(r0, r1)); 3968 break; 3969 } 3970 3971 case Primitive::kPrimDouble: { 3972 // Processing a Dex `double-to-long' instruction. 3973 InvokeRuntimeCallingConventionARMVIXL calling_convention; 3974 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0), 3975 calling_convention.GetFpuRegisterAt(1))); 3976 locations->SetOut(LocationFrom(r0, r1)); 3977 break; 3978 } 3979 3980 default: 3981 LOG(FATAL) << "Unexpected type conversion from " << input_type 3982 << " to " << result_type; 3983 } 3984 break; 3985 3986 case Primitive::kPrimChar: 3987 switch (input_type) { 3988 case Primitive::kPrimLong: 3989 // Type conversion from long to char is a result of code transformations. 3990 case Primitive::kPrimBoolean: 3991 // Boolean input is a result of code transformations. 3992 case Primitive::kPrimByte: 3993 case Primitive::kPrimShort: 3994 case Primitive::kPrimInt: 3995 // Processing a Dex `int-to-char' instruction. 3996 locations->SetInAt(0, Location::RequiresRegister()); 3997 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3998 break; 3999 4000 default: 4001 LOG(FATAL) << "Unexpected type conversion from " << input_type 4002 << " to " << result_type; 4003 } 4004 break; 4005 4006 case Primitive::kPrimFloat: 4007 switch (input_type) { 4008 case Primitive::kPrimBoolean: 4009 // Boolean input is a result of code transformations. 4010 case Primitive::kPrimByte: 4011 case Primitive::kPrimShort: 4012 case Primitive::kPrimInt: 4013 case Primitive::kPrimChar: 4014 // Processing a Dex `int-to-float' instruction. 4015 locations->SetInAt(0, Location::RequiresRegister()); 4016 locations->SetOut(Location::RequiresFpuRegister()); 4017 break; 4018 4019 case Primitive::kPrimLong: { 4020 // Processing a Dex `long-to-float' instruction. 4021 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4022 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0), 4023 calling_convention.GetRegisterAt(1))); 4024 locations->SetOut(LocationFrom(calling_convention.GetFpuRegisterAt(0))); 4025 break; 4026 } 4027 4028 case Primitive::kPrimDouble: 4029 // Processing a Dex `double-to-float' instruction. 4030 locations->SetInAt(0, Location::RequiresFpuRegister()); 4031 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4032 break; 4033 4034 default: 4035 LOG(FATAL) << "Unexpected type conversion from " << input_type 4036 << " to " << result_type; 4037 }; 4038 break; 4039 4040 case Primitive::kPrimDouble: 4041 switch (input_type) { 4042 case Primitive::kPrimBoolean: 4043 // Boolean input is a result of code transformations. 4044 case Primitive::kPrimByte: 4045 case Primitive::kPrimShort: 4046 case Primitive::kPrimInt: 4047 case Primitive::kPrimChar: 4048 // Processing a Dex `int-to-double' instruction. 4049 locations->SetInAt(0, Location::RequiresRegister()); 4050 locations->SetOut(Location::RequiresFpuRegister()); 4051 break; 4052 4053 case Primitive::kPrimLong: 4054 // Processing a Dex `long-to-double' instruction. 4055 locations->SetInAt(0, Location::RequiresRegister()); 4056 locations->SetOut(Location::RequiresFpuRegister()); 4057 locations->AddTemp(Location::RequiresFpuRegister()); 4058 locations->AddTemp(Location::RequiresFpuRegister()); 4059 break; 4060 4061 case Primitive::kPrimFloat: 4062 // Processing a Dex `float-to-double' instruction. 4063 locations->SetInAt(0, Location::RequiresFpuRegister()); 4064 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4065 break; 4066 4067 default: 4068 LOG(FATAL) << "Unexpected type conversion from " << input_type 4069 << " to " << result_type; 4070 }; 4071 break; 4072 4073 default: 4074 LOG(FATAL) << "Unexpected type conversion from " << input_type 4075 << " to " << result_type; 4076 } 4077 } 4078 4079 void InstructionCodeGeneratorARMVIXL::VisitTypeConversion(HTypeConversion* conversion) { 4080 LocationSummary* locations = conversion->GetLocations(); 4081 Location out = locations->Out(); 4082 Location in = locations->InAt(0); 4083 Primitive::Type result_type = conversion->GetResultType(); 4084 Primitive::Type input_type = conversion->GetInputType(); 4085 DCHECK_NE(result_type, input_type); 4086 switch (result_type) { 4087 case Primitive::kPrimByte: 4088 switch (input_type) { 4089 case Primitive::kPrimLong: 4090 // Type conversion from long to byte is a result of code transformations. 4091 __ Sbfx(OutputRegister(conversion), LowRegisterFrom(in), 0, 8); 4092 break; 4093 case Primitive::kPrimBoolean: 4094 // Boolean input is a result of code transformations. 4095 case Primitive::kPrimShort: 4096 case Primitive::kPrimInt: 4097 case Primitive::kPrimChar: 4098 // Processing a Dex `int-to-byte' instruction. 4099 __ Sbfx(OutputRegister(conversion), InputRegisterAt(conversion, 0), 0, 8); 4100 break; 4101 4102 default: 4103 LOG(FATAL) << "Unexpected type conversion from " << input_type 4104 << " to " << result_type; 4105 } 4106 break; 4107 4108 case Primitive::kPrimShort: 4109 switch (input_type) { 4110 case Primitive::kPrimLong: 4111 // Type conversion from long to short is a result of code transformations. 4112 __ Sbfx(OutputRegister(conversion), LowRegisterFrom(in), 0, 16); 4113 break; 4114 case Primitive::kPrimBoolean: 4115 // Boolean input is a result of code transformations. 4116 case Primitive::kPrimByte: 4117 case Primitive::kPrimInt: 4118 case Primitive::kPrimChar: 4119 // Processing a Dex `int-to-short' instruction. 4120 __ Sbfx(OutputRegister(conversion), InputRegisterAt(conversion, 0), 0, 16); 4121 break; 4122 4123 default: 4124 LOG(FATAL) << "Unexpected type conversion from " << input_type 4125 << " to " << result_type; 4126 } 4127 break; 4128 4129 case Primitive::kPrimInt: 4130 switch (input_type) { 4131 case Primitive::kPrimLong: 4132 // Processing a Dex `long-to-int' instruction. 4133 DCHECK(out.IsRegister()); 4134 if (in.IsRegisterPair()) { 4135 __ Mov(OutputRegister(conversion), LowRegisterFrom(in)); 4136 } else if (in.IsDoubleStackSlot()) { 4137 GetAssembler()->LoadFromOffset(kLoadWord, 4138 OutputRegister(conversion), 4139 sp, 4140 in.GetStackIndex()); 4141 } else { 4142 DCHECK(in.IsConstant()); 4143 DCHECK(in.GetConstant()->IsLongConstant()); 4144 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 4145 __ Mov(OutputRegister(conversion), static_cast<int32_t>(value)); 4146 } 4147 break; 4148 4149 case Primitive::kPrimFloat: { 4150 // Processing a Dex `float-to-int' instruction. 4151 vixl32::SRegister temp = LowSRegisterFrom(locations->GetTemp(0)); 4152 __ Vcvt(S32, F32, temp, InputSRegisterAt(conversion, 0)); 4153 __ Vmov(OutputRegister(conversion), temp); 4154 break; 4155 } 4156 4157 case Primitive::kPrimDouble: { 4158 // Processing a Dex `double-to-int' instruction. 4159 vixl32::SRegister temp_s = LowSRegisterFrom(locations->GetTemp(0)); 4160 __ Vcvt(S32, F64, temp_s, DRegisterFrom(in)); 4161 __ Vmov(OutputRegister(conversion), temp_s); 4162 break; 4163 } 4164 4165 default: 4166 LOG(FATAL) << "Unexpected type conversion from " << input_type 4167 << " to " << result_type; 4168 } 4169 break; 4170 4171 case Primitive::kPrimLong: 4172 switch (input_type) { 4173 case Primitive::kPrimBoolean: 4174 // Boolean input is a result of code transformations. 4175 case Primitive::kPrimByte: 4176 case Primitive::kPrimShort: 4177 case Primitive::kPrimInt: 4178 case Primitive::kPrimChar: 4179 // Processing a Dex `int-to-long' instruction. 4180 DCHECK(out.IsRegisterPair()); 4181 DCHECK(in.IsRegister()); 4182 __ Mov(LowRegisterFrom(out), InputRegisterAt(conversion, 0)); 4183 // Sign extension. 4184 __ Asr(HighRegisterFrom(out), LowRegisterFrom(out), 31); 4185 break; 4186 4187 case Primitive::kPrimFloat: 4188 // Processing a Dex `float-to-long' instruction. 4189 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 4190 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 4191 break; 4192 4193 case Primitive::kPrimDouble: 4194 // Processing a Dex `double-to-long' instruction. 4195 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 4196 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 4197 break; 4198 4199 default: 4200 LOG(FATAL) << "Unexpected type conversion from " << input_type 4201 << " to " << result_type; 4202 } 4203 break; 4204 4205 case Primitive::kPrimChar: 4206 switch (input_type) { 4207 case Primitive::kPrimLong: 4208 // Type conversion from long to char is a result of code transformations. 4209 __ Ubfx(OutputRegister(conversion), LowRegisterFrom(in), 0, 16); 4210 break; 4211 case Primitive::kPrimBoolean: 4212 // Boolean input is a result of code transformations. 4213 case Primitive::kPrimByte: 4214 case Primitive::kPrimShort: 4215 case Primitive::kPrimInt: 4216 // Processing a Dex `int-to-char' instruction. 4217 __ Ubfx(OutputRegister(conversion), InputRegisterAt(conversion, 0), 0, 16); 4218 break; 4219 4220 default: 4221 LOG(FATAL) << "Unexpected type conversion from " << input_type 4222 << " to " << result_type; 4223 } 4224 break; 4225 4226 case Primitive::kPrimFloat: 4227 switch (input_type) { 4228 case Primitive::kPrimBoolean: 4229 // Boolean input is a result of code transformations. 4230 case Primitive::kPrimByte: 4231 case Primitive::kPrimShort: 4232 case Primitive::kPrimInt: 4233 case Primitive::kPrimChar: { 4234 // Processing a Dex `int-to-float' instruction. 4235 __ Vmov(OutputSRegister(conversion), InputRegisterAt(conversion, 0)); 4236 __ Vcvt(F32, S32, OutputSRegister(conversion), OutputSRegister(conversion)); 4237 break; 4238 } 4239 4240 case Primitive::kPrimLong: 4241 // Processing a Dex `long-to-float' instruction. 4242 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 4243 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 4244 break; 4245 4246 case Primitive::kPrimDouble: 4247 // Processing a Dex `double-to-float' instruction. 4248 __ Vcvt(F32, F64, OutputSRegister(conversion), DRegisterFrom(in)); 4249 break; 4250 4251 default: 4252 LOG(FATAL) << "Unexpected type conversion from " << input_type 4253 << " to " << result_type; 4254 }; 4255 break; 4256 4257 case Primitive::kPrimDouble: 4258 switch (input_type) { 4259 case Primitive::kPrimBoolean: 4260 // Boolean input is a result of code transformations. 4261 case Primitive::kPrimByte: 4262 case Primitive::kPrimShort: 4263 case Primitive::kPrimInt: 4264 case Primitive::kPrimChar: { 4265 // Processing a Dex `int-to-double' instruction. 4266 __ Vmov(LowSRegisterFrom(out), InputRegisterAt(conversion, 0)); 4267 __ Vcvt(F64, S32, DRegisterFrom(out), LowSRegisterFrom(out)); 4268 break; 4269 } 4270 4271 case Primitive::kPrimLong: { 4272 // Processing a Dex `long-to-double' instruction. 4273 vixl32::Register low = LowRegisterFrom(in); 4274 vixl32::Register high = HighRegisterFrom(in); 4275 vixl32::SRegister out_s = LowSRegisterFrom(out); 4276 vixl32::DRegister out_d = DRegisterFrom(out); 4277 vixl32::SRegister temp_s = LowSRegisterFrom(locations->GetTemp(0)); 4278 vixl32::DRegister temp_d = DRegisterFrom(locations->GetTemp(0)); 4279 vixl32::DRegister constant_d = DRegisterFrom(locations->GetTemp(1)); 4280 4281 // temp_d = int-to-double(high) 4282 __ Vmov(temp_s, high); 4283 __ Vcvt(F64, S32, temp_d, temp_s); 4284 // constant_d = k2Pow32EncodingForDouble 4285 __ Vmov(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 4286 // out_d = unsigned-to-double(low) 4287 __ Vmov(out_s, low); 4288 __ Vcvt(F64, U32, out_d, out_s); 4289 // out_d += temp_d * constant_d 4290 __ Vmla(F64, out_d, temp_d, constant_d); 4291 break; 4292 } 4293 4294 case Primitive::kPrimFloat: 4295 // Processing a Dex `float-to-double' instruction. 4296 __ Vcvt(F64, F32, DRegisterFrom(out), InputSRegisterAt(conversion, 0)); 4297 break; 4298 4299 default: 4300 LOG(FATAL) << "Unexpected type conversion from " << input_type 4301 << " to " << result_type; 4302 }; 4303 break; 4304 4305 default: 4306 LOG(FATAL) << "Unexpected type conversion from " << input_type 4307 << " to " << result_type; 4308 } 4309 } 4310 4311 void LocationsBuilderARMVIXL::VisitAdd(HAdd* add) { 4312 LocationSummary* locations = 4313 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 4314 switch (add->GetResultType()) { 4315 case Primitive::kPrimInt: { 4316 locations->SetInAt(0, Location::RequiresRegister()); 4317 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 4318 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4319 break; 4320 } 4321 4322 case Primitive::kPrimLong: { 4323 locations->SetInAt(0, Location::RequiresRegister()); 4324 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 4325 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4326 break; 4327 } 4328 4329 case Primitive::kPrimFloat: 4330 case Primitive::kPrimDouble: { 4331 locations->SetInAt(0, Location::RequiresFpuRegister()); 4332 locations->SetInAt(1, Location::RequiresFpuRegister()); 4333 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4334 break; 4335 } 4336 4337 default: 4338 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 4339 } 4340 } 4341 4342 void InstructionCodeGeneratorARMVIXL::VisitAdd(HAdd* add) { 4343 LocationSummary* locations = add->GetLocations(); 4344 Location out = locations->Out(); 4345 Location first = locations->InAt(0); 4346 Location second = locations->InAt(1); 4347 4348 switch (add->GetResultType()) { 4349 case Primitive::kPrimInt: { 4350 __ Add(OutputRegister(add), InputRegisterAt(add, 0), InputOperandAt(add, 1)); 4351 } 4352 break; 4353 4354 case Primitive::kPrimLong: { 4355 if (second.IsConstant()) { 4356 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 4357 GenerateAddLongConst(out, first, value); 4358 } else { 4359 DCHECK(second.IsRegisterPair()); 4360 __ Adds(LowRegisterFrom(out), LowRegisterFrom(first), LowRegisterFrom(second)); 4361 __ Adc(HighRegisterFrom(out), HighRegisterFrom(first), HighRegisterFrom(second)); 4362 } 4363 break; 4364 } 4365 4366 case Primitive::kPrimFloat: 4367 case Primitive::kPrimDouble: 4368 __ Vadd(OutputVRegister(add), InputVRegisterAt(add, 0), InputVRegisterAt(add, 1)); 4369 break; 4370 4371 default: 4372 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 4373 } 4374 } 4375 4376 void LocationsBuilderARMVIXL::VisitSub(HSub* sub) { 4377 LocationSummary* locations = 4378 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 4379 switch (sub->GetResultType()) { 4380 case Primitive::kPrimInt: { 4381 locations->SetInAt(0, Location::RequiresRegister()); 4382 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 4383 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4384 break; 4385 } 4386 4387 case Primitive::kPrimLong: { 4388 locations->SetInAt(0, Location::RequiresRegister()); 4389 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 4390 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4391 break; 4392 } 4393 case Primitive::kPrimFloat: 4394 case Primitive::kPrimDouble: { 4395 locations->SetInAt(0, Location::RequiresFpuRegister()); 4396 locations->SetInAt(1, Location::RequiresFpuRegister()); 4397 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4398 break; 4399 } 4400 default: 4401 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 4402 } 4403 } 4404 4405 void InstructionCodeGeneratorARMVIXL::VisitSub(HSub* sub) { 4406 LocationSummary* locations = sub->GetLocations(); 4407 Location out = locations->Out(); 4408 Location first = locations->InAt(0); 4409 Location second = locations->InAt(1); 4410 switch (sub->GetResultType()) { 4411 case Primitive::kPrimInt: { 4412 __ Sub(OutputRegister(sub), InputRegisterAt(sub, 0), InputOperandAt(sub, 1)); 4413 break; 4414 } 4415 4416 case Primitive::kPrimLong: { 4417 if (second.IsConstant()) { 4418 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 4419 GenerateAddLongConst(out, first, -value); 4420 } else { 4421 DCHECK(second.IsRegisterPair()); 4422 __ Subs(LowRegisterFrom(out), LowRegisterFrom(first), LowRegisterFrom(second)); 4423 __ Sbc(HighRegisterFrom(out), HighRegisterFrom(first), HighRegisterFrom(second)); 4424 } 4425 break; 4426 } 4427 4428 case Primitive::kPrimFloat: 4429 case Primitive::kPrimDouble: 4430 __ Vsub(OutputVRegister(sub), InputVRegisterAt(sub, 0), InputVRegisterAt(sub, 1)); 4431 break; 4432 4433 default: 4434 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 4435 } 4436 } 4437 4438 void LocationsBuilderARMVIXL::VisitMul(HMul* mul) { 4439 LocationSummary* locations = 4440 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 4441 switch (mul->GetResultType()) { 4442 case Primitive::kPrimInt: 4443 case Primitive::kPrimLong: { 4444 locations->SetInAt(0, Location::RequiresRegister()); 4445 locations->SetInAt(1, Location::RequiresRegister()); 4446 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4447 break; 4448 } 4449 4450 case Primitive::kPrimFloat: 4451 case Primitive::kPrimDouble: { 4452 locations->SetInAt(0, Location::RequiresFpuRegister()); 4453 locations->SetInAt(1, Location::RequiresFpuRegister()); 4454 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4455 break; 4456 } 4457 4458 default: 4459 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 4460 } 4461 } 4462 4463 void InstructionCodeGeneratorARMVIXL::VisitMul(HMul* mul) { 4464 LocationSummary* locations = mul->GetLocations(); 4465 Location out = locations->Out(); 4466 Location first = locations->InAt(0); 4467 Location second = locations->InAt(1); 4468 switch (mul->GetResultType()) { 4469 case Primitive::kPrimInt: { 4470 __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1)); 4471 break; 4472 } 4473 case Primitive::kPrimLong: { 4474 vixl32::Register out_hi = HighRegisterFrom(out); 4475 vixl32::Register out_lo = LowRegisterFrom(out); 4476 vixl32::Register in1_hi = HighRegisterFrom(first); 4477 vixl32::Register in1_lo = LowRegisterFrom(first); 4478 vixl32::Register in2_hi = HighRegisterFrom(second); 4479 vixl32::Register in2_lo = LowRegisterFrom(second); 4480 4481 // Extra checks to protect caused by the existence of R1_R2. 4482 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 4483 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 4484 DCHECK(!out_hi.Is(in1_lo)); 4485 DCHECK(!out_hi.Is(in2_lo)); 4486 4487 // input: in1 - 64 bits, in2 - 64 bits 4488 // output: out 4489 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 4490 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 4491 // parts: out.lo = (in1.lo * in2.lo)[31:0] 4492 4493 UseScratchRegisterScope temps(GetVIXLAssembler()); 4494 vixl32::Register temp = temps.Acquire(); 4495 // temp <- in1.lo * in2.hi 4496 __ Mul(temp, in1_lo, in2_hi); 4497 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 4498 __ Mla(out_hi, in1_hi, in2_lo, temp); 4499 // out.lo <- (in1.lo * in2.lo)[31:0]; 4500 __ Umull(out_lo, temp, in1_lo, in2_lo); 4501 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 4502 __ Add(out_hi, out_hi, temp); 4503 break; 4504 } 4505 4506 case Primitive::kPrimFloat: 4507 case Primitive::kPrimDouble: 4508 __ Vmul(OutputVRegister(mul), InputVRegisterAt(mul, 0), InputVRegisterAt(mul, 1)); 4509 break; 4510 4511 default: 4512 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 4513 } 4514 } 4515 4516 void InstructionCodeGeneratorARMVIXL::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 4517 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4518 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4519 4520 Location second = instruction->GetLocations()->InAt(1); 4521 DCHECK(second.IsConstant()); 4522 4523 vixl32::Register out = OutputRegister(instruction); 4524 vixl32::Register dividend = InputRegisterAt(instruction, 0); 4525 int32_t imm = Int32ConstantFrom(second); 4526 DCHECK(imm == 1 || imm == -1); 4527 4528 if (instruction->IsRem()) { 4529 __ Mov(out, 0); 4530 } else { 4531 if (imm == 1) { 4532 __ Mov(out, dividend); 4533 } else { 4534 __ Rsb(out, dividend, 0); 4535 } 4536 } 4537 } 4538 4539 void InstructionCodeGeneratorARMVIXL::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 4540 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4541 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4542 4543 LocationSummary* locations = instruction->GetLocations(); 4544 Location second = locations->InAt(1); 4545 DCHECK(second.IsConstant()); 4546 4547 vixl32::Register out = OutputRegister(instruction); 4548 vixl32::Register dividend = InputRegisterAt(instruction, 0); 4549 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 4550 int32_t imm = Int32ConstantFrom(second); 4551 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 4552 int ctz_imm = CTZ(abs_imm); 4553 4554 if (ctz_imm == 1) { 4555 __ Lsr(temp, dividend, 32 - ctz_imm); 4556 } else { 4557 __ Asr(temp, dividend, 31); 4558 __ Lsr(temp, temp, 32 - ctz_imm); 4559 } 4560 __ Add(out, temp, dividend); 4561 4562 if (instruction->IsDiv()) { 4563 __ Asr(out, out, ctz_imm); 4564 if (imm < 0) { 4565 __ Rsb(out, out, 0); 4566 } 4567 } else { 4568 __ Ubfx(out, out, 0, ctz_imm); 4569 __ Sub(out, out, temp); 4570 } 4571 } 4572 4573 void InstructionCodeGeneratorARMVIXL::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 4574 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4575 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4576 4577 LocationSummary* locations = instruction->GetLocations(); 4578 Location second = locations->InAt(1); 4579 DCHECK(second.IsConstant()); 4580 4581 vixl32::Register out = OutputRegister(instruction); 4582 vixl32::Register dividend = InputRegisterAt(instruction, 0); 4583 vixl32::Register temp1 = RegisterFrom(locations->GetTemp(0)); 4584 vixl32::Register temp2 = RegisterFrom(locations->GetTemp(1)); 4585 int32_t imm = Int32ConstantFrom(second); 4586 4587 int64_t magic; 4588 int shift; 4589 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 4590 4591 // TODO(VIXL): Change the static cast to Operand::From() after VIXL is fixed. 4592 __ Mov(temp1, static_cast<int32_t>(magic)); 4593 __ Smull(temp2, temp1, dividend, temp1); 4594 4595 if (imm > 0 && magic < 0) { 4596 __ Add(temp1, temp1, dividend); 4597 } else if (imm < 0 && magic > 0) { 4598 __ Sub(temp1, temp1, dividend); 4599 } 4600 4601 if (shift != 0) { 4602 __ Asr(temp1, temp1, shift); 4603 } 4604 4605 if (instruction->IsDiv()) { 4606 __ Sub(out, temp1, Operand(temp1, vixl32::Shift(ASR), 31)); 4607 } else { 4608 __ Sub(temp1, temp1, Operand(temp1, vixl32::Shift(ASR), 31)); 4609 // TODO: Strength reduction for mls. 4610 __ Mov(temp2, imm); 4611 __ Mls(out, temp1, temp2, dividend); 4612 } 4613 } 4614 4615 void InstructionCodeGeneratorARMVIXL::GenerateDivRemConstantIntegral( 4616 HBinaryOperation* instruction) { 4617 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4618 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4619 4620 Location second = instruction->GetLocations()->InAt(1); 4621 DCHECK(second.IsConstant()); 4622 4623 int32_t imm = Int32ConstantFrom(second); 4624 if (imm == 0) { 4625 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 4626 } else if (imm == 1 || imm == -1) { 4627 DivRemOneOrMinusOne(instruction); 4628 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 4629 DivRemByPowerOfTwo(instruction); 4630 } else { 4631 DCHECK(imm <= -2 || imm >= 2); 4632 GenerateDivRemWithAnyConstant(instruction); 4633 } 4634 } 4635 4636 void LocationsBuilderARMVIXL::VisitDiv(HDiv* div) { 4637 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 4638 if (div->GetResultType() == Primitive::kPrimLong) { 4639 // pLdiv runtime call. 4640 call_kind = LocationSummary::kCallOnMainOnly; 4641 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 4642 // sdiv will be replaced by other instruction sequence. 4643 } else if (div->GetResultType() == Primitive::kPrimInt && 4644 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4645 // pIdivmod runtime call. 4646 call_kind = LocationSummary::kCallOnMainOnly; 4647 } 4648 4649 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 4650 4651 switch (div->GetResultType()) { 4652 case Primitive::kPrimInt: { 4653 if (div->InputAt(1)->IsConstant()) { 4654 locations->SetInAt(0, Location::RequiresRegister()); 4655 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 4656 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4657 int32_t value = Int32ConstantFrom(div->InputAt(1)); 4658 if (value == 1 || value == 0 || value == -1) { 4659 // No temp register required. 4660 } else { 4661 locations->AddTemp(Location::RequiresRegister()); 4662 if (!IsPowerOfTwo(AbsOrMin(value))) { 4663 locations->AddTemp(Location::RequiresRegister()); 4664 } 4665 } 4666 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4667 locations->SetInAt(0, Location::RequiresRegister()); 4668 locations->SetInAt(1, Location::RequiresRegister()); 4669 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4670 } else { 4671 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4672 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 4673 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 4674 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4675 // we only need the former. 4676 locations->SetOut(LocationFrom(r0)); 4677 } 4678 break; 4679 } 4680 case Primitive::kPrimLong: { 4681 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4682 locations->SetInAt(0, LocationFrom( 4683 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4684 locations->SetInAt(1, LocationFrom( 4685 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4686 locations->SetOut(LocationFrom(r0, r1)); 4687 break; 4688 } 4689 case Primitive::kPrimFloat: 4690 case Primitive::kPrimDouble: { 4691 locations->SetInAt(0, Location::RequiresFpuRegister()); 4692 locations->SetInAt(1, Location::RequiresFpuRegister()); 4693 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4694 break; 4695 } 4696 4697 default: 4698 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4699 } 4700 } 4701 4702 void InstructionCodeGeneratorARMVIXL::VisitDiv(HDiv* div) { 4703 Location lhs = div->GetLocations()->InAt(0); 4704 Location rhs = div->GetLocations()->InAt(1); 4705 4706 switch (div->GetResultType()) { 4707 case Primitive::kPrimInt: { 4708 if (rhs.IsConstant()) { 4709 GenerateDivRemConstantIntegral(div); 4710 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4711 __ Sdiv(OutputRegister(div), InputRegisterAt(div, 0), InputRegisterAt(div, 1)); 4712 } else { 4713 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4714 DCHECK(calling_convention.GetRegisterAt(0).Is(RegisterFrom(lhs))); 4715 DCHECK(calling_convention.GetRegisterAt(1).Is(RegisterFrom(rhs))); 4716 DCHECK(r0.Is(OutputRegister(div))); 4717 4718 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 4719 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4720 } 4721 break; 4722 } 4723 4724 case Primitive::kPrimLong: { 4725 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4726 DCHECK(calling_convention.GetRegisterAt(0).Is(LowRegisterFrom(lhs))); 4727 DCHECK(calling_convention.GetRegisterAt(1).Is(HighRegisterFrom(lhs))); 4728 DCHECK(calling_convention.GetRegisterAt(2).Is(LowRegisterFrom(rhs))); 4729 DCHECK(calling_convention.GetRegisterAt(3).Is(HighRegisterFrom(rhs))); 4730 DCHECK(LowRegisterFrom(div->GetLocations()->Out()).Is(r0)); 4731 DCHECK(HighRegisterFrom(div->GetLocations()->Out()).Is(r1)); 4732 4733 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 4734 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 4735 break; 4736 } 4737 4738 case Primitive::kPrimFloat: 4739 case Primitive::kPrimDouble: 4740 __ Vdiv(OutputVRegister(div), InputVRegisterAt(div, 0), InputVRegisterAt(div, 1)); 4741 break; 4742 4743 default: 4744 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4745 } 4746 } 4747 4748 void LocationsBuilderARMVIXL::VisitRem(HRem* rem) { 4749 Primitive::Type type = rem->GetResultType(); 4750 4751 // Most remainders are implemented in the runtime. 4752 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 4753 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 4754 // sdiv will be replaced by other instruction sequence. 4755 call_kind = LocationSummary::kNoCall; 4756 } else if ((rem->GetResultType() == Primitive::kPrimInt) 4757 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4758 // Have hardware divide instruction for int, do it with three instructions. 4759 call_kind = LocationSummary::kNoCall; 4760 } 4761 4762 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 4763 4764 switch (type) { 4765 case Primitive::kPrimInt: { 4766 if (rem->InputAt(1)->IsConstant()) { 4767 locations->SetInAt(0, Location::RequiresRegister()); 4768 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 4769 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4770 int32_t value = Int32ConstantFrom(rem->InputAt(1)); 4771 if (value == 1 || value == 0 || value == -1) { 4772 // No temp register required. 4773 } else { 4774 locations->AddTemp(Location::RequiresRegister()); 4775 if (!IsPowerOfTwo(AbsOrMin(value))) { 4776 locations->AddTemp(Location::RequiresRegister()); 4777 } 4778 } 4779 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4780 locations->SetInAt(0, Location::RequiresRegister()); 4781 locations->SetInAt(1, Location::RequiresRegister()); 4782 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4783 locations->AddTemp(Location::RequiresRegister()); 4784 } else { 4785 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4786 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 4787 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 4788 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4789 // we only need the latter. 4790 locations->SetOut(LocationFrom(r1)); 4791 } 4792 break; 4793 } 4794 case Primitive::kPrimLong: { 4795 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4796 locations->SetInAt(0, LocationFrom( 4797 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4798 locations->SetInAt(1, LocationFrom( 4799 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4800 // The runtime helper puts the output in R2,R3. 4801 locations->SetOut(LocationFrom(r2, r3)); 4802 break; 4803 } 4804 case Primitive::kPrimFloat: { 4805 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4806 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0))); 4807 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1))); 4808 locations->SetOut(LocationFrom(s0)); 4809 break; 4810 } 4811 4812 case Primitive::kPrimDouble: { 4813 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4814 locations->SetInAt(0, LocationFrom( 4815 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 4816 locations->SetInAt(1, LocationFrom( 4817 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 4818 locations->SetOut(LocationFrom(s0, s1)); 4819 break; 4820 } 4821 4822 default: 4823 LOG(FATAL) << "Unexpected rem type " << type; 4824 } 4825 } 4826 4827 void InstructionCodeGeneratorARMVIXL::VisitRem(HRem* rem) { 4828 LocationSummary* locations = rem->GetLocations(); 4829 Location second = locations->InAt(1); 4830 4831 Primitive::Type type = rem->GetResultType(); 4832 switch (type) { 4833 case Primitive::kPrimInt: { 4834 vixl32::Register reg1 = InputRegisterAt(rem, 0); 4835 vixl32::Register out_reg = OutputRegister(rem); 4836 if (second.IsConstant()) { 4837 GenerateDivRemConstantIntegral(rem); 4838 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4839 vixl32::Register reg2 = RegisterFrom(second); 4840 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 4841 4842 // temp = reg1 / reg2 (integer division) 4843 // dest = reg1 - temp * reg2 4844 __ Sdiv(temp, reg1, reg2); 4845 __ Mls(out_reg, temp, reg2, reg1); 4846 } else { 4847 InvokeRuntimeCallingConventionARMVIXL calling_convention; 4848 DCHECK(reg1.Is(calling_convention.GetRegisterAt(0))); 4849 DCHECK(RegisterFrom(second).Is(calling_convention.GetRegisterAt(1))); 4850 DCHECK(out_reg.Is(r1)); 4851 4852 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 4853 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4854 } 4855 break; 4856 } 4857 4858 case Primitive::kPrimLong: { 4859 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 4860 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 4861 break; 4862 } 4863 4864 case Primitive::kPrimFloat: { 4865 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 4866 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 4867 break; 4868 } 4869 4870 case Primitive::kPrimDouble: { 4871 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 4872 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 4873 break; 4874 } 4875 4876 default: 4877 LOG(FATAL) << "Unexpected rem type " << type; 4878 } 4879 } 4880 4881 4882 void LocationsBuilderARMVIXL::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4883 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4884 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 4885 } 4886 4887 void InstructionCodeGeneratorARMVIXL::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4888 DivZeroCheckSlowPathARMVIXL* slow_path = 4889 new (GetGraph()->GetArena()) DivZeroCheckSlowPathARMVIXL(instruction); 4890 codegen_->AddSlowPath(slow_path); 4891 4892 LocationSummary* locations = instruction->GetLocations(); 4893 Location value = locations->InAt(0); 4894 4895 switch (instruction->GetType()) { 4896 case Primitive::kPrimBoolean: 4897 case Primitive::kPrimByte: 4898 case Primitive::kPrimChar: 4899 case Primitive::kPrimShort: 4900 case Primitive::kPrimInt: { 4901 if (value.IsRegister()) { 4902 __ CompareAndBranchIfZero(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel()); 4903 } else { 4904 DCHECK(value.IsConstant()) << value; 4905 if (Int32ConstantFrom(value) == 0) { 4906 __ B(slow_path->GetEntryLabel()); 4907 } 4908 } 4909 break; 4910 } 4911 case Primitive::kPrimLong: { 4912 if (value.IsRegisterPair()) { 4913 UseScratchRegisterScope temps(GetVIXLAssembler()); 4914 vixl32::Register temp = temps.Acquire(); 4915 __ Orrs(temp, LowRegisterFrom(value), HighRegisterFrom(value)); 4916 __ B(eq, slow_path->GetEntryLabel()); 4917 } else { 4918 DCHECK(value.IsConstant()) << value; 4919 if (Int64ConstantFrom(value) == 0) { 4920 __ B(slow_path->GetEntryLabel()); 4921 } 4922 } 4923 break; 4924 } 4925 default: 4926 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 4927 } 4928 } 4929 4930 void InstructionCodeGeneratorARMVIXL::HandleIntegerRotate(HRor* ror) { 4931 LocationSummary* locations = ror->GetLocations(); 4932 vixl32::Register in = InputRegisterAt(ror, 0); 4933 Location rhs = locations->InAt(1); 4934 vixl32::Register out = OutputRegister(ror); 4935 4936 if (rhs.IsConstant()) { 4937 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 4938 // so map all rotations to a +ve. equivalent in that range. 4939 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 4940 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 4941 if (rot) { 4942 // Rotate, mapping left rotations to right equivalents if necessary. 4943 // (e.g. left by 2 bits == right by 30.) 4944 __ Ror(out, in, rot); 4945 } else if (!out.Is(in)) { 4946 __ Mov(out, in); 4947 } 4948 } else { 4949 __ Ror(out, in, RegisterFrom(rhs)); 4950 } 4951 } 4952 4953 // Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 4954 // rotates by swapping input regs (effectively rotating by the first 32-bits of 4955 // a larger rotation) or flipping direction (thus treating larger right/left 4956 // rotations as sub-word sized rotations in the other direction) as appropriate. 4957 void InstructionCodeGeneratorARMVIXL::HandleLongRotate(HRor* ror) { 4958 LocationSummary* locations = ror->GetLocations(); 4959 vixl32::Register in_reg_lo = LowRegisterFrom(locations->InAt(0)); 4960 vixl32::Register in_reg_hi = HighRegisterFrom(locations->InAt(0)); 4961 Location rhs = locations->InAt(1); 4962 vixl32::Register out_reg_lo = LowRegisterFrom(locations->Out()); 4963 vixl32::Register out_reg_hi = HighRegisterFrom(locations->Out()); 4964 4965 if (rhs.IsConstant()) { 4966 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 4967 // Map all rotations to +ve. equivalents on the interval [0,63]. 4968 rot &= kMaxLongShiftDistance; 4969 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 4970 // logic below to a simple pair of binary orr. 4971 // (e.g. 34 bits == in_reg swap + 2 bits right.) 4972 if (rot >= kArmBitsPerWord) { 4973 rot -= kArmBitsPerWord; 4974 std::swap(in_reg_hi, in_reg_lo); 4975 } 4976 // Rotate, or mov to out for zero or word size rotations. 4977 if (rot != 0u) { 4978 __ Lsr(out_reg_hi, in_reg_hi, Operand::From(rot)); 4979 __ Orr(out_reg_hi, out_reg_hi, Operand(in_reg_lo, ShiftType::LSL, kArmBitsPerWord - rot)); 4980 __ Lsr(out_reg_lo, in_reg_lo, Operand::From(rot)); 4981 __ Orr(out_reg_lo, out_reg_lo, Operand(in_reg_hi, ShiftType::LSL, kArmBitsPerWord - rot)); 4982 } else { 4983 __ Mov(out_reg_lo, in_reg_lo); 4984 __ Mov(out_reg_hi, in_reg_hi); 4985 } 4986 } else { 4987 vixl32::Register shift_right = RegisterFrom(locations->GetTemp(0)); 4988 vixl32::Register shift_left = RegisterFrom(locations->GetTemp(1)); 4989 vixl32::Label end; 4990 vixl32::Label shift_by_32_plus_shift_right; 4991 vixl32::Label* final_label = codegen_->GetFinalLabel(ror, &end); 4992 4993 __ And(shift_right, RegisterFrom(rhs), 0x1F); 4994 __ Lsrs(shift_left, RegisterFrom(rhs), 6); 4995 __ Rsb(LeaveFlags, shift_left, shift_right, Operand::From(kArmBitsPerWord)); 4996 __ B(cc, &shift_by_32_plus_shift_right, /* far_target */ false); 4997 4998 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 4999 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 5000 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 5001 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 5002 __ Add(out_reg_hi, out_reg_hi, out_reg_lo); 5003 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 5004 __ Lsr(shift_left, in_reg_hi, shift_right); 5005 __ Add(out_reg_lo, out_reg_lo, shift_left); 5006 __ B(final_label); 5007 5008 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 5009 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 5010 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 5011 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 5012 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 5013 __ Add(out_reg_hi, out_reg_hi, out_reg_lo); 5014 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 5015 __ Lsl(shift_right, in_reg_hi, shift_left); 5016 __ Add(out_reg_lo, out_reg_lo, shift_right); 5017 5018 if (end.IsReferenced()) { 5019 __ Bind(&end); 5020 } 5021 } 5022 } 5023 5024 void LocationsBuilderARMVIXL::VisitRor(HRor* ror) { 5025 LocationSummary* locations = 5026 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 5027 switch (ror->GetResultType()) { 5028 case Primitive::kPrimInt: { 5029 locations->SetInAt(0, Location::RequiresRegister()); 5030 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 5031 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5032 break; 5033 } 5034 case Primitive::kPrimLong: { 5035 locations->SetInAt(0, Location::RequiresRegister()); 5036 if (ror->InputAt(1)->IsConstant()) { 5037 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 5038 } else { 5039 locations->SetInAt(1, Location::RequiresRegister()); 5040 locations->AddTemp(Location::RequiresRegister()); 5041 locations->AddTemp(Location::RequiresRegister()); 5042 } 5043 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 5044 break; 5045 } 5046 default: 5047 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 5048 } 5049 } 5050 5051 void InstructionCodeGeneratorARMVIXL::VisitRor(HRor* ror) { 5052 Primitive::Type type = ror->GetResultType(); 5053 switch (type) { 5054 case Primitive::kPrimInt: { 5055 HandleIntegerRotate(ror); 5056 break; 5057 } 5058 case Primitive::kPrimLong: { 5059 HandleLongRotate(ror); 5060 break; 5061 } 5062 default: 5063 LOG(FATAL) << "Unexpected operation type " << type; 5064 UNREACHABLE(); 5065 } 5066 } 5067 5068 void LocationsBuilderARMVIXL::HandleShift(HBinaryOperation* op) { 5069 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 5070 5071 LocationSummary* locations = 5072 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 5073 5074 switch (op->GetResultType()) { 5075 case Primitive::kPrimInt: { 5076 locations->SetInAt(0, Location::RequiresRegister()); 5077 if (op->InputAt(1)->IsConstant()) { 5078 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 5079 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5080 } else { 5081 locations->SetInAt(1, Location::RequiresRegister()); 5082 // Make the output overlap, as it will be used to hold the masked 5083 // second input. 5084 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 5085 } 5086 break; 5087 } 5088 case Primitive::kPrimLong: { 5089 locations->SetInAt(0, Location::RequiresRegister()); 5090 if (op->InputAt(1)->IsConstant()) { 5091 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 5092 // For simplicity, use kOutputOverlap even though we only require that low registers 5093 // don't clash with high registers which the register allocator currently guarantees. 5094 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 5095 } else { 5096 locations->SetInAt(1, Location::RequiresRegister()); 5097 locations->AddTemp(Location::RequiresRegister()); 5098 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 5099 } 5100 break; 5101 } 5102 default: 5103 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 5104 } 5105 } 5106 5107 void InstructionCodeGeneratorARMVIXL::HandleShift(HBinaryOperation* op) { 5108 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 5109 5110 LocationSummary* locations = op->GetLocations(); 5111 Location out = locations->Out(); 5112 Location first = locations->InAt(0); 5113 Location second = locations->InAt(1); 5114 5115 Primitive::Type type = op->GetResultType(); 5116 switch (type) { 5117 case Primitive::kPrimInt: { 5118 vixl32::Register out_reg = OutputRegister(op); 5119 vixl32::Register first_reg = InputRegisterAt(op, 0); 5120 if (second.IsRegister()) { 5121 vixl32::Register second_reg = RegisterFrom(second); 5122 // ARM doesn't mask the shift count so we need to do it ourselves. 5123 __ And(out_reg, second_reg, kMaxIntShiftDistance); 5124 if (op->IsShl()) { 5125 __ Lsl(out_reg, first_reg, out_reg); 5126 } else if (op->IsShr()) { 5127 __ Asr(out_reg, first_reg, out_reg); 5128 } else { 5129 __ Lsr(out_reg, first_reg, out_reg); 5130 } 5131 } else { 5132 int32_t cst = Int32ConstantFrom(second); 5133 uint32_t shift_value = cst & kMaxIntShiftDistance; 5134 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 5135 __ Mov(out_reg, first_reg); 5136 } else if (op->IsShl()) { 5137 __ Lsl(out_reg, first_reg, shift_value); 5138 } else if (op->IsShr()) { 5139 __ Asr(out_reg, first_reg, shift_value); 5140 } else { 5141 __ Lsr(out_reg, first_reg, shift_value); 5142 } 5143 } 5144 break; 5145 } 5146 case Primitive::kPrimLong: { 5147 vixl32::Register o_h = HighRegisterFrom(out); 5148 vixl32::Register o_l = LowRegisterFrom(out); 5149 5150 vixl32::Register high = HighRegisterFrom(first); 5151 vixl32::Register low = LowRegisterFrom(first); 5152 5153 if (second.IsRegister()) { 5154 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 5155 5156 vixl32::Register second_reg = RegisterFrom(second); 5157 5158 if (op->IsShl()) { 5159 __ And(o_l, second_reg, kMaxLongShiftDistance); 5160 // Shift the high part 5161 __ Lsl(o_h, high, o_l); 5162 // Shift the low part and `or` what overflew on the high part 5163 __ Rsb(temp, o_l, Operand::From(kArmBitsPerWord)); 5164 __ Lsr(temp, low, temp); 5165 __ Orr(o_h, o_h, temp); 5166 // If the shift is > 32 bits, override the high part 5167 __ Subs(temp, o_l, Operand::From(kArmBitsPerWord)); 5168 { 5169 ExactAssemblyScope guard(GetVIXLAssembler(), 5170 2 * vixl32::kMaxInstructionSizeInBytes, 5171 CodeBufferCheckScope::kMaximumSize); 5172 __ it(pl); 5173 __ lsl(pl, o_h, low, temp); 5174 } 5175 // Shift the low part 5176 __ Lsl(o_l, low, o_l); 5177 } else if (op->IsShr()) { 5178 __ And(o_h, second_reg, kMaxLongShiftDistance); 5179 // Shift the low part 5180 __ Lsr(o_l, low, o_h); 5181 // Shift the high part and `or` what underflew on the low part 5182 __ Rsb(temp, o_h, Operand::From(kArmBitsPerWord)); 5183 __ Lsl(temp, high, temp); 5184 __ Orr(o_l, o_l, temp); 5185 // If the shift is > 32 bits, override the low part 5186 __ Subs(temp, o_h, Operand::From(kArmBitsPerWord)); 5187 { 5188 ExactAssemblyScope guard(GetVIXLAssembler(), 5189 2 * vixl32::kMaxInstructionSizeInBytes, 5190 CodeBufferCheckScope::kMaximumSize); 5191 __ it(pl); 5192 __ asr(pl, o_l, high, temp); 5193 } 5194 // Shift the high part 5195 __ Asr(o_h, high, o_h); 5196 } else { 5197 __ And(o_h, second_reg, kMaxLongShiftDistance); 5198 // same as Shr except we use `Lsr`s and not `Asr`s 5199 __ Lsr(o_l, low, o_h); 5200 __ Rsb(temp, o_h, Operand::From(kArmBitsPerWord)); 5201 __ Lsl(temp, high, temp); 5202 __ Orr(o_l, o_l, temp); 5203 __ Subs(temp, o_h, Operand::From(kArmBitsPerWord)); 5204 { 5205 ExactAssemblyScope guard(GetVIXLAssembler(), 5206 2 * vixl32::kMaxInstructionSizeInBytes, 5207 CodeBufferCheckScope::kMaximumSize); 5208 __ it(pl); 5209 __ lsr(pl, o_l, high, temp); 5210 } 5211 __ Lsr(o_h, high, o_h); 5212 } 5213 } else { 5214 // Register allocator doesn't create partial overlap. 5215 DCHECK(!o_l.Is(high)); 5216 DCHECK(!o_h.Is(low)); 5217 int32_t cst = Int32ConstantFrom(second); 5218 uint32_t shift_value = cst & kMaxLongShiftDistance; 5219 if (shift_value > 32) { 5220 if (op->IsShl()) { 5221 __ Lsl(o_h, low, shift_value - 32); 5222 __ Mov(o_l, 0); 5223 } else if (op->IsShr()) { 5224 __ Asr(o_l, high, shift_value - 32); 5225 __ Asr(o_h, high, 31); 5226 } else { 5227 __ Lsr(o_l, high, shift_value - 32); 5228 __ Mov(o_h, 0); 5229 } 5230 } else if (shift_value == 32) { 5231 if (op->IsShl()) { 5232 __ Mov(o_h, low); 5233 __ Mov(o_l, 0); 5234 } else if (op->IsShr()) { 5235 __ Mov(o_l, high); 5236 __ Asr(o_h, high, 31); 5237 } else { 5238 __ Mov(o_l, high); 5239 __ Mov(o_h, 0); 5240 } 5241 } else if (shift_value == 1) { 5242 if (op->IsShl()) { 5243 __ Lsls(o_l, low, 1); 5244 __ Adc(o_h, high, high); 5245 } else if (op->IsShr()) { 5246 __ Asrs(o_h, high, 1); 5247 __ Rrx(o_l, low); 5248 } else { 5249 __ Lsrs(o_h, high, 1); 5250 __ Rrx(o_l, low); 5251 } 5252 } else { 5253 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 5254 if (op->IsShl()) { 5255 __ Lsl(o_h, high, shift_value); 5256 __ Orr(o_h, o_h, Operand(low, ShiftType::LSR, 32 - shift_value)); 5257 __ Lsl(o_l, low, shift_value); 5258 } else if (op->IsShr()) { 5259 __ Lsr(o_l, low, shift_value); 5260 __ Orr(o_l, o_l, Operand(high, ShiftType::LSL, 32 - shift_value)); 5261 __ Asr(o_h, high, shift_value); 5262 } else { 5263 __ Lsr(o_l, low, shift_value); 5264 __ Orr(o_l, o_l, Operand(high, ShiftType::LSL, 32 - shift_value)); 5265 __ Lsr(o_h, high, shift_value); 5266 } 5267 } 5268 } 5269 break; 5270 } 5271 default: 5272 LOG(FATAL) << "Unexpected operation type " << type; 5273 UNREACHABLE(); 5274 } 5275 } 5276 5277 void LocationsBuilderARMVIXL::VisitShl(HShl* shl) { 5278 HandleShift(shl); 5279 } 5280 5281 void InstructionCodeGeneratorARMVIXL::VisitShl(HShl* shl) { 5282 HandleShift(shl); 5283 } 5284 5285 void LocationsBuilderARMVIXL::VisitShr(HShr* shr) { 5286 HandleShift(shr); 5287 } 5288 5289 void InstructionCodeGeneratorARMVIXL::VisitShr(HShr* shr) { 5290 HandleShift(shr); 5291 } 5292 5293 void LocationsBuilderARMVIXL::VisitUShr(HUShr* ushr) { 5294 HandleShift(ushr); 5295 } 5296 5297 void InstructionCodeGeneratorARMVIXL::VisitUShr(HUShr* ushr) { 5298 HandleShift(ushr); 5299 } 5300 5301 void LocationsBuilderARMVIXL::VisitNewInstance(HNewInstance* instruction) { 5302 LocationSummary* locations = 5303 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 5304 if (instruction->IsStringAlloc()) { 5305 locations->AddTemp(LocationFrom(kMethodRegister)); 5306 } else { 5307 InvokeRuntimeCallingConventionARMVIXL calling_convention; 5308 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 5309 } 5310 locations->SetOut(LocationFrom(r0)); 5311 } 5312 5313 void InstructionCodeGeneratorARMVIXL::VisitNewInstance(HNewInstance* instruction) { 5314 // Note: if heap poisoning is enabled, the entry point takes cares 5315 // of poisoning the reference. 5316 if (instruction->IsStringAlloc()) { 5317 // String is allocated through StringFactory. Call NewEmptyString entry point. 5318 vixl32::Register temp = RegisterFrom(instruction->GetLocations()->GetTemp(0)); 5319 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 5320 GetAssembler()->LoadFromOffset(kLoadWord, temp, tr, QUICK_ENTRY_POINT(pNewEmptyString)); 5321 GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, code_offset.Int32Value()); 5322 // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. 5323 ExactAssemblyScope aas(GetVIXLAssembler(), 5324 vixl32::k16BitT32InstructionSizeInBytes, 5325 CodeBufferCheckScope::kExactSize); 5326 __ blx(lr); 5327 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 5328 } else { 5329 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 5330 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 5331 } 5332 } 5333 5334 void LocationsBuilderARMVIXL::VisitNewArray(HNewArray* instruction) { 5335 LocationSummary* locations = 5336 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 5337 InvokeRuntimeCallingConventionARMVIXL calling_convention; 5338 locations->SetOut(LocationFrom(r0)); 5339 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 5340 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 5341 } 5342 5343 void InstructionCodeGeneratorARMVIXL::VisitNewArray(HNewArray* instruction) { 5344 // Note: if heap poisoning is enabled, the entry point takes cares 5345 // of poisoning the reference. 5346 QuickEntrypointEnum entrypoint = 5347 CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); 5348 codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); 5349 CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); 5350 DCHECK(!codegen_->IsLeafMethod()); 5351 } 5352 5353 void LocationsBuilderARMVIXL::VisitParameterValue(HParameterValue* instruction) { 5354 LocationSummary* locations = 5355 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5356 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 5357 if (location.IsStackSlot()) { 5358 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 5359 } else if (location.IsDoubleStackSlot()) { 5360 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 5361 } 5362 locations->SetOut(location); 5363 } 5364 5365 void InstructionCodeGeneratorARMVIXL::VisitParameterValue( 5366 HParameterValue* instruction ATTRIBUTE_UNUSED) { 5367 // Nothing to do, the parameter is already at its location. 5368 } 5369 5370 void LocationsBuilderARMVIXL::VisitCurrentMethod(HCurrentMethod* instruction) { 5371 LocationSummary* locations = 5372 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5373 locations->SetOut(LocationFrom(kMethodRegister)); 5374 } 5375 5376 void InstructionCodeGeneratorARMVIXL::VisitCurrentMethod( 5377 HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 5378 // Nothing to do, the method is already at its location. 5379 } 5380 5381 void LocationsBuilderARMVIXL::VisitNot(HNot* not_) { 5382 LocationSummary* locations = 5383 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 5384 locations->SetInAt(0, Location::RequiresRegister()); 5385 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5386 } 5387 5388 void InstructionCodeGeneratorARMVIXL::VisitNot(HNot* not_) { 5389 LocationSummary* locations = not_->GetLocations(); 5390 Location out = locations->Out(); 5391 Location in = locations->InAt(0); 5392 switch (not_->GetResultType()) { 5393 case Primitive::kPrimInt: 5394 __ Mvn(OutputRegister(not_), InputRegisterAt(not_, 0)); 5395 break; 5396 5397 case Primitive::kPrimLong: 5398 __ Mvn(LowRegisterFrom(out), LowRegisterFrom(in)); 5399 __ Mvn(HighRegisterFrom(out), HighRegisterFrom(in)); 5400 break; 5401 5402 default: 5403 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 5404 } 5405 } 5406 5407 void LocationsBuilderARMVIXL::VisitBooleanNot(HBooleanNot* bool_not) { 5408 LocationSummary* locations = 5409 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 5410 locations->SetInAt(0, Location::RequiresRegister()); 5411 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5412 } 5413 5414 void InstructionCodeGeneratorARMVIXL::VisitBooleanNot(HBooleanNot* bool_not) { 5415 __ Eor(OutputRegister(bool_not), InputRegister(bool_not), 1); 5416 } 5417 5418 void LocationsBuilderARMVIXL::VisitCompare(HCompare* compare) { 5419 LocationSummary* locations = 5420 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 5421 switch (compare->InputAt(0)->GetType()) { 5422 case Primitive::kPrimBoolean: 5423 case Primitive::kPrimByte: 5424 case Primitive::kPrimShort: 5425 case Primitive::kPrimChar: 5426 case Primitive::kPrimInt: 5427 case Primitive::kPrimLong: { 5428 locations->SetInAt(0, Location::RequiresRegister()); 5429 locations->SetInAt(1, Location::RequiresRegister()); 5430 // Output overlaps because it is written before doing the low comparison. 5431 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 5432 break; 5433 } 5434 case Primitive::kPrimFloat: 5435 case Primitive::kPrimDouble: { 5436 locations->SetInAt(0, Location::RequiresFpuRegister()); 5437 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 5438 locations->SetOut(Location::RequiresRegister()); 5439 break; 5440 } 5441 default: 5442 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 5443 } 5444 } 5445 5446 void InstructionCodeGeneratorARMVIXL::VisitCompare(HCompare* compare) { 5447 LocationSummary* locations = compare->GetLocations(); 5448 vixl32::Register out = OutputRegister(compare); 5449 Location left = locations->InAt(0); 5450 Location right = locations->InAt(1); 5451 5452 vixl32::Label less, greater, done; 5453 vixl32::Label* final_label = codegen_->GetFinalLabel(compare, &done); 5454 Primitive::Type type = compare->InputAt(0)->GetType(); 5455 vixl32::Condition less_cond = vixl32::Condition(kNone); 5456 switch (type) { 5457 case Primitive::kPrimBoolean: 5458 case Primitive::kPrimByte: 5459 case Primitive::kPrimShort: 5460 case Primitive::kPrimChar: 5461 case Primitive::kPrimInt: { 5462 // Emit move to `out` before the `Cmp`, as `Mov` might affect the status flags. 5463 __ Mov(out, 0); 5464 __ Cmp(RegisterFrom(left), RegisterFrom(right)); // Signed compare. 5465 less_cond = lt; 5466 break; 5467 } 5468 case Primitive::kPrimLong: { 5469 __ Cmp(HighRegisterFrom(left), HighRegisterFrom(right)); // Signed compare. 5470 __ B(lt, &less, /* far_target */ false); 5471 __ B(gt, &greater, /* far_target */ false); 5472 // Emit move to `out` before the last `Cmp`, as `Mov` might affect the status flags. 5473 __ Mov(out, 0); 5474 __ Cmp(LowRegisterFrom(left), LowRegisterFrom(right)); // Unsigned compare. 5475 less_cond = lo; 5476 break; 5477 } 5478 case Primitive::kPrimFloat: 5479 case Primitive::kPrimDouble: { 5480 __ Mov(out, 0); 5481 GenerateVcmp(compare, codegen_); 5482 // To branch on the FP compare result we transfer FPSCR to APSR (encoded as PC in VMRS). 5483 __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); 5484 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 5485 break; 5486 } 5487 default: 5488 LOG(FATAL) << "Unexpected compare type " << type; 5489 UNREACHABLE(); 5490 } 5491 5492 __ B(eq, final_label, /* far_target */ false); 5493 __ B(less_cond, &less, /* far_target */ false); 5494 5495 __ Bind(&greater); 5496 __ Mov(out, 1); 5497 __ B(final_label); 5498 5499 __ Bind(&less); 5500 __ Mov(out, -1); 5501 5502 if (done.IsReferenced()) { 5503 __ Bind(&done); 5504 } 5505 } 5506 5507 void LocationsBuilderARMVIXL::VisitPhi(HPhi* instruction) { 5508 LocationSummary* locations = 5509 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5510 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 5511 locations->SetInAt(i, Location::Any()); 5512 } 5513 locations->SetOut(Location::Any()); 5514 } 5515 5516 void InstructionCodeGeneratorARMVIXL::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 5517 LOG(FATAL) << "Unreachable"; 5518 } 5519 5520 void CodeGeneratorARMVIXL::GenerateMemoryBarrier(MemBarrierKind kind) { 5521 // TODO (ported from quick): revisit ARM barrier kinds. 5522 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 5523 switch (kind) { 5524 case MemBarrierKind::kAnyStore: 5525 case MemBarrierKind::kLoadAny: 5526 case MemBarrierKind::kAnyAny: { 5527 flavor = DmbOptions::ISH; 5528 break; 5529 } 5530 case MemBarrierKind::kStoreStore: { 5531 flavor = DmbOptions::ISHST; 5532 break; 5533 } 5534 default: 5535 LOG(FATAL) << "Unexpected memory barrier " << kind; 5536 } 5537 __ Dmb(flavor); 5538 } 5539 5540 void InstructionCodeGeneratorARMVIXL::GenerateWideAtomicLoad(vixl32::Register addr, 5541 uint32_t offset, 5542 vixl32::Register out_lo, 5543 vixl32::Register out_hi) { 5544 UseScratchRegisterScope temps(GetVIXLAssembler()); 5545 if (offset != 0) { 5546 vixl32::Register temp = temps.Acquire(); 5547 __ Add(temp, addr, offset); 5548 addr = temp; 5549 } 5550 __ Ldrexd(out_lo, out_hi, MemOperand(addr)); 5551 } 5552 5553 void InstructionCodeGeneratorARMVIXL::GenerateWideAtomicStore(vixl32::Register addr, 5554 uint32_t offset, 5555 vixl32::Register value_lo, 5556 vixl32::Register value_hi, 5557 vixl32::Register temp1, 5558 vixl32::Register temp2, 5559 HInstruction* instruction) { 5560 UseScratchRegisterScope temps(GetVIXLAssembler()); 5561 vixl32::Label fail; 5562 if (offset != 0) { 5563 vixl32::Register temp = temps.Acquire(); 5564 __ Add(temp, addr, offset); 5565 addr = temp; 5566 } 5567 __ Bind(&fail); 5568 { 5569 // Ensure the pc position is recorded immediately after the `ldrexd` instruction. 5570 ExactAssemblyScope aas(GetVIXLAssembler(), 5571 vixl32::kMaxInstructionSizeInBytes, 5572 CodeBufferCheckScope::kMaximumSize); 5573 // We need a load followed by store. (The address used in a STREX instruction must 5574 // be the same as the address in the most recently executed LDREX instruction.) 5575 __ ldrexd(temp1, temp2, MemOperand(addr)); 5576 codegen_->MaybeRecordImplicitNullCheck(instruction); 5577 } 5578 __ Strexd(temp1, value_lo, value_hi, MemOperand(addr)); 5579 __ CompareAndBranchIfNonZero(temp1, &fail); 5580 } 5581 5582 void LocationsBuilderARMVIXL::HandleFieldSet( 5583 HInstruction* instruction, const FieldInfo& field_info) { 5584 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5585 5586 LocationSummary* locations = 5587 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5588 locations->SetInAt(0, Location::RequiresRegister()); 5589 5590 Primitive::Type field_type = field_info.GetFieldType(); 5591 if (Primitive::IsFloatingPointType(field_type)) { 5592 locations->SetInAt(1, Location::RequiresFpuRegister()); 5593 } else { 5594 locations->SetInAt(1, Location::RequiresRegister()); 5595 } 5596 5597 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 5598 bool generate_volatile = field_info.IsVolatile() 5599 && is_wide 5600 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5601 bool needs_write_barrier = 5602 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5603 // Temporary registers for the write barrier. 5604 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 5605 if (needs_write_barrier) { 5606 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 5607 locations->AddTemp(Location::RequiresRegister()); 5608 } else if (generate_volatile) { 5609 // ARM encoding have some additional constraints for ldrexd/strexd: 5610 // - registers need to be consecutive 5611 // - the first register should be even but not R14. 5612 // We don't test for ARM yet, and the assertion makes sure that we 5613 // revisit this if we ever enable ARM encoding. 5614 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5615 5616 locations->AddTemp(Location::RequiresRegister()); 5617 locations->AddTemp(Location::RequiresRegister()); 5618 if (field_type == Primitive::kPrimDouble) { 5619 // For doubles we need two more registers to copy the value. 5620 locations->AddTemp(LocationFrom(r2)); 5621 locations->AddTemp(LocationFrom(r3)); 5622 } 5623 } 5624 } 5625 5626 void InstructionCodeGeneratorARMVIXL::HandleFieldSet(HInstruction* instruction, 5627 const FieldInfo& field_info, 5628 bool value_can_be_null) { 5629 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5630 5631 LocationSummary* locations = instruction->GetLocations(); 5632 vixl32::Register base = InputRegisterAt(instruction, 0); 5633 Location value = locations->InAt(1); 5634 5635 bool is_volatile = field_info.IsVolatile(); 5636 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5637 Primitive::Type field_type = field_info.GetFieldType(); 5638 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5639 bool needs_write_barrier = 5640 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5641 5642 if (is_volatile) { 5643 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 5644 } 5645 5646 switch (field_type) { 5647 case Primitive::kPrimBoolean: 5648 case Primitive::kPrimByte: { 5649 GetAssembler()->StoreToOffset(kStoreByte, RegisterFrom(value), base, offset); 5650 break; 5651 } 5652 5653 case Primitive::kPrimShort: 5654 case Primitive::kPrimChar: { 5655 GetAssembler()->StoreToOffset(kStoreHalfword, RegisterFrom(value), base, offset); 5656 break; 5657 } 5658 5659 case Primitive::kPrimInt: 5660 case Primitive::kPrimNot: { 5661 if (kPoisonHeapReferences && needs_write_barrier) { 5662 // Note that in the case where `value` is a null reference, 5663 // we do not enter this block, as a null reference does not 5664 // need poisoning. 5665 DCHECK_EQ(field_type, Primitive::kPrimNot); 5666 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 5667 __ Mov(temp, RegisterFrom(value)); 5668 GetAssembler()->PoisonHeapReference(temp); 5669 GetAssembler()->StoreToOffset(kStoreWord, temp, base, offset); 5670 } else { 5671 GetAssembler()->StoreToOffset(kStoreWord, RegisterFrom(value), base, offset); 5672 } 5673 break; 5674 } 5675 5676 case Primitive::kPrimLong: { 5677 if (is_volatile && !atomic_ldrd_strd) { 5678 GenerateWideAtomicStore(base, 5679 offset, 5680 LowRegisterFrom(value), 5681 HighRegisterFrom(value), 5682 RegisterFrom(locations->GetTemp(0)), 5683 RegisterFrom(locations->GetTemp(1)), 5684 instruction); 5685 } else { 5686 GetAssembler()->StoreToOffset(kStoreWordPair, LowRegisterFrom(value), base, offset); 5687 codegen_->MaybeRecordImplicitNullCheck(instruction); 5688 } 5689 break; 5690 } 5691 5692 case Primitive::kPrimFloat: { 5693 GetAssembler()->StoreSToOffset(SRegisterFrom(value), base, offset); 5694 break; 5695 } 5696 5697 case Primitive::kPrimDouble: { 5698 vixl32::DRegister value_reg = DRegisterFrom(value); 5699 if (is_volatile && !atomic_ldrd_strd) { 5700 vixl32::Register value_reg_lo = RegisterFrom(locations->GetTemp(0)); 5701 vixl32::Register value_reg_hi = RegisterFrom(locations->GetTemp(1)); 5702 5703 __ Vmov(value_reg_lo, value_reg_hi, value_reg); 5704 5705 GenerateWideAtomicStore(base, 5706 offset, 5707 value_reg_lo, 5708 value_reg_hi, 5709 RegisterFrom(locations->GetTemp(2)), 5710 RegisterFrom(locations->GetTemp(3)), 5711 instruction); 5712 } else { 5713 GetAssembler()->StoreDToOffset(value_reg, base, offset); 5714 codegen_->MaybeRecordImplicitNullCheck(instruction); 5715 } 5716 break; 5717 } 5718 5719 case Primitive::kPrimVoid: 5720 LOG(FATAL) << "Unreachable type " << field_type; 5721 UNREACHABLE(); 5722 } 5723 5724 // Longs and doubles are handled in the switch. 5725 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 5726 // TODO(VIXL): Here and for other calls to `MaybeRecordImplicitNullCheck` in this method, we 5727 // should use a scope and the assembler to emit the store instruction to guarantee that we 5728 // record the pc at the correct position. But the `Assembler` does not automatically handle 5729 // unencodable offsets. Practically, everything is fine because the helper and VIXL, at the time 5730 // of writing, do generate the store instruction last. 5731 codegen_->MaybeRecordImplicitNullCheck(instruction); 5732 } 5733 5734 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 5735 vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); 5736 vixl32::Register card = RegisterFrom(locations->GetTemp(1)); 5737 codegen_->MarkGCCard(temp, card, base, RegisterFrom(value), value_can_be_null); 5738 } 5739 5740 if (is_volatile) { 5741 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 5742 } 5743 } 5744 5745 void LocationsBuilderARMVIXL::HandleFieldGet(HInstruction* instruction, 5746 const FieldInfo& field_info) { 5747 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5748 5749 bool object_field_get_with_read_barrier = 5750 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 5751 LocationSummary* locations = 5752 new (GetGraph()->GetArena()) LocationSummary(instruction, 5753 object_field_get_with_read_barrier ? 5754 LocationSummary::kCallOnSlowPath : 5755 LocationSummary::kNoCall); 5756 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5757 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5758 } 5759 locations->SetInAt(0, Location::RequiresRegister()); 5760 5761 bool volatile_for_double = field_info.IsVolatile() 5762 && (field_info.GetFieldType() == Primitive::kPrimDouble) 5763 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5764 // The output overlaps in case of volatile long: we don't want the 5765 // code generated by GenerateWideAtomicLoad to overwrite the 5766 // object's location. Likewise, in the case of an object field get 5767 // with read barriers enabled, we do not want the load to overwrite 5768 // the object's location, as we need it to emit the read barrier. 5769 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 5770 object_field_get_with_read_barrier; 5771 5772 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5773 locations->SetOut(Location::RequiresFpuRegister()); 5774 } else { 5775 locations->SetOut(Location::RequiresRegister(), 5776 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 5777 } 5778 if (volatile_for_double) { 5779 // ARM encoding have some additional constraints for ldrexd/strexd: 5780 // - registers need to be consecutive 5781 // - the first register should be even but not R14. 5782 // We don't test for ARM yet, and the assertion makes sure that we 5783 // revisit this if we ever enable ARM encoding. 5784 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5785 locations->AddTemp(Location::RequiresRegister()); 5786 locations->AddTemp(Location::RequiresRegister()); 5787 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5788 // We need a temporary register for the read barrier marking slow 5789 // path in CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier. 5790 if (kBakerReadBarrierLinkTimeThunksEnableForFields && 5791 !Runtime::Current()->UseJitCompilation()) { 5792 // If link-time thunks for the Baker read barrier are enabled, for AOT 5793 // loads we need a temporary only if the offset is too big. 5794 if (field_info.GetFieldOffset().Uint32Value() >= kReferenceLoadMinFarOffset) { 5795 locations->AddTemp(Location::RequiresRegister()); 5796 } 5797 // And we always need the reserved entrypoint register. 5798 locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister.GetCode())); 5799 } else { 5800 locations->AddTemp(Location::RequiresRegister()); 5801 } 5802 } 5803 } 5804 5805 Location LocationsBuilderARMVIXL::ArithmeticZeroOrFpuRegister(HInstruction* input) { 5806 DCHECK(Primitive::IsFloatingPointType(input->GetType())) << input->GetType(); 5807 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 5808 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 5809 return Location::ConstantLocation(input->AsConstant()); 5810 } else { 5811 return Location::RequiresFpuRegister(); 5812 } 5813 } 5814 5815 Location LocationsBuilderARMVIXL::ArmEncodableConstantOrRegister(HInstruction* constant, 5816 Opcode opcode) { 5817 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 5818 if (constant->IsConstant() && 5819 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 5820 return Location::ConstantLocation(constant->AsConstant()); 5821 } 5822 return Location::RequiresRegister(); 5823 } 5824 5825 bool LocationsBuilderARMVIXL::CanEncodeConstantAsImmediate(HConstant* input_cst, 5826 Opcode opcode) { 5827 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 5828 if (Primitive::Is64BitType(input_cst->GetType())) { 5829 Opcode high_opcode = opcode; 5830 SetCc low_set_cc = kCcDontCare; 5831 switch (opcode) { 5832 case SUB: 5833 // Flip the operation to an ADD. 5834 value = -value; 5835 opcode = ADD; 5836 FALLTHROUGH_INTENDED; 5837 case ADD: 5838 if (Low32Bits(value) == 0u) { 5839 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 5840 } 5841 high_opcode = ADC; 5842 low_set_cc = kCcSet; 5843 break; 5844 default: 5845 break; 5846 } 5847 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 5848 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 5849 } else { 5850 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 5851 } 5852 } 5853 5854 // TODO(VIXL): Replace art::arm::SetCc` with `vixl32::FlagsUpdate after flags set optimization 5855 // enabled. 5856 bool LocationsBuilderARMVIXL::CanEncodeConstantAsImmediate(uint32_t value, 5857 Opcode opcode, 5858 SetCc set_cc) { 5859 ArmVIXLAssembler* assembler = codegen_->GetAssembler(); 5860 if (assembler->ShifterOperandCanHold(opcode, value, set_cc)) { 5861 return true; 5862 } 5863 Opcode neg_opcode = kNoOperand; 5864 uint32_t neg_value = 0; 5865 switch (opcode) { 5866 case AND: neg_opcode = BIC; neg_value = ~value; break; 5867 case ORR: neg_opcode = ORN; neg_value = ~value; break; 5868 case ADD: neg_opcode = SUB; neg_value = -value; break; 5869 case ADC: neg_opcode = SBC; neg_value = ~value; break; 5870 case SUB: neg_opcode = ADD; neg_value = -value; break; 5871 case SBC: neg_opcode = ADC; neg_value = ~value; break; 5872 case MOV: neg_opcode = MVN; neg_value = ~value; break; 5873 default: 5874 return false; 5875 } 5876 5877 if (assembler->ShifterOperandCanHold(neg_opcode, neg_value, set_cc)) { 5878 return true; 5879 } 5880 5881 return opcode == AND && IsPowerOfTwo(value + 1); 5882 } 5883 5884 void InstructionCodeGeneratorARMVIXL::HandleFieldGet(HInstruction* instruction, 5885 const FieldInfo& field_info) { 5886 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5887 5888 LocationSummary* locations = instruction->GetLocations(); 5889 vixl32::Register base = InputRegisterAt(instruction, 0); 5890 Location out = locations->Out(); 5891 bool is_volatile = field_info.IsVolatile(); 5892 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5893 Primitive::Type field_type = field_info.GetFieldType(); 5894 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5895 5896 switch (field_type) { 5897 case Primitive::kPrimBoolean: 5898 GetAssembler()->LoadFromOffset(kLoadUnsignedByte, RegisterFrom(out), base, offset); 5899 break; 5900 5901 case Primitive::kPrimByte: 5902 GetAssembler()->LoadFromOffset(kLoadSignedByte, RegisterFrom(out), base, offset); 5903 break; 5904 5905 case Primitive::kPrimShort: 5906 GetAssembler()->LoadFromOffset(kLoadSignedHalfword, RegisterFrom(out), base, offset); 5907 break; 5908 5909 case Primitive::kPrimChar: 5910 GetAssembler()->LoadFromOffset(kLoadUnsignedHalfword, RegisterFrom(out), base, offset); 5911 break; 5912 5913 case Primitive::kPrimInt: 5914 GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(out), base, offset); 5915 break; 5916 5917 case Primitive::kPrimNot: { 5918 // /* HeapReference<Object> */ out = *(base + offset) 5919 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5920 Location temp_loc = locations->GetTemp(0); 5921 // Note that a potential implicit null check is handled in this 5922 // CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier call. 5923 codegen_->GenerateFieldLoadWithBakerReadBarrier( 5924 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 5925 if (is_volatile) { 5926 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5927 } 5928 } else { 5929 GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(out), base, offset); 5930 codegen_->MaybeRecordImplicitNullCheck(instruction); 5931 if (is_volatile) { 5932 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5933 } 5934 // If read barriers are enabled, emit read barriers other than 5935 // Baker's using a slow path (and also unpoison the loaded 5936 // reference, if heap poisoning is enabled). 5937 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, locations->InAt(0), offset); 5938 } 5939 break; 5940 } 5941 5942 case Primitive::kPrimLong: 5943 if (is_volatile && !atomic_ldrd_strd) { 5944 GenerateWideAtomicLoad(base, offset, LowRegisterFrom(out), HighRegisterFrom(out)); 5945 } else { 5946 GetAssembler()->LoadFromOffset(kLoadWordPair, LowRegisterFrom(out), base, offset); 5947 } 5948 break; 5949 5950 case Primitive::kPrimFloat: 5951 GetAssembler()->LoadSFromOffset(SRegisterFrom(out), base, offset); 5952 break; 5953 5954 case Primitive::kPrimDouble: { 5955 vixl32::DRegister out_dreg = DRegisterFrom(out); 5956 if (is_volatile && !atomic_ldrd_strd) { 5957 vixl32::Register lo = RegisterFrom(locations->GetTemp(0)); 5958 vixl32::Register hi = RegisterFrom(locations->GetTemp(1)); 5959 GenerateWideAtomicLoad(base, offset, lo, hi); 5960 // TODO(VIXL): Do we need to be immediately after the ldrexd instruction? If so we need a 5961 // scope. 5962 codegen_->MaybeRecordImplicitNullCheck(instruction); 5963 __ Vmov(out_dreg, lo, hi); 5964 } else { 5965 GetAssembler()->LoadDFromOffset(out_dreg, base, offset); 5966 codegen_->MaybeRecordImplicitNullCheck(instruction); 5967 } 5968 break; 5969 } 5970 5971 case Primitive::kPrimVoid: 5972 LOG(FATAL) << "Unreachable type " << field_type; 5973 UNREACHABLE(); 5974 } 5975 5976 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 5977 // Potential implicit null checks, in the case of reference or 5978 // double fields, are handled in the previous switch statement. 5979 } else { 5980 // Address cases other than reference and double that may require an implicit null check. 5981 // TODO(VIXL): Here and for other calls to `MaybeRecordImplicitNullCheck` in this method, we 5982 // should use a scope and the assembler to emit the load instruction to guarantee that we 5983 // record the pc at the correct position. But the `Assembler` does not automatically handle 5984 // unencodable offsets. Practically, everything is fine because the helper and VIXL, at the time 5985 // of writing, do generate the store instruction last. 5986 codegen_->MaybeRecordImplicitNullCheck(instruction); 5987 } 5988 5989 if (is_volatile) { 5990 if (field_type == Primitive::kPrimNot) { 5991 // Memory barriers, in the case of references, are also handled 5992 // in the previous switch statement. 5993 } else { 5994 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5995 } 5996 } 5997 } 5998 5999 void LocationsBuilderARMVIXL::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 6000 HandleFieldSet(instruction, instruction->GetFieldInfo()); 6001 } 6002 6003 void InstructionCodeGeneratorARMVIXL::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 6004 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 6005 } 6006 6007 void LocationsBuilderARMVIXL::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 6008 HandleFieldGet(instruction, instruction->GetFieldInfo()); 6009 } 6010 6011 void InstructionCodeGeneratorARMVIXL::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 6012 HandleFieldGet(instruction, instruction->GetFieldInfo()); 6013 } 6014 6015 void LocationsBuilderARMVIXL::VisitStaticFieldGet(HStaticFieldGet* instruction) { 6016 HandleFieldGet(instruction, instruction->GetFieldInfo()); 6017 } 6018 6019 void InstructionCodeGeneratorARMVIXL::VisitStaticFieldGet(HStaticFieldGet* instruction) { 6020 HandleFieldGet(instruction, instruction->GetFieldInfo()); 6021 } 6022 6023 void LocationsBuilderARMVIXL::VisitStaticFieldSet(HStaticFieldSet* instruction) { 6024 HandleFieldSet(instruction, instruction->GetFieldInfo()); 6025 } 6026 6027 void InstructionCodeGeneratorARMVIXL::VisitStaticFieldSet(HStaticFieldSet* instruction) { 6028 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 6029 } 6030 6031 void LocationsBuilderARMVIXL::VisitUnresolvedInstanceFieldGet( 6032 HUnresolvedInstanceFieldGet* instruction) { 6033 FieldAccessCallingConventionARMVIXL calling_convention; 6034 codegen_->CreateUnresolvedFieldLocationSummary( 6035 instruction, instruction->GetFieldType(), calling_convention); 6036 } 6037 6038 void InstructionCodeGeneratorARMVIXL::VisitUnresolvedInstanceFieldGet( 6039 HUnresolvedInstanceFieldGet* instruction) { 6040 FieldAccessCallingConventionARMVIXL calling_convention; 6041 codegen_->GenerateUnresolvedFieldAccess(instruction, 6042 instruction->GetFieldType(), 6043 instruction->GetFieldIndex(), 6044 instruction->GetDexPc(), 6045 calling_convention); 6046 } 6047 6048 void LocationsBuilderARMVIXL::VisitUnresolvedInstanceFieldSet( 6049 HUnresolvedInstanceFieldSet* instruction) { 6050 FieldAccessCallingConventionARMVIXL calling_convention; 6051 codegen_->CreateUnresolvedFieldLocationSummary( 6052 instruction, instruction->GetFieldType(), calling_convention); 6053 } 6054 6055 void InstructionCodeGeneratorARMVIXL::VisitUnresolvedInstanceFieldSet( 6056 HUnresolvedInstanceFieldSet* instruction) { 6057 FieldAccessCallingConventionARMVIXL calling_convention; 6058 codegen_->GenerateUnresolvedFieldAccess(instruction, 6059 instruction->GetFieldType(), 6060 instruction->GetFieldIndex(), 6061 instruction->GetDexPc(), 6062 calling_convention); 6063 } 6064 6065 void LocationsBuilderARMVIXL::VisitUnresolvedStaticFieldGet( 6066 HUnresolvedStaticFieldGet* instruction) { 6067 FieldAccessCallingConventionARMVIXL calling_convention; 6068 codegen_->CreateUnresolvedFieldLocationSummary( 6069 instruction, instruction->GetFieldType(), calling_convention); 6070 } 6071 6072 void InstructionCodeGeneratorARMVIXL::VisitUnresolvedStaticFieldGet( 6073 HUnresolvedStaticFieldGet* instruction) { 6074 FieldAccessCallingConventionARMVIXL calling_convention; 6075 codegen_->GenerateUnresolvedFieldAccess(instruction, 6076 instruction->GetFieldType(), 6077 instruction->GetFieldIndex(), 6078 instruction->GetDexPc(), 6079 calling_convention); 6080 } 6081 6082 void LocationsBuilderARMVIXL::VisitUnresolvedStaticFieldSet( 6083 HUnresolvedStaticFieldSet* instruction) { 6084 FieldAccessCallingConventionARMVIXL calling_convention; 6085 codegen_->CreateUnresolvedFieldLocationSummary( 6086 instruction, instruction->GetFieldType(), calling_convention); 6087 } 6088 6089 void InstructionCodeGeneratorARMVIXL::VisitUnresolvedStaticFieldSet( 6090 HUnresolvedStaticFieldSet* instruction) { 6091 FieldAccessCallingConventionARMVIXL calling_convention; 6092 codegen_->GenerateUnresolvedFieldAccess(instruction, 6093 instruction->GetFieldType(), 6094 instruction->GetFieldIndex(), 6095 instruction->GetDexPc(), 6096 calling_convention); 6097 } 6098 6099 void LocationsBuilderARMVIXL::VisitNullCheck(HNullCheck* instruction) { 6100 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 6101 locations->SetInAt(0, Location::RequiresRegister()); 6102 } 6103 6104 void CodeGeneratorARMVIXL::GenerateImplicitNullCheck(HNullCheck* instruction) { 6105 if (CanMoveNullCheckToUser(instruction)) { 6106 return; 6107 } 6108 6109 UseScratchRegisterScope temps(GetVIXLAssembler()); 6110 // Ensure the pc position is recorded immediately after the `ldr` instruction. 6111 ExactAssemblyScope aas(GetVIXLAssembler(), 6112 vixl32::kMaxInstructionSizeInBytes, 6113 CodeBufferCheckScope::kMaximumSize); 6114 __ ldr(temps.Acquire(), MemOperand(InputRegisterAt(instruction, 0))); 6115 RecordPcInfo(instruction, instruction->GetDexPc()); 6116 } 6117 6118 void CodeGeneratorARMVIXL::GenerateExplicitNullCheck(HNullCheck* instruction) { 6119 NullCheckSlowPathARMVIXL* slow_path = 6120 new (GetGraph()->GetArena()) NullCheckSlowPathARMVIXL(instruction); 6121 AddSlowPath(slow_path); 6122 __ CompareAndBranchIfZero(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel()); 6123 } 6124 6125 void InstructionCodeGeneratorARMVIXL::VisitNullCheck(HNullCheck* instruction) { 6126 codegen_->GenerateNullCheck(instruction); 6127 } 6128 6129 static LoadOperandType GetLoadOperandType(Primitive::Type type) { 6130 switch (type) { 6131 case Primitive::kPrimNot: 6132 return kLoadWord; 6133 case Primitive::kPrimBoolean: 6134 return kLoadUnsignedByte; 6135 case Primitive::kPrimByte: 6136 return kLoadSignedByte; 6137 case Primitive::kPrimChar: 6138 return kLoadUnsignedHalfword; 6139 case Primitive::kPrimShort: 6140 return kLoadSignedHalfword; 6141 case Primitive::kPrimInt: 6142 return kLoadWord; 6143 case Primitive::kPrimLong: 6144 return kLoadWordPair; 6145 case Primitive::kPrimFloat: 6146 return kLoadSWord; 6147 case Primitive::kPrimDouble: 6148 return kLoadDWord; 6149 default: 6150 LOG(FATAL) << "Unreachable type " << type; 6151 UNREACHABLE(); 6152 } 6153 } 6154 6155 static StoreOperandType GetStoreOperandType(Primitive::Type type) { 6156 switch (type) { 6157 case Primitive::kPrimNot: 6158 return kStoreWord; 6159 case Primitive::kPrimBoolean: 6160 case Primitive::kPrimByte: 6161 return kStoreByte; 6162 case Primitive::kPrimChar: 6163 case Primitive::kPrimShort: 6164 return kStoreHalfword; 6165 case Primitive::kPrimInt: 6166 return kStoreWord; 6167 case Primitive::kPrimLong: 6168 return kStoreWordPair; 6169 case Primitive::kPrimFloat: 6170 return kStoreSWord; 6171 case Primitive::kPrimDouble: 6172 return kStoreDWord; 6173 default: 6174 LOG(FATAL) << "Unreachable type " << type; 6175 UNREACHABLE(); 6176 } 6177 } 6178 6179 void CodeGeneratorARMVIXL::LoadFromShiftedRegOffset(Primitive::Type type, 6180 Location out_loc, 6181 vixl32::Register base, 6182 vixl32::Register reg_index, 6183 vixl32::Condition cond) { 6184 uint32_t shift_count = Primitive::ComponentSizeShift(type); 6185 MemOperand mem_address(base, reg_index, vixl32::LSL, shift_count); 6186 6187 switch (type) { 6188 case Primitive::kPrimByte: 6189 __ Ldrsb(cond, RegisterFrom(out_loc), mem_address); 6190 break; 6191 case Primitive::kPrimBoolean: 6192 __ Ldrb(cond, RegisterFrom(out_loc), mem_address); 6193 break; 6194 case Primitive::kPrimShort: 6195 __ Ldrsh(cond, RegisterFrom(out_loc), mem_address); 6196 break; 6197 case Primitive::kPrimChar: 6198 __ Ldrh(cond, RegisterFrom(out_loc), mem_address); 6199 break; 6200 case Primitive::kPrimNot: 6201 case Primitive::kPrimInt: 6202 __ Ldr(cond, RegisterFrom(out_loc), mem_address); 6203 break; 6204 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 6205 case Primitive::kPrimLong: 6206 case Primitive::kPrimFloat: 6207 case Primitive::kPrimDouble: 6208 default: 6209 LOG(FATAL) << "Unreachable type " << type; 6210 UNREACHABLE(); 6211 } 6212 } 6213 6214 void CodeGeneratorARMVIXL::StoreToShiftedRegOffset(Primitive::Type type, 6215 Location loc, 6216 vixl32::Register base, 6217 vixl32::Register reg_index, 6218 vixl32::Condition cond) { 6219 uint32_t shift_count = Primitive::ComponentSizeShift(type); 6220 MemOperand mem_address(base, reg_index, vixl32::LSL, shift_count); 6221 6222 switch (type) { 6223 case Primitive::kPrimByte: 6224 case Primitive::kPrimBoolean: 6225 __ Strb(cond, RegisterFrom(loc), mem_address); 6226 break; 6227 case Primitive::kPrimShort: 6228 case Primitive::kPrimChar: 6229 __ Strh(cond, RegisterFrom(loc), mem_address); 6230 break; 6231 case Primitive::kPrimNot: 6232 case Primitive::kPrimInt: 6233 __ Str(cond, RegisterFrom(loc), mem_address); 6234 break; 6235 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 6236 case Primitive::kPrimLong: 6237 case Primitive::kPrimFloat: 6238 case Primitive::kPrimDouble: 6239 default: 6240 LOG(FATAL) << "Unreachable type " << type; 6241 UNREACHABLE(); 6242 } 6243 } 6244 6245 void LocationsBuilderARMVIXL::VisitArrayGet(HArrayGet* instruction) { 6246 bool object_array_get_with_read_barrier = 6247 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 6248 LocationSummary* locations = 6249 new (GetGraph()->GetArena()) LocationSummary(instruction, 6250 object_array_get_with_read_barrier ? 6251 LocationSummary::kCallOnSlowPath : 6252 LocationSummary::kNoCall); 6253 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 6254 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6255 } 6256 locations->SetInAt(0, Location::RequiresRegister()); 6257 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 6258 if (Primitive::IsFloatingPointType(instruction->GetType())) { 6259 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 6260 } else { 6261 // The output overlaps in the case of an object array get with 6262 // read barriers enabled: we do not want the move to overwrite the 6263 // array's location, as we need it to emit the read barrier. 6264 locations->SetOut( 6265 Location::RequiresRegister(), 6266 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 6267 } 6268 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 6269 // We need a temporary register for the read barrier marking slow 6270 // path in CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier. 6271 if (kBakerReadBarrierLinkTimeThunksEnableForFields && 6272 !Runtime::Current()->UseJitCompilation() && 6273 instruction->GetIndex()->IsConstant()) { 6274 // Array loads with constant index are treated as field loads. 6275 // If link-time thunks for the Baker read barrier are enabled, for AOT 6276 // constant index loads we need a temporary only if the offset is too big. 6277 uint32_t offset = CodeGenerator::GetArrayDataOffset(instruction); 6278 uint32_t index = instruction->GetIndex()->AsIntConstant()->GetValue(); 6279 offset += index << Primitive::ComponentSizeShift(Primitive::kPrimNot); 6280 if (offset >= kReferenceLoadMinFarOffset) { 6281 locations->AddTemp(Location::RequiresRegister()); 6282 } 6283 // And we always need the reserved entrypoint register. 6284 locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister.GetCode())); 6285 } else if (kBakerReadBarrierLinkTimeThunksEnableForArrays && 6286 !Runtime::Current()->UseJitCompilation() && 6287 !instruction->GetIndex()->IsConstant()) { 6288 // We need a non-scratch temporary for the array data pointer. 6289 locations->AddTemp(Location::RequiresRegister()); 6290 // And we always need the reserved entrypoint register. 6291 locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister.GetCode())); 6292 } else { 6293 locations->AddTemp(Location::RequiresRegister()); 6294 } 6295 } else if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { 6296 // Also need a temporary for String compression feature. 6297 locations->AddTemp(Location::RequiresRegister()); 6298 } 6299 } 6300 6301 void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { 6302 LocationSummary* locations = instruction->GetLocations(); 6303 Location obj_loc = locations->InAt(0); 6304 vixl32::Register obj = InputRegisterAt(instruction, 0); 6305 Location index = locations->InAt(1); 6306 Location out_loc = locations->Out(); 6307 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 6308 Primitive::Type type = instruction->GetType(); 6309 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 6310 instruction->IsStringCharAt(); 6311 HInstruction* array_instr = instruction->GetArray(); 6312 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 6313 6314 switch (type) { 6315 case Primitive::kPrimBoolean: 6316 case Primitive::kPrimByte: 6317 case Primitive::kPrimShort: 6318 case Primitive::kPrimChar: 6319 case Primitive::kPrimInt: { 6320 vixl32::Register length; 6321 if (maybe_compressed_char_at) { 6322 length = RegisterFrom(locations->GetTemp(0)); 6323 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 6324 GetAssembler()->LoadFromOffset(kLoadWord, length, obj, count_offset); 6325 codegen_->MaybeRecordImplicitNullCheck(instruction); 6326 } 6327 if (index.IsConstant()) { 6328 int32_t const_index = Int32ConstantFrom(index); 6329 if (maybe_compressed_char_at) { 6330 vixl32::Label uncompressed_load, done; 6331 vixl32::Label* final_label = codegen_->GetFinalLabel(instruction, &done); 6332 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 6333 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 6334 "Expecting 0=compressed, 1=uncompressed"); 6335 __ B(cs, &uncompressed_load, /* far_target */ false); 6336 GetAssembler()->LoadFromOffset(kLoadUnsignedByte, 6337 RegisterFrom(out_loc), 6338 obj, 6339 data_offset + const_index); 6340 __ B(final_label); 6341 __ Bind(&uncompressed_load); 6342 GetAssembler()->LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 6343 RegisterFrom(out_loc), 6344 obj, 6345 data_offset + (const_index << 1)); 6346 if (done.IsReferenced()) { 6347 __ Bind(&done); 6348 } 6349 } else { 6350 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 6351 6352 LoadOperandType load_type = GetLoadOperandType(type); 6353 GetAssembler()->LoadFromOffset(load_type, RegisterFrom(out_loc), obj, full_offset); 6354 } 6355 } else { 6356 UseScratchRegisterScope temps(GetVIXLAssembler()); 6357 vixl32::Register temp = temps.Acquire(); 6358 6359 if (has_intermediate_address) { 6360 // We do not need to compute the intermediate address from the array: the 6361 // input instruction has done it already. See the comment in 6362 // `TryExtractArrayAccessAddress()`. 6363 if (kIsDebugBuild) { 6364 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 6365 DCHECK_EQ(Uint64ConstantFrom(tmp->GetOffset()), data_offset); 6366 } 6367 temp = obj; 6368 } else { 6369 __ Add(temp, obj, data_offset); 6370 } 6371 if (maybe_compressed_char_at) { 6372 vixl32::Label uncompressed_load, done; 6373 vixl32::Label* final_label = codegen_->GetFinalLabel(instruction, &done); 6374 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 6375 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 6376 "Expecting 0=compressed, 1=uncompressed"); 6377 __ B(cs, &uncompressed_load, /* far_target */ false); 6378 __ Ldrb(RegisterFrom(out_loc), MemOperand(temp, RegisterFrom(index), vixl32::LSL, 0)); 6379 __ B(final_label); 6380 __ Bind(&uncompressed_load); 6381 __ Ldrh(RegisterFrom(out_loc), MemOperand(temp, RegisterFrom(index), vixl32::LSL, 1)); 6382 if (done.IsReferenced()) { 6383 __ Bind(&done); 6384 } 6385 } else { 6386 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, RegisterFrom(index)); 6387 } 6388 } 6389 break; 6390 } 6391 6392 case Primitive::kPrimNot: { 6393 // The read barrier instrumentation of object ArrayGet 6394 // instructions does not support the HIntermediateAddress 6395 // instruction. 6396 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 6397 6398 static_assert( 6399 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6400 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6401 // /* HeapReference<Object> */ out = 6402 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6403 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 6404 Location temp = locations->GetTemp(0); 6405 // Note that a potential implicit null check is handled in this 6406 // CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier call. 6407 DCHECK(!instruction->CanDoImplicitNullCheckOn(instruction->InputAt(0))); 6408 if (index.IsConstant()) { 6409 // Array load with a constant index can be treated as a field load. 6410 data_offset += Int32ConstantFrom(index) << Primitive::ComponentSizeShift(type); 6411 codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, 6412 out_loc, 6413 obj, 6414 data_offset, 6415 locations->GetTemp(0), 6416 /* needs_null_check */ false); 6417 } else { 6418 codegen_->GenerateArrayLoadWithBakerReadBarrier( 6419 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ false); 6420 } 6421 } else { 6422 vixl32::Register out = OutputRegister(instruction); 6423 if (index.IsConstant()) { 6424 size_t offset = 6425 (Int32ConstantFrom(index) << TIMES_4) + data_offset; 6426 GetAssembler()->LoadFromOffset(kLoadWord, out, obj, offset); 6427 // TODO(VIXL): Here and for other calls to `MaybeRecordImplicitNullCheck` in this method, 6428 // we should use a scope and the assembler to emit the load instruction to guarantee that 6429 // we record the pc at the correct position. But the `Assembler` does not automatically 6430 // handle unencodable offsets. Practically, everything is fine because the helper and 6431 // VIXL, at the time of writing, do generate the store instruction last. 6432 codegen_->MaybeRecordImplicitNullCheck(instruction); 6433 // If read barriers are enabled, emit read barriers other than 6434 // Baker's using a slow path (and also unpoison the loaded 6435 // reference, if heap poisoning is enabled). 6436 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 6437 } else { 6438 UseScratchRegisterScope temps(GetVIXLAssembler()); 6439 vixl32::Register temp = temps.Acquire(); 6440 6441 if (has_intermediate_address) { 6442 // We do not need to compute the intermediate address from the array: the 6443 // input instruction has done it already. See the comment in 6444 // `TryExtractArrayAccessAddress()`. 6445 if (kIsDebugBuild) { 6446 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 6447 DCHECK_EQ(Uint64ConstantFrom(tmp->GetOffset()), data_offset); 6448 } 6449 temp = obj; 6450 } else { 6451 __ Add(temp, obj, data_offset); 6452 } 6453 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, RegisterFrom(index)); 6454 temps.Close(); 6455 // TODO(VIXL): Use a scope to ensure that we record the pc position immediately after the 6456 // load instruction. Practically, everything is fine because the helper and VIXL, at the 6457 // time of writing, do generate the store instruction last. 6458 codegen_->MaybeRecordImplicitNullCheck(instruction); 6459 // If read barriers are enabled, emit read barriers other than 6460 // Baker's using a slow path (and also unpoison the loaded 6461 // reference, if heap poisoning is enabled). 6462 codegen_->MaybeGenerateReadBarrierSlow( 6463 instruction, out_loc, out_loc, obj_loc, data_offset, index); 6464 } 6465 } 6466 break; 6467 } 6468 6469 case Primitive::kPrimLong: { 6470 if (index.IsConstant()) { 6471 size_t offset = 6472 (Int32ConstantFrom(index) << TIMES_8) + data_offset; 6473 GetAssembler()->LoadFromOffset(kLoadWordPair, LowRegisterFrom(out_loc), obj, offset); 6474 } else { 6475 UseScratchRegisterScope temps(GetVIXLAssembler()); 6476 vixl32::Register temp = temps.Acquire(); 6477 __ Add(temp, obj, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); 6478 GetAssembler()->LoadFromOffset(kLoadWordPair, LowRegisterFrom(out_loc), temp, data_offset); 6479 } 6480 break; 6481 } 6482 6483 case Primitive::kPrimFloat: { 6484 vixl32::SRegister out = SRegisterFrom(out_loc); 6485 if (index.IsConstant()) { 6486 size_t offset = (Int32ConstantFrom(index) << TIMES_4) + data_offset; 6487 GetAssembler()->LoadSFromOffset(out, obj, offset); 6488 } else { 6489 UseScratchRegisterScope temps(GetVIXLAssembler()); 6490 vixl32::Register temp = temps.Acquire(); 6491 __ Add(temp, obj, Operand(RegisterFrom(index), vixl32::LSL, TIMES_4)); 6492 GetAssembler()->LoadSFromOffset(out, temp, data_offset); 6493 } 6494 break; 6495 } 6496 6497 case Primitive::kPrimDouble: { 6498 if (index.IsConstant()) { 6499 size_t offset = (Int32ConstantFrom(index) << TIMES_8) + data_offset; 6500 GetAssembler()->LoadDFromOffset(DRegisterFrom(out_loc), obj, offset); 6501 } else { 6502 UseScratchRegisterScope temps(GetVIXLAssembler()); 6503 vixl32::Register temp = temps.Acquire(); 6504 __ Add(temp, obj, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); 6505 GetAssembler()->LoadDFromOffset(DRegisterFrom(out_loc), temp, data_offset); 6506 } 6507 break; 6508 } 6509 6510 case Primitive::kPrimVoid: 6511 LOG(FATAL) << "Unreachable type " << type; 6512 UNREACHABLE(); 6513 } 6514 6515 if (type == Primitive::kPrimNot) { 6516 // Potential implicit null checks, in the case of reference 6517 // arrays, are handled in the previous switch statement. 6518 } else if (!maybe_compressed_char_at) { 6519 // TODO(VIXL): Use a scope to ensure we record the pc info immediately after 6520 // the preceding load instruction. 6521 codegen_->MaybeRecordImplicitNullCheck(instruction); 6522 } 6523 } 6524 6525 void LocationsBuilderARMVIXL::VisitArraySet(HArraySet* instruction) { 6526 Primitive::Type value_type = instruction->GetComponentType(); 6527 6528 bool needs_write_barrier = 6529 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 6530 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 6531 6532 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 6533 instruction, 6534 may_need_runtime_call_for_type_check ? 6535 LocationSummary::kCallOnSlowPath : 6536 LocationSummary::kNoCall); 6537 6538 locations->SetInAt(0, Location::RequiresRegister()); 6539 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 6540 if (Primitive::IsFloatingPointType(value_type)) { 6541 locations->SetInAt(2, Location::RequiresFpuRegister()); 6542 } else { 6543 locations->SetInAt(2, Location::RequiresRegister()); 6544 } 6545 if (needs_write_barrier) { 6546 // Temporary registers for the write barrier. 6547 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 6548 locations->AddTemp(Location::RequiresRegister()); 6549 } 6550 } 6551 6552 void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { 6553 LocationSummary* locations = instruction->GetLocations(); 6554 vixl32::Register array = InputRegisterAt(instruction, 0); 6555 Location index = locations->InAt(1); 6556 Primitive::Type value_type = instruction->GetComponentType(); 6557 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 6558 bool needs_write_barrier = 6559 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 6560 uint32_t data_offset = 6561 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 6562 Location value_loc = locations->InAt(2); 6563 HInstruction* array_instr = instruction->GetArray(); 6564 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 6565 6566 switch (value_type) { 6567 case Primitive::kPrimBoolean: 6568 case Primitive::kPrimByte: 6569 case Primitive::kPrimShort: 6570 case Primitive::kPrimChar: 6571 case Primitive::kPrimInt: { 6572 if (index.IsConstant()) { 6573 int32_t const_index = Int32ConstantFrom(index); 6574 uint32_t full_offset = 6575 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 6576 StoreOperandType store_type = GetStoreOperandType(value_type); 6577 GetAssembler()->StoreToOffset(store_type, RegisterFrom(value_loc), array, full_offset); 6578 } else { 6579 UseScratchRegisterScope temps(GetVIXLAssembler()); 6580 vixl32::Register temp = temps.Acquire(); 6581 6582 if (has_intermediate_address) { 6583 // We do not need to compute the intermediate address from the array: the 6584 // input instruction has done it already. See the comment in 6585 // `TryExtractArrayAccessAddress()`. 6586 if (kIsDebugBuild) { 6587 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 6588 DCHECK_EQ(Uint64ConstantFrom(tmp->GetOffset()), data_offset); 6589 } 6590 temp = array; 6591 } else { 6592 __ Add(temp, array, data_offset); 6593 } 6594 codegen_->StoreToShiftedRegOffset(value_type, value_loc, temp, RegisterFrom(index)); 6595 } 6596 break; 6597 } 6598 6599 case Primitive::kPrimNot: { 6600 vixl32::Register value = RegisterFrom(value_loc); 6601 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 6602 // See the comment in instruction_simplifier_shared.cc. 6603 DCHECK(!has_intermediate_address); 6604 6605 if (instruction->InputAt(2)->IsNullConstant()) { 6606 // Just setting null. 6607 if (index.IsConstant()) { 6608 size_t offset = 6609 (Int32ConstantFrom(index) << TIMES_4) + data_offset; 6610 GetAssembler()->StoreToOffset(kStoreWord, value, array, offset); 6611 } else { 6612 DCHECK(index.IsRegister()) << index; 6613 UseScratchRegisterScope temps(GetVIXLAssembler()); 6614 vixl32::Register temp = temps.Acquire(); 6615 __ Add(temp, array, data_offset); 6616 codegen_->StoreToShiftedRegOffset(value_type, value_loc, temp, RegisterFrom(index)); 6617 } 6618 // TODO(VIXL): Use a scope to ensure we record the pc info immediately after the preceding 6619 // store instruction. 6620 codegen_->MaybeRecordImplicitNullCheck(instruction); 6621 DCHECK(!needs_write_barrier); 6622 DCHECK(!may_need_runtime_call_for_type_check); 6623 break; 6624 } 6625 6626 DCHECK(needs_write_barrier); 6627 Location temp1_loc = locations->GetTemp(0); 6628 vixl32::Register temp1 = RegisterFrom(temp1_loc); 6629 Location temp2_loc = locations->GetTemp(1); 6630 vixl32::Register temp2 = RegisterFrom(temp2_loc); 6631 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6632 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6633 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6634 vixl32::Label done; 6635 vixl32::Label* final_label = codegen_->GetFinalLabel(instruction, &done); 6636 SlowPathCodeARMVIXL* slow_path = nullptr; 6637 6638 if (may_need_runtime_call_for_type_check) { 6639 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARMVIXL(instruction); 6640 codegen_->AddSlowPath(slow_path); 6641 if (instruction->GetValueCanBeNull()) { 6642 vixl32::Label non_zero; 6643 __ CompareAndBranchIfNonZero(value, &non_zero); 6644 if (index.IsConstant()) { 6645 size_t offset = 6646 (Int32ConstantFrom(index) << TIMES_4) + data_offset; 6647 GetAssembler()->StoreToOffset(kStoreWord, value, array, offset); 6648 } else { 6649 DCHECK(index.IsRegister()) << index; 6650 UseScratchRegisterScope temps(GetVIXLAssembler()); 6651 vixl32::Register temp = temps.Acquire(); 6652 __ Add(temp, array, data_offset); 6653 codegen_->StoreToShiftedRegOffset(value_type, value_loc, temp, RegisterFrom(index)); 6654 } 6655 // TODO(VIXL): Use a scope to ensure we record the pc info immediately after the preceding 6656 // store instruction. 6657 codegen_->MaybeRecordImplicitNullCheck(instruction); 6658 __ B(final_label); 6659 __ Bind(&non_zero); 6660 } 6661 6662 // Note that when read barriers are enabled, the type checks 6663 // are performed without read barriers. This is fine, even in 6664 // the case where a class object is in the from-space after 6665 // the flip, as a comparison involving such a type would not 6666 // produce a false positive; it may of course produce a false 6667 // negative, in which case we would take the ArraySet slow 6668 // path. 6669 6670 { 6671 // Ensure we record the pc position immediately after the `ldr` instruction. 6672 ExactAssemblyScope aas(GetVIXLAssembler(), 6673 vixl32::kMaxInstructionSizeInBytes, 6674 CodeBufferCheckScope::kMaximumSize); 6675 // /* HeapReference<Class> */ temp1 = array->klass_ 6676 __ ldr(temp1, MemOperand(array, class_offset)); 6677 codegen_->MaybeRecordImplicitNullCheck(instruction); 6678 } 6679 GetAssembler()->MaybeUnpoisonHeapReference(temp1); 6680 6681 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 6682 GetAssembler()->LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 6683 // /* HeapReference<Class> */ temp2 = value->klass_ 6684 GetAssembler()->LoadFromOffset(kLoadWord, temp2, value, class_offset); 6685 // If heap poisoning is enabled, no need to unpoison `temp1` 6686 // nor `temp2`, as we are comparing two poisoned references. 6687 __ Cmp(temp1, temp2); 6688 6689 if (instruction->StaticTypeOfArrayIsObjectArray()) { 6690 vixl32::Label do_put; 6691 __ B(eq, &do_put, /* far_target */ false); 6692 // If heap poisoning is enabled, the `temp1` reference has 6693 // not been unpoisoned yet; unpoison it now. 6694 GetAssembler()->MaybeUnpoisonHeapReference(temp1); 6695 6696 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 6697 GetAssembler()->LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 6698 // If heap poisoning is enabled, no need to unpoison 6699 // `temp1`, as we are comparing against null below. 6700 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 6701 __ Bind(&do_put); 6702 } else { 6703 __ B(ne, slow_path->GetEntryLabel()); 6704 } 6705 } 6706 6707 vixl32::Register source = value; 6708 if (kPoisonHeapReferences) { 6709 // Note that in the case where `value` is a null reference, 6710 // we do not enter this block, as a null reference does not 6711 // need poisoning. 6712 DCHECK_EQ(value_type, Primitive::kPrimNot); 6713 __ Mov(temp1, value); 6714 GetAssembler()->PoisonHeapReference(temp1); 6715 source = temp1; 6716 } 6717 6718 if (index.IsConstant()) { 6719 size_t offset = 6720 (Int32ConstantFrom(index) << TIMES_4) + data_offset; 6721 GetAssembler()->StoreToOffset(kStoreWord, source, array, offset); 6722 } else { 6723 DCHECK(index.IsRegister()) << index; 6724 6725 UseScratchRegisterScope temps(GetVIXLAssembler()); 6726 vixl32::Register temp = temps.Acquire(); 6727 __ Add(temp, array, data_offset); 6728 codegen_->StoreToShiftedRegOffset(value_type, 6729 LocationFrom(source), 6730 temp, 6731 RegisterFrom(index)); 6732 } 6733 6734 if (!may_need_runtime_call_for_type_check) { 6735 // TODO(VIXL): Ensure we record the pc position immediately after the preceding store 6736 // instruction. 6737 codegen_->MaybeRecordImplicitNullCheck(instruction); 6738 } 6739 6740 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 6741 6742 if (done.IsReferenced()) { 6743 __ Bind(&done); 6744 } 6745 6746 if (slow_path != nullptr) { 6747 __ Bind(slow_path->GetExitLabel()); 6748 } 6749 6750 break; 6751 } 6752 6753 case Primitive::kPrimLong: { 6754 Location value = locations->InAt(2); 6755 if (index.IsConstant()) { 6756 size_t offset = 6757 (Int32ConstantFrom(index) << TIMES_8) + data_offset; 6758 GetAssembler()->StoreToOffset(kStoreWordPair, LowRegisterFrom(value), array, offset); 6759 } else { 6760 UseScratchRegisterScope temps(GetVIXLAssembler()); 6761 vixl32::Register temp = temps.Acquire(); 6762 __ Add(temp, array, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); 6763 GetAssembler()->StoreToOffset(kStoreWordPair, LowRegisterFrom(value), temp, data_offset); 6764 } 6765 break; 6766 } 6767 6768 case Primitive::kPrimFloat: { 6769 Location value = locations->InAt(2); 6770 DCHECK(value.IsFpuRegister()); 6771 if (index.IsConstant()) { 6772 size_t offset = (Int32ConstantFrom(index) << TIMES_4) + data_offset; 6773 GetAssembler()->StoreSToOffset(SRegisterFrom(value), array, offset); 6774 } else { 6775 UseScratchRegisterScope temps(GetVIXLAssembler()); 6776 vixl32::Register temp = temps.Acquire(); 6777 __ Add(temp, array, Operand(RegisterFrom(index), vixl32::LSL, TIMES_4)); 6778 GetAssembler()->StoreSToOffset(SRegisterFrom(value), temp, data_offset); 6779 } 6780 break; 6781 } 6782 6783 case Primitive::kPrimDouble: { 6784 Location value = locations->InAt(2); 6785 DCHECK(value.IsFpuRegisterPair()); 6786 if (index.IsConstant()) { 6787 size_t offset = (Int32ConstantFrom(index) << TIMES_8) + data_offset; 6788 GetAssembler()->StoreDToOffset(DRegisterFrom(value), array, offset); 6789 } else { 6790 UseScratchRegisterScope temps(GetVIXLAssembler()); 6791 vixl32::Register temp = temps.Acquire(); 6792 __ Add(temp, array, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); 6793 GetAssembler()->StoreDToOffset(DRegisterFrom(value), temp, data_offset); 6794 } 6795 break; 6796 } 6797 6798 case Primitive::kPrimVoid: 6799 LOG(FATAL) << "Unreachable type " << value_type; 6800 UNREACHABLE(); 6801 } 6802 6803 // Objects are handled in the switch. 6804 if (value_type != Primitive::kPrimNot) { 6805 // TODO(VIXL): Ensure we record the pc position immediately after the preceding store 6806 // instruction. 6807 codegen_->MaybeRecordImplicitNullCheck(instruction); 6808 } 6809 } 6810 6811 void LocationsBuilderARMVIXL::VisitArrayLength(HArrayLength* instruction) { 6812 LocationSummary* locations = 6813 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6814 locations->SetInAt(0, Location::RequiresRegister()); 6815 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6816 } 6817 6818 void InstructionCodeGeneratorARMVIXL::VisitArrayLength(HArrayLength* instruction) { 6819 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 6820 vixl32::Register obj = InputRegisterAt(instruction, 0); 6821 vixl32::Register out = OutputRegister(instruction); 6822 { 6823 ExactAssemblyScope aas(GetVIXLAssembler(), 6824 vixl32::kMaxInstructionSizeInBytes, 6825 CodeBufferCheckScope::kMaximumSize); 6826 __ ldr(out, MemOperand(obj, offset)); 6827 codegen_->MaybeRecordImplicitNullCheck(instruction); 6828 } 6829 // Mask out compression flag from String's array length. 6830 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 6831 __ Lsr(out, out, 1u); 6832 } 6833 } 6834 6835 void LocationsBuilderARMVIXL::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6836 LocationSummary* locations = 6837 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6838 6839 locations->SetInAt(0, Location::RequiresRegister()); 6840 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 6841 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6842 } 6843 6844 void InstructionCodeGeneratorARMVIXL::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6845 vixl32::Register out = OutputRegister(instruction); 6846 vixl32::Register first = InputRegisterAt(instruction, 0); 6847 Location second = instruction->GetLocations()->InAt(1); 6848 6849 if (second.IsRegister()) { 6850 __ Add(out, first, RegisterFrom(second)); 6851 } else { 6852 __ Add(out, first, Int32ConstantFrom(second)); 6853 } 6854 } 6855 6856 void LocationsBuilderARMVIXL::VisitIntermediateAddressIndex( 6857 HIntermediateAddressIndex* instruction) { 6858 LOG(FATAL) << "Unreachable " << instruction->GetId(); 6859 } 6860 6861 void InstructionCodeGeneratorARMVIXL::VisitIntermediateAddressIndex( 6862 HIntermediateAddressIndex* instruction) { 6863 LOG(FATAL) << "Unreachable " << instruction->GetId(); 6864 } 6865 6866 void LocationsBuilderARMVIXL::VisitBoundsCheck(HBoundsCheck* instruction) { 6867 RegisterSet caller_saves = RegisterSet::Empty(); 6868 InvokeRuntimeCallingConventionARMVIXL calling_convention; 6869 caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0))); 6870 caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(1))); 6871 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 6872 6873 HInstruction* index = instruction->InputAt(0); 6874 HInstruction* length = instruction->InputAt(1); 6875 // If both index and length are constants we can statically check the bounds. But if at least one 6876 // of them is not encodable ArmEncodableConstantOrRegister will create 6877 // Location::RequiresRegister() which is not desired to happen. Instead we create constant 6878 // locations. 6879 bool both_const = index->IsConstant() && length->IsConstant(); 6880 locations->SetInAt(0, both_const 6881 ? Location::ConstantLocation(index->AsConstant()) 6882 : ArmEncodableConstantOrRegister(index, CMP)); 6883 locations->SetInAt(1, both_const 6884 ? Location::ConstantLocation(length->AsConstant()) 6885 : ArmEncodableConstantOrRegister(length, CMP)); 6886 } 6887 6888 void InstructionCodeGeneratorARMVIXL::VisitBoundsCheck(HBoundsCheck* instruction) { 6889 LocationSummary* locations = instruction->GetLocations(); 6890 Location index_loc = locations->InAt(0); 6891 Location length_loc = locations->InAt(1); 6892 6893 if (length_loc.IsConstant()) { 6894 int32_t length = Int32ConstantFrom(length_loc); 6895 if (index_loc.IsConstant()) { 6896 // BCE will remove the bounds check if we are guaranteed to pass. 6897 int32_t index = Int32ConstantFrom(index_loc); 6898 if (index < 0 || index >= length) { 6899 SlowPathCodeARMVIXL* slow_path = 6900 new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction); 6901 codegen_->AddSlowPath(slow_path); 6902 __ B(slow_path->GetEntryLabel()); 6903 } else { 6904 // Some optimization after BCE may have generated this, and we should not 6905 // generate a bounds check if it is a valid range. 6906 } 6907 return; 6908 } 6909 6910 SlowPathCodeARMVIXL* slow_path = 6911 new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction); 6912 __ Cmp(RegisterFrom(index_loc), length); 6913 codegen_->AddSlowPath(slow_path); 6914 __ B(hs, slow_path->GetEntryLabel()); 6915 } else { 6916 SlowPathCodeARMVIXL* slow_path = 6917 new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction); 6918 __ Cmp(RegisterFrom(length_loc), InputOperandAt(instruction, 0)); 6919 codegen_->AddSlowPath(slow_path); 6920 __ B(ls, slow_path->GetEntryLabel()); 6921 } 6922 } 6923 6924 void CodeGeneratorARMVIXL::MarkGCCard(vixl32::Register temp, 6925 vixl32::Register card, 6926 vixl32::Register object, 6927 vixl32::Register value, 6928 bool can_be_null) { 6929 vixl32::Label is_null; 6930 if (can_be_null) { 6931 __ CompareAndBranchIfZero(value, &is_null); 6932 } 6933 GetAssembler()->LoadFromOffset( 6934 kLoadWord, card, tr, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 6935 __ Lsr(temp, object, Operand::From(gc::accounting::CardTable::kCardShift)); 6936 __ Strb(card, MemOperand(card, temp)); 6937 if (can_be_null) { 6938 __ Bind(&is_null); 6939 } 6940 } 6941 6942 void LocationsBuilderARMVIXL::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 6943 LOG(FATAL) << "Unreachable"; 6944 } 6945 6946 void InstructionCodeGeneratorARMVIXL::VisitParallelMove(HParallelMove* instruction) { 6947 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 6948 } 6949 6950 void LocationsBuilderARMVIXL::VisitSuspendCheck(HSuspendCheck* instruction) { 6951 LocationSummary* locations = 6952 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 6953 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6954 } 6955 6956 void InstructionCodeGeneratorARMVIXL::VisitSuspendCheck(HSuspendCheck* instruction) { 6957 HBasicBlock* block = instruction->GetBlock(); 6958 if (block->GetLoopInformation() != nullptr) { 6959 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 6960 // The back edge will generate the suspend check. 6961 return; 6962 } 6963 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 6964 // The goto will generate the suspend check. 6965 return; 6966 } 6967 GenerateSuspendCheck(instruction, nullptr); 6968 } 6969 6970 void InstructionCodeGeneratorARMVIXL::GenerateSuspendCheck(HSuspendCheck* instruction, 6971 HBasicBlock* successor) { 6972 SuspendCheckSlowPathARMVIXL* slow_path = 6973 down_cast<SuspendCheckSlowPathARMVIXL*>(instruction->GetSlowPath()); 6974 if (slow_path == nullptr) { 6975 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARMVIXL(instruction, successor); 6976 instruction->SetSlowPath(slow_path); 6977 codegen_->AddSlowPath(slow_path); 6978 if (successor != nullptr) { 6979 DCHECK(successor->IsLoopHeader()); 6980 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 6981 } 6982 } else { 6983 DCHECK_EQ(slow_path->GetSuccessor(), successor); 6984 } 6985 6986 UseScratchRegisterScope temps(GetVIXLAssembler()); 6987 vixl32::Register temp = temps.Acquire(); 6988 GetAssembler()->LoadFromOffset( 6989 kLoadUnsignedHalfword, temp, tr, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 6990 if (successor == nullptr) { 6991 __ CompareAndBranchIfNonZero(temp, slow_path->GetEntryLabel()); 6992 __ Bind(slow_path->GetReturnLabel()); 6993 } else { 6994 __ CompareAndBranchIfZero(temp, codegen_->GetLabelOf(successor)); 6995 __ B(slow_path->GetEntryLabel()); 6996 } 6997 } 6998 6999 ArmVIXLAssembler* ParallelMoveResolverARMVIXL::GetAssembler() const { 7000 return codegen_->GetAssembler(); 7001 } 7002 7003 void ParallelMoveResolverARMVIXL::EmitMove(size_t index) { 7004 UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); 7005 MoveOperands* move = moves_[index]; 7006 Location source = move->GetSource(); 7007 Location destination = move->GetDestination(); 7008 7009 if (source.IsRegister()) { 7010 if (destination.IsRegister()) { 7011 __ Mov(RegisterFrom(destination), RegisterFrom(source)); 7012 } else if (destination.IsFpuRegister()) { 7013 __ Vmov(SRegisterFrom(destination), RegisterFrom(source)); 7014 } else { 7015 DCHECK(destination.IsStackSlot()); 7016 GetAssembler()->StoreToOffset(kStoreWord, 7017 RegisterFrom(source), 7018 sp, 7019 destination.GetStackIndex()); 7020 } 7021 } else if (source.IsStackSlot()) { 7022 if (destination.IsRegister()) { 7023 GetAssembler()->LoadFromOffset(kLoadWord, 7024 RegisterFrom(destination), 7025 sp, 7026 source.GetStackIndex()); 7027 } else if (destination.IsFpuRegister()) { 7028 GetAssembler()->LoadSFromOffset(SRegisterFrom(destination), sp, source.GetStackIndex()); 7029 } else { 7030 DCHECK(destination.IsStackSlot()); 7031 vixl32::Register temp = temps.Acquire(); 7032 GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, source.GetStackIndex()); 7033 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); 7034 } 7035 } else if (source.IsFpuRegister()) { 7036 if (destination.IsRegister()) { 7037 __ Vmov(RegisterFrom(destination), SRegisterFrom(source)); 7038 } else if (destination.IsFpuRegister()) { 7039 __ Vmov(SRegisterFrom(destination), SRegisterFrom(source)); 7040 } else { 7041 DCHECK(destination.IsStackSlot()); 7042 GetAssembler()->StoreSToOffset(SRegisterFrom(source), sp, destination.GetStackIndex()); 7043 } 7044 } else if (source.IsDoubleStackSlot()) { 7045 if (destination.IsDoubleStackSlot()) { 7046 vixl32::DRegister temp = temps.AcquireD(); 7047 GetAssembler()->LoadDFromOffset(temp, sp, source.GetStackIndex()); 7048 GetAssembler()->StoreDToOffset(temp, sp, destination.GetStackIndex()); 7049 } else if (destination.IsRegisterPair()) { 7050 DCHECK(ExpectedPairLayout(destination)); 7051 GetAssembler()->LoadFromOffset( 7052 kLoadWordPair, LowRegisterFrom(destination), sp, source.GetStackIndex()); 7053 } else { 7054 DCHECK(destination.IsFpuRegisterPair()) << destination; 7055 GetAssembler()->LoadDFromOffset(DRegisterFrom(destination), sp, source.GetStackIndex()); 7056 } 7057 } else if (source.IsRegisterPair()) { 7058 if (destination.IsRegisterPair()) { 7059 __ Mov(LowRegisterFrom(destination), LowRegisterFrom(source)); 7060 __ Mov(HighRegisterFrom(destination), HighRegisterFrom(source)); 7061 } else if (destination.IsFpuRegisterPair()) { 7062 __ Vmov(DRegisterFrom(destination), LowRegisterFrom(source), HighRegisterFrom(source)); 7063 } else { 7064 DCHECK(destination.IsDoubleStackSlot()) << destination; 7065 DCHECK(ExpectedPairLayout(source)); 7066 GetAssembler()->StoreToOffset(kStoreWordPair, 7067 LowRegisterFrom(source), 7068 sp, 7069 destination.GetStackIndex()); 7070 } 7071 } else if (source.IsFpuRegisterPair()) { 7072 if (destination.IsRegisterPair()) { 7073 __ Vmov(LowRegisterFrom(destination), HighRegisterFrom(destination), DRegisterFrom(source)); 7074 } else if (destination.IsFpuRegisterPair()) { 7075 __ Vmov(DRegisterFrom(destination), DRegisterFrom(source)); 7076 } else { 7077 DCHECK(destination.IsDoubleStackSlot()) << destination; 7078 GetAssembler()->StoreDToOffset(DRegisterFrom(source), sp, destination.GetStackIndex()); 7079 } 7080 } else { 7081 DCHECK(source.IsConstant()) << source; 7082 HConstant* constant = source.GetConstant(); 7083 if (constant->IsIntConstant() || constant->IsNullConstant()) { 7084 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 7085 if (destination.IsRegister()) { 7086 __ Mov(RegisterFrom(destination), value); 7087 } else { 7088 DCHECK(destination.IsStackSlot()); 7089 vixl32::Register temp = temps.Acquire(); 7090 __ Mov(temp, value); 7091 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); 7092 } 7093 } else if (constant->IsLongConstant()) { 7094 int64_t value = Int64ConstantFrom(source); 7095 if (destination.IsRegisterPair()) { 7096 __ Mov(LowRegisterFrom(destination), Low32Bits(value)); 7097 __ Mov(HighRegisterFrom(destination), High32Bits(value)); 7098 } else { 7099 DCHECK(destination.IsDoubleStackSlot()) << destination; 7100 vixl32::Register temp = temps.Acquire(); 7101 __ Mov(temp, Low32Bits(value)); 7102 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); 7103 __ Mov(temp, High32Bits(value)); 7104 GetAssembler()->StoreToOffset(kStoreWord, 7105 temp, 7106 sp, 7107 destination.GetHighStackIndex(kArmWordSize)); 7108 } 7109 } else if (constant->IsDoubleConstant()) { 7110 double value = constant->AsDoubleConstant()->GetValue(); 7111 if (destination.IsFpuRegisterPair()) { 7112 __ Vmov(DRegisterFrom(destination), value); 7113 } else { 7114 DCHECK(destination.IsDoubleStackSlot()) << destination; 7115 uint64_t int_value = bit_cast<uint64_t, double>(value); 7116 vixl32::Register temp = temps.Acquire(); 7117 __ Mov(temp, Low32Bits(int_value)); 7118 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); 7119 __ Mov(temp, High32Bits(int_value)); 7120 GetAssembler()->StoreToOffset(kStoreWord, 7121 temp, 7122 sp, 7123 destination.GetHighStackIndex(kArmWordSize)); 7124 } 7125 } else { 7126 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 7127 float value = constant->AsFloatConstant()->GetValue(); 7128 if (destination.IsFpuRegister()) { 7129 __ Vmov(SRegisterFrom(destination), value); 7130 } else { 7131 DCHECK(destination.IsStackSlot()); 7132 vixl32::Register temp = temps.Acquire(); 7133 __ Mov(temp, bit_cast<int32_t, float>(value)); 7134 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); 7135 } 7136 } 7137 } 7138 } 7139 7140 void ParallelMoveResolverARMVIXL::Exchange(vixl32::Register reg, int mem) { 7141 UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); 7142 vixl32::Register temp = temps.Acquire(); 7143 __ Mov(temp, reg); 7144 GetAssembler()->LoadFromOffset(kLoadWord, reg, sp, mem); 7145 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, mem); 7146 } 7147 7148 void ParallelMoveResolverARMVIXL::Exchange(int mem1, int mem2) { 7149 // TODO(VIXL32): Double check the performance of this implementation. 7150 UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); 7151 vixl32::Register temp1 = temps.Acquire(); 7152 ScratchRegisterScope ensure_scratch( 7153 this, temp1.GetCode(), r0.GetCode(), codegen_->GetNumberOfCoreRegisters()); 7154 vixl32::Register temp2(ensure_scratch.GetRegister()); 7155 7156 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 7157 GetAssembler()->LoadFromOffset(kLoadWord, temp1, sp, mem1 + stack_offset); 7158 GetAssembler()->LoadFromOffset(kLoadWord, temp2, sp, mem2 + stack_offset); 7159 GetAssembler()->StoreToOffset(kStoreWord, temp1, sp, mem2 + stack_offset); 7160 GetAssembler()->StoreToOffset(kStoreWord, temp2, sp, mem1 + stack_offset); 7161 } 7162 7163 void ParallelMoveResolverARMVIXL::EmitSwap(size_t index) { 7164 MoveOperands* move = moves_[index]; 7165 Location source = move->GetSource(); 7166 Location destination = move->GetDestination(); 7167 UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); 7168 7169 if (source.IsRegister() && destination.IsRegister()) { 7170 vixl32::Register temp = temps.Acquire(); 7171 DCHECK(!RegisterFrom(source).Is(temp)); 7172 DCHECK(!RegisterFrom(destination).Is(temp)); 7173 __ Mov(temp, RegisterFrom(destination)); 7174 __ Mov(RegisterFrom(destination), RegisterFrom(source)); 7175 __ Mov(RegisterFrom(source), temp); 7176 } else if (source.IsRegister() && destination.IsStackSlot()) { 7177 Exchange(RegisterFrom(source), destination.GetStackIndex()); 7178 } else if (source.IsStackSlot() && destination.IsRegister()) { 7179 Exchange(RegisterFrom(destination), source.GetStackIndex()); 7180 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 7181 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 7182 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 7183 vixl32::Register temp = temps.Acquire(); 7184 __ Vmov(temp, SRegisterFrom(source)); 7185 __ Vmov(SRegisterFrom(source), SRegisterFrom(destination)); 7186 __ Vmov(SRegisterFrom(destination), temp); 7187 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 7188 vixl32::DRegister temp = temps.AcquireD(); 7189 __ Vmov(temp, LowRegisterFrom(source), HighRegisterFrom(source)); 7190 __ Mov(LowRegisterFrom(source), LowRegisterFrom(destination)); 7191 __ Mov(HighRegisterFrom(source), HighRegisterFrom(destination)); 7192 __ Vmov(LowRegisterFrom(destination), HighRegisterFrom(destination), temp); 7193 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 7194 vixl32::Register low_reg = LowRegisterFrom(source.IsRegisterPair() ? source : destination); 7195 int mem = source.IsRegisterPair() ? destination.GetStackIndex() : source.GetStackIndex(); 7196 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 7197 vixl32::DRegister temp = temps.AcquireD(); 7198 __ Vmov(temp, low_reg, vixl32::Register(low_reg.GetCode() + 1)); 7199 GetAssembler()->LoadFromOffset(kLoadWordPair, low_reg, sp, mem); 7200 GetAssembler()->StoreDToOffset(temp, sp, mem); 7201 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 7202 vixl32::DRegister first = DRegisterFrom(source); 7203 vixl32::DRegister second = DRegisterFrom(destination); 7204 vixl32::DRegister temp = temps.AcquireD(); 7205 __ Vmov(temp, first); 7206 __ Vmov(first, second); 7207 __ Vmov(second, temp); 7208 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 7209 vixl32::DRegister reg = source.IsFpuRegisterPair() 7210 ? DRegisterFrom(source) 7211 : DRegisterFrom(destination); 7212 int mem = source.IsFpuRegisterPair() 7213 ? destination.GetStackIndex() 7214 : source.GetStackIndex(); 7215 vixl32::DRegister temp = temps.AcquireD(); 7216 __ Vmov(temp, reg); 7217 GetAssembler()->LoadDFromOffset(reg, sp, mem); 7218 GetAssembler()->StoreDToOffset(temp, sp, mem); 7219 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 7220 vixl32::SRegister reg = source.IsFpuRegister() 7221 ? SRegisterFrom(source) 7222 : SRegisterFrom(destination); 7223 int mem = source.IsFpuRegister() 7224 ? destination.GetStackIndex() 7225 : source.GetStackIndex(); 7226 vixl32::Register temp = temps.Acquire(); 7227 __ Vmov(temp, reg); 7228 GetAssembler()->LoadSFromOffset(reg, sp, mem); 7229 GetAssembler()->StoreToOffset(kStoreWord, temp, sp, mem); 7230 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 7231 vixl32::DRegister temp1 = temps.AcquireD(); 7232 vixl32::DRegister temp2 = temps.AcquireD(); 7233 __ Vldr(temp1, MemOperand(sp, source.GetStackIndex())); 7234 __ Vldr(temp2, MemOperand(sp, destination.GetStackIndex())); 7235 __ Vstr(temp1, MemOperand(sp, destination.GetStackIndex())); 7236 __ Vstr(temp2, MemOperand(sp, source.GetStackIndex())); 7237 } else { 7238 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 7239 } 7240 } 7241 7242 void ParallelMoveResolverARMVIXL::SpillScratch(int reg) { 7243 __ Push(vixl32::Register(reg)); 7244 } 7245 7246 void ParallelMoveResolverARMVIXL::RestoreScratch(int reg) { 7247 __ Pop(vixl32::Register(reg)); 7248 } 7249 7250 HLoadClass::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadClassKind( 7251 HLoadClass::LoadKind desired_class_load_kind) { 7252 switch (desired_class_load_kind) { 7253 case HLoadClass::LoadKind::kInvalid: 7254 LOG(FATAL) << "UNREACHABLE"; 7255 UNREACHABLE(); 7256 case HLoadClass::LoadKind::kReferrersClass: 7257 break; 7258 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 7259 case HLoadClass::LoadKind::kBssEntry: 7260 DCHECK(!Runtime::Current()->UseJitCompilation()); 7261 break; 7262 case HLoadClass::LoadKind::kJitTableAddress: 7263 DCHECK(Runtime::Current()->UseJitCompilation()); 7264 break; 7265 case HLoadClass::LoadKind::kBootImageAddress: 7266 case HLoadClass::LoadKind::kRuntimeCall: 7267 break; 7268 } 7269 return desired_class_load_kind; 7270 } 7271 7272 void LocationsBuilderARMVIXL::VisitLoadClass(HLoadClass* cls) { 7273 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 7274 if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { 7275 InvokeRuntimeCallingConventionARMVIXL calling_convention; 7276 CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( 7277 cls, 7278 LocationFrom(calling_convention.GetRegisterAt(0)), 7279 LocationFrom(r0)); 7280 DCHECK(calling_convention.GetRegisterAt(0).Is(r0)); 7281 return; 7282 } 7283 DCHECK(!cls->NeedsAccessCheck()); 7284 7285 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 7286 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 7287 ? LocationSummary::kCallOnSlowPath 7288 : LocationSummary::kNoCall; 7289 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 7290 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 7291 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 7292 } 7293 7294 if (load_kind == HLoadClass::LoadKind::kReferrersClass) { 7295 locations->SetInAt(0, Location::RequiresRegister()); 7296 } 7297 locations->SetOut(Location::RequiresRegister()); 7298 if (load_kind == HLoadClass::LoadKind::kBssEntry) { 7299 if (!kUseReadBarrier || kUseBakerReadBarrier) { 7300 // Rely on the type resolution or initialization and marking to save everything we need. 7301 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 7302 // to the custom calling convention) or by marking, so we request a different temp. 7303 locations->AddTemp(Location::RequiresRegister()); 7304 RegisterSet caller_saves = RegisterSet::Empty(); 7305 InvokeRuntimeCallingConventionARMVIXL calling_convention; 7306 caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0))); 7307 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 7308 // that the the kPrimNot result register is the same as the first argument register. 7309 locations->SetCustomSlowPathCallerSaves(caller_saves); 7310 } else { 7311 // For non-Baker read barrier we have a temp-clobbering call. 7312 } 7313 } 7314 if (kUseBakerReadBarrier && kBakerReadBarrierLinkTimeThunksEnableForGcRoots) { 7315 if (load_kind == HLoadClass::LoadKind::kBssEntry || 7316 (load_kind == HLoadClass::LoadKind::kReferrersClass && 7317 !Runtime::Current()->UseJitCompilation())) { 7318 locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister.GetCode())); 7319 } 7320 } 7321 } 7322 7323 // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 7324 // move. 7325 void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { 7326 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 7327 if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { 7328 codegen_->GenerateLoadClassRuntimeCall(cls); 7329 return; 7330 } 7331 DCHECK(!cls->NeedsAccessCheck()); 7332 7333 LocationSummary* locations = cls->GetLocations(); 7334 Location out_loc = locations->Out(); 7335 vixl32::Register out = OutputRegister(cls); 7336 7337 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 7338 ? kWithoutReadBarrier 7339 : kCompilerReadBarrierOption; 7340 bool generate_null_check = false; 7341 switch (load_kind) { 7342 case HLoadClass::LoadKind::kReferrersClass: { 7343 DCHECK(!cls->CanCallRuntime()); 7344 DCHECK(!cls->MustGenerateClinitCheck()); 7345 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 7346 vixl32::Register current_method = InputRegisterAt(cls, 0); 7347 GenerateGcRootFieldLoad(cls, 7348 out_loc, 7349 current_method, 7350 ArtMethod::DeclaringClassOffset().Int32Value(), 7351 read_barrier_option); 7352 break; 7353 } 7354 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 7355 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 7356 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 7357 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = 7358 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 7359 codegen_->EmitMovwMovtPlaceholder(labels, out); 7360 break; 7361 } 7362 case HLoadClass::LoadKind::kBootImageAddress: { 7363 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 7364 uint32_t address = dchecked_integral_cast<uint32_t>( 7365 reinterpret_cast<uintptr_t>(cls->GetClass().Get())); 7366 DCHECK_NE(address, 0u); 7367 __ Ldr(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 7368 break; 7369 } 7370 case HLoadClass::LoadKind::kBssEntry: { 7371 vixl32::Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 7372 ? RegisterFrom(locations->GetTemp(0)) 7373 : out; 7374 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = 7375 codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); 7376 codegen_->EmitMovwMovtPlaceholder(labels, temp); 7377 GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); 7378 generate_null_check = true; 7379 break; 7380 } 7381 case HLoadClass::LoadKind::kJitTableAddress: { 7382 __ Ldr(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 7383 cls->GetTypeIndex(), 7384 cls->GetClass())); 7385 // /* GcRoot<mirror::Class> */ out = *out 7386 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); 7387 break; 7388 } 7389 case HLoadClass::LoadKind::kRuntimeCall: 7390 case HLoadClass::LoadKind::kInvalid: 7391 LOG(FATAL) << "UNREACHABLE"; 7392 UNREACHABLE(); 7393 } 7394 7395 if (generate_null_check || cls->MustGenerateClinitCheck()) { 7396 DCHECK(cls->CanCallRuntime()); 7397 LoadClassSlowPathARMVIXL* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARMVIXL( 7398 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 7399 codegen_->AddSlowPath(slow_path); 7400 if (generate_null_check) { 7401 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 7402 } 7403 if (cls->MustGenerateClinitCheck()) { 7404 GenerateClassInitializationCheck(slow_path, out); 7405 } else { 7406 __ Bind(slow_path->GetExitLabel()); 7407 } 7408 } 7409 } 7410 7411 void LocationsBuilderARMVIXL::VisitClinitCheck(HClinitCheck* check) { 7412 LocationSummary* locations = 7413 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 7414 locations->SetInAt(0, Location::RequiresRegister()); 7415 if (check->HasUses()) { 7416 locations->SetOut(Location::SameAsFirstInput()); 7417 } 7418 } 7419 7420 void InstructionCodeGeneratorARMVIXL::VisitClinitCheck(HClinitCheck* check) { 7421 // We assume the class is not null. 7422 LoadClassSlowPathARMVIXL* slow_path = 7423 new (GetGraph()->GetArena()) LoadClassSlowPathARMVIXL(check->GetLoadClass(), 7424 check, 7425 check->GetDexPc(), 7426 /* do_clinit */ true); 7427 codegen_->AddSlowPath(slow_path); 7428 GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0)); 7429 } 7430 7431 void InstructionCodeGeneratorARMVIXL::GenerateClassInitializationCheck( 7432 LoadClassSlowPathARMVIXL* slow_path, vixl32::Register class_reg) { 7433 UseScratchRegisterScope temps(GetVIXLAssembler()); 7434 vixl32::Register temp = temps.Acquire(); 7435 GetAssembler()->LoadFromOffset(kLoadWord, 7436 temp, 7437 class_reg, 7438 mirror::Class::StatusOffset().Int32Value()); 7439 __ Cmp(temp, mirror::Class::kStatusInitialized); 7440 __ B(lt, slow_path->GetEntryLabel()); 7441 // Even if the initialized flag is set, we may be in a situation where caches are not synced 7442 // properly. Therefore, we do a memory fence. 7443 __ Dmb(ISH); 7444 __ Bind(slow_path->GetExitLabel()); 7445 } 7446 7447 HLoadString::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadStringKind( 7448 HLoadString::LoadKind desired_string_load_kind) { 7449 switch (desired_string_load_kind) { 7450 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 7451 case HLoadString::LoadKind::kBssEntry: 7452 DCHECK(!Runtime::Current()->UseJitCompilation()); 7453 break; 7454 case HLoadString::LoadKind::kJitTableAddress: 7455 DCHECK(Runtime::Current()->UseJitCompilation()); 7456 break; 7457 case HLoadString::LoadKind::kBootImageAddress: 7458 case HLoadString::LoadKind::kRuntimeCall: 7459 break; 7460 } 7461 return desired_string_load_kind; 7462 } 7463 7464 void LocationsBuilderARMVIXL::VisitLoadString(HLoadString* load) { 7465 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 7466 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 7467 HLoadString::LoadKind load_kind = load->GetLoadKind(); 7468 if (load_kind == HLoadString::LoadKind::kRuntimeCall) { 7469 locations->SetOut(LocationFrom(r0)); 7470 } else { 7471 locations->SetOut(Location::RequiresRegister()); 7472 if (load_kind == HLoadString::LoadKind::kBssEntry) { 7473 if (!kUseReadBarrier || kUseBakerReadBarrier) { 7474 // Rely on the pResolveString and marking to save everything we need, including temps. 7475 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 7476 // to the custom calling convention) or by marking, so we request a different temp. 7477 locations->AddTemp(Location::RequiresRegister()); 7478 RegisterSet caller_saves = RegisterSet::Empty(); 7479 InvokeRuntimeCallingConventionARMVIXL calling_convention; 7480 caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0))); 7481 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 7482 // that the the kPrimNot result register is the same as the first argument register. 7483 locations->SetCustomSlowPathCallerSaves(caller_saves); 7484 if (kUseBakerReadBarrier && kBakerReadBarrierLinkTimeThunksEnableForGcRoots) { 7485 locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister.GetCode())); 7486 } 7487 } else { 7488 // For non-Baker read barrier we have a temp-clobbering call. 7489 } 7490 } 7491 } 7492 } 7493 7494 // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 7495 // move. 7496 void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { 7497 LocationSummary* locations = load->GetLocations(); 7498 Location out_loc = locations->Out(); 7499 vixl32::Register out = OutputRegister(load); 7500 HLoadString::LoadKind load_kind = load->GetLoadKind(); 7501 7502 switch (load_kind) { 7503 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 7504 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 7505 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = 7506 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 7507 codegen_->EmitMovwMovtPlaceholder(labels, out); 7508 return; // No dex cache slow path. 7509 } 7510 case HLoadString::LoadKind::kBootImageAddress: { 7511 uint32_t address = dchecked_integral_cast<uint32_t>( 7512 reinterpret_cast<uintptr_t>(load->GetString().Get())); 7513 DCHECK_NE(address, 0u); 7514 __ Ldr(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 7515 return; // No dex cache slow path. 7516 } 7517 case HLoadString::LoadKind::kBssEntry: { 7518 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 7519 vixl32::Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 7520 ? RegisterFrom(locations->GetTemp(0)) 7521 : out; 7522 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = 7523 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 7524 codegen_->EmitMovwMovtPlaceholder(labels, temp); 7525 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 7526 LoadStringSlowPathARMVIXL* slow_path = 7527 new (GetGraph()->GetArena()) LoadStringSlowPathARMVIXL(load); 7528 codegen_->AddSlowPath(slow_path); 7529 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 7530 __ Bind(slow_path->GetExitLabel()); 7531 return; 7532 } 7533 case HLoadString::LoadKind::kJitTableAddress: { 7534 __ Ldr(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 7535 load->GetStringIndex(), 7536 load->GetString())); 7537 // /* GcRoot<mirror::String> */ out = *out 7538 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 7539 return; 7540 } 7541 default: 7542 break; 7543 } 7544 7545 // TODO: Re-add the compiler code to do string dex cache lookup again. 7546 DCHECK_EQ(load->GetLoadKind(), HLoadString::LoadKind::kRuntimeCall); 7547 InvokeRuntimeCallingConventionARMVIXL calling_convention; 7548 __ Mov(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 7549 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 7550 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 7551 } 7552 7553 static int32_t GetExceptionTlsOffset() { 7554 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 7555 } 7556 7557 void LocationsBuilderARMVIXL::VisitLoadException(HLoadException* load) { 7558 LocationSummary* locations = 7559 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 7560 locations->SetOut(Location::RequiresRegister()); 7561 } 7562 7563 void InstructionCodeGeneratorARMVIXL::VisitLoadException(HLoadException* load) { 7564 vixl32::Register out = OutputRegister(load); 7565 GetAssembler()->LoadFromOffset(kLoadWord, out, tr, GetExceptionTlsOffset()); 7566 } 7567 7568 7569 void LocationsBuilderARMVIXL::VisitClearException(HClearException* clear) { 7570 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 7571 } 7572 7573 void InstructionCodeGeneratorARMVIXL::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 7574 UseScratchRegisterScope temps(GetVIXLAssembler()); 7575 vixl32::Register temp = temps.Acquire(); 7576 __ Mov(temp, 0); 7577 GetAssembler()->StoreToOffset(kStoreWord, temp, tr, GetExceptionTlsOffset()); 7578 } 7579 7580 void LocationsBuilderARMVIXL::VisitThrow(HThrow* instruction) { 7581 LocationSummary* locations = 7582 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 7583 InvokeRuntimeCallingConventionARMVIXL calling_convention; 7584 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 7585 } 7586 7587 void InstructionCodeGeneratorARMVIXL::VisitThrow(HThrow* instruction) { 7588 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 7589 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 7590 } 7591 7592 // Temp is used for read barrier. 7593 static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 7594 if (kEmitCompilerReadBarrier && 7595 (kUseBakerReadBarrier || 7596 type_check_kind == TypeCheckKind::kAbstractClassCheck || 7597 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 7598 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 7599 return 1; 7600 } 7601 return 0; 7602 } 7603 7604 // Interface case has 3 temps, one for holding the number of interfaces, one for the current 7605 // interface pointer, one for loading the current interface. 7606 // The other checks have one temp for loading the object's class. 7607 static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 7608 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 7609 return 3; 7610 } 7611 return 1 + NumberOfInstanceOfTemps(type_check_kind); 7612 } 7613 7614 void LocationsBuilderARMVIXL::VisitInstanceOf(HInstanceOf* instruction) { 7615 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 7616 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7617 bool baker_read_barrier_slow_path = false; 7618 switch (type_check_kind) { 7619 case TypeCheckKind::kExactCheck: 7620 case TypeCheckKind::kAbstractClassCheck: 7621 case TypeCheckKind::kClassHierarchyCheck: 7622 case TypeCheckKind::kArrayObjectCheck: 7623 call_kind = 7624 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 7625 baker_read_barrier_slow_path = kUseBakerReadBarrier; 7626 break; 7627 case TypeCheckKind::kArrayCheck: 7628 case TypeCheckKind::kUnresolvedCheck: 7629 case TypeCheckKind::kInterfaceCheck: 7630 call_kind = LocationSummary::kCallOnSlowPath; 7631 break; 7632 } 7633 7634 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7635 if (baker_read_barrier_slow_path) { 7636 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 7637 } 7638 locations->SetInAt(0, Location::RequiresRegister()); 7639 locations->SetInAt(1, Location::RequiresRegister()); 7640 // The "out" register is used as a temporary, so it overlaps with the inputs. 7641 // Note that TypeCheckSlowPathARM uses this register too. 7642 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 7643 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 7644 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 7645 codegen_->MaybeAddBakerCcEntrypointTempForFields(locations); 7646 } 7647 } 7648 7649 void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) { 7650 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7651 LocationSummary* locations = instruction->GetLocations(); 7652 Location obj_loc = locations->InAt(0); 7653 vixl32::Register obj = InputRegisterAt(instruction, 0); 7654 vixl32::Register cls = InputRegisterAt(instruction, 1); 7655 Location out_loc = locations->Out(); 7656 vixl32::Register out = OutputRegister(instruction); 7657 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 7658 DCHECK_LE(num_temps, 1u); 7659 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 7660 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7661 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7662 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7663 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7664 vixl32::Label done; 7665 vixl32::Label* const final_label = codegen_->GetFinalLabel(instruction, &done); 7666 SlowPathCodeARMVIXL* slow_path = nullptr; 7667 7668 // Return 0 if `obj` is null. 7669 // avoid null check if we know obj is not null. 7670 if (instruction->MustDoNullCheck()) { 7671 DCHECK(!out.Is(obj)); 7672 __ Mov(out, 0); 7673 __ CompareAndBranchIfZero(obj, final_label, /* far_target */ false); 7674 } 7675 7676 switch (type_check_kind) { 7677 case TypeCheckKind::kExactCheck: { 7678 // /* HeapReference<Class> */ out = obj->klass_ 7679 GenerateReferenceLoadTwoRegisters(instruction, 7680 out_loc, 7681 obj_loc, 7682 class_offset, 7683 maybe_temp_loc, 7684 kCompilerReadBarrierOption); 7685 // Classes must be equal for the instanceof to succeed. 7686 __ Cmp(out, cls); 7687 // We speculatively set the result to false without changing the condition 7688 // flags, which allows us to avoid some branching later. 7689 __ Mov(LeaveFlags, out, 0); 7690 7691 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 7692 // we check that the output is in a low register, so that a 16-bit MOV 7693 // encoding can be used. 7694 if (out.IsLow()) { 7695 // We use the scope because of the IT block that follows. 7696 ExactAssemblyScope guard(GetVIXLAssembler(), 7697 2 * vixl32::k16BitT32InstructionSizeInBytes, 7698 CodeBufferCheckScope::kExactSize); 7699 7700 __ it(eq); 7701 __ mov(eq, out, 1); 7702 } else { 7703 __ B(ne, final_label, /* far_target */ false); 7704 __ Mov(out, 1); 7705 } 7706 7707 break; 7708 } 7709 7710 case TypeCheckKind::kAbstractClassCheck: { 7711 // /* HeapReference<Class> */ out = obj->klass_ 7712 GenerateReferenceLoadTwoRegisters(instruction, 7713 out_loc, 7714 obj_loc, 7715 class_offset, 7716 maybe_temp_loc, 7717 kCompilerReadBarrierOption); 7718 // If the class is abstract, we eagerly fetch the super class of the 7719 // object to avoid doing a comparison we know will fail. 7720 vixl32::Label loop; 7721 __ Bind(&loop); 7722 // /* HeapReference<Class> */ out = out->super_class_ 7723 GenerateReferenceLoadOneRegister(instruction, 7724 out_loc, 7725 super_offset, 7726 maybe_temp_loc, 7727 kCompilerReadBarrierOption); 7728 // If `out` is null, we use it for the result, and jump to the final label. 7729 __ CompareAndBranchIfZero(out, final_label, /* far_target */ false); 7730 __ Cmp(out, cls); 7731 __ B(ne, &loop, /* far_target */ false); 7732 __ Mov(out, 1); 7733 break; 7734 } 7735 7736 case TypeCheckKind::kClassHierarchyCheck: { 7737 // /* HeapReference<Class> */ out = obj->klass_ 7738 GenerateReferenceLoadTwoRegisters(instruction, 7739 out_loc, 7740 obj_loc, 7741 class_offset, 7742 maybe_temp_loc, 7743 kCompilerReadBarrierOption); 7744 // Walk over the class hierarchy to find a match. 7745 vixl32::Label loop, success; 7746 __ Bind(&loop); 7747 __ Cmp(out, cls); 7748 __ B(eq, &success, /* far_target */ false); 7749 // /* HeapReference<Class> */ out = out->super_class_ 7750 GenerateReferenceLoadOneRegister(instruction, 7751 out_loc, 7752 super_offset, 7753 maybe_temp_loc, 7754 kCompilerReadBarrierOption); 7755 // This is essentially a null check, but it sets the condition flags to the 7756 // proper value for the code that follows the loop, i.e. not `eq`. 7757 __ Cmp(out, 1); 7758 __ B(hs, &loop, /* far_target */ false); 7759 7760 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 7761 // we check that the output is in a low register, so that a 16-bit MOV 7762 // encoding can be used. 7763 if (out.IsLow()) { 7764 // If `out` is null, we use it for the result, and the condition flags 7765 // have already been set to `ne`, so the IT block that comes afterwards 7766 // (and which handles the successful case) turns into a NOP (instead of 7767 // overwriting `out`). 7768 __ Bind(&success); 7769 7770 // We use the scope because of the IT block that follows. 7771 ExactAssemblyScope guard(GetVIXLAssembler(), 7772 2 * vixl32::k16BitT32InstructionSizeInBytes, 7773 CodeBufferCheckScope::kExactSize); 7774 7775 // There is only one branch to the `success` label (which is bound to this 7776 // IT block), and it has the same condition, `eq`, so in that case the MOV 7777 // is executed. 7778 __ it(eq); 7779 __ mov(eq, out, 1); 7780 } else { 7781 // If `out` is null, we use it for the result, and jump to the final label. 7782 __ B(final_label); 7783 __ Bind(&success); 7784 __ Mov(out, 1); 7785 } 7786 7787 break; 7788 } 7789 7790 case TypeCheckKind::kArrayObjectCheck: { 7791 // /* HeapReference<Class> */ out = obj->klass_ 7792 GenerateReferenceLoadTwoRegisters(instruction, 7793 out_loc, 7794 obj_loc, 7795 class_offset, 7796 maybe_temp_loc, 7797 kCompilerReadBarrierOption); 7798 // Do an exact check. 7799 vixl32::Label exact_check; 7800 __ Cmp(out, cls); 7801 __ B(eq, &exact_check, /* far_target */ false); 7802 // Otherwise, we need to check that the object's class is a non-primitive array. 7803 // /* HeapReference<Class> */ out = out->component_type_ 7804 GenerateReferenceLoadOneRegister(instruction, 7805 out_loc, 7806 component_offset, 7807 maybe_temp_loc, 7808 kCompilerReadBarrierOption); 7809 // If `out` is null, we use it for the result, and jump to the final label. 7810 __ CompareAndBranchIfZero(out, final_label, /* far_target */ false); 7811 GetAssembler()->LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 7812 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 7813 __ Cmp(out, 0); 7814 // We speculatively set the result to false without changing the condition 7815 // flags, which allows us to avoid some branching later. 7816 __ Mov(LeaveFlags, out, 0); 7817 7818 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 7819 // we check that the output is in a low register, so that a 16-bit MOV 7820 // encoding can be used. 7821 if (out.IsLow()) { 7822 __ Bind(&exact_check); 7823 7824 // We use the scope because of the IT block that follows. 7825 ExactAssemblyScope guard(GetVIXLAssembler(), 7826 2 * vixl32::k16BitT32InstructionSizeInBytes, 7827 CodeBufferCheckScope::kExactSize); 7828 7829 __ it(eq); 7830 __ mov(eq, out, 1); 7831 } else { 7832 __ B(ne, final_label, /* far_target */ false); 7833 __ Bind(&exact_check); 7834 __ Mov(out, 1); 7835 } 7836 7837 break; 7838 } 7839 7840 case TypeCheckKind::kArrayCheck: { 7841 // No read barrier since the slow path will retry upon failure. 7842 // /* HeapReference<Class> */ out = obj->klass_ 7843 GenerateReferenceLoadTwoRegisters(instruction, 7844 out_loc, 7845 obj_loc, 7846 class_offset, 7847 maybe_temp_loc, 7848 kWithoutReadBarrier); 7849 __ Cmp(out, cls); 7850 DCHECK(locations->OnlyCallsOnSlowPath()); 7851 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARMVIXL(instruction, 7852 /* is_fatal */ false); 7853 codegen_->AddSlowPath(slow_path); 7854 __ B(ne, slow_path->GetEntryLabel()); 7855 __ Mov(out, 1); 7856 break; 7857 } 7858 7859 case TypeCheckKind::kUnresolvedCheck: 7860 case TypeCheckKind::kInterfaceCheck: { 7861 // Note that we indeed only call on slow path, but we always go 7862 // into the slow path for the unresolved and interface check 7863 // cases. 7864 // 7865 // We cannot directly call the InstanceofNonTrivial runtime 7866 // entry point without resorting to a type checking slow path 7867 // here (i.e. by calling InvokeRuntime directly), as it would 7868 // require to assign fixed registers for the inputs of this 7869 // HInstanceOf instruction (following the runtime calling 7870 // convention), which might be cluttered by the potential first 7871 // read barrier emission at the beginning of this method. 7872 // 7873 // TODO: Introduce a new runtime entry point taking the object 7874 // to test (instead of its class) as argument, and let it deal 7875 // with the read barrier issues. This will let us refactor this 7876 // case of the `switch` code as it was previously (with a direct 7877 // call to the runtime not using a type checking slow path). 7878 // This should also be beneficial for the other cases above. 7879 DCHECK(locations->OnlyCallsOnSlowPath()); 7880 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARMVIXL(instruction, 7881 /* is_fatal */ false); 7882 codegen_->AddSlowPath(slow_path); 7883 __ B(slow_path->GetEntryLabel()); 7884 break; 7885 } 7886 } 7887 7888 if (done.IsReferenced()) { 7889 __ Bind(&done); 7890 } 7891 7892 if (slow_path != nullptr) { 7893 __ Bind(slow_path->GetExitLabel()); 7894 } 7895 } 7896 7897 void LocationsBuilderARMVIXL::VisitCheckCast(HCheckCast* instruction) { 7898 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 7899 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 7900 7901 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7902 switch (type_check_kind) { 7903 case TypeCheckKind::kExactCheck: 7904 case TypeCheckKind::kAbstractClassCheck: 7905 case TypeCheckKind::kClassHierarchyCheck: 7906 case TypeCheckKind::kArrayObjectCheck: 7907 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 7908 LocationSummary::kCallOnSlowPath : 7909 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 7910 break; 7911 case TypeCheckKind::kArrayCheck: 7912 case TypeCheckKind::kUnresolvedCheck: 7913 case TypeCheckKind::kInterfaceCheck: 7914 call_kind = LocationSummary::kCallOnSlowPath; 7915 break; 7916 } 7917 7918 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7919 locations->SetInAt(0, Location::RequiresRegister()); 7920 locations->SetInAt(1, Location::RequiresRegister()); 7921 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 7922 } 7923 7924 void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { 7925 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7926 LocationSummary* locations = instruction->GetLocations(); 7927 Location obj_loc = locations->InAt(0); 7928 vixl32::Register obj = InputRegisterAt(instruction, 0); 7929 vixl32::Register cls = InputRegisterAt(instruction, 1); 7930 Location temp_loc = locations->GetTemp(0); 7931 vixl32::Register temp = RegisterFrom(temp_loc); 7932 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 7933 DCHECK_LE(num_temps, 3u); 7934 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 7935 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 7936 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7937 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7938 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7939 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7940 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 7941 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 7942 const uint32_t object_array_data_offset = 7943 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 7944 7945 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 7946 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 7947 // read barriers is done for performance and code size reasons. 7948 bool is_type_check_slow_path_fatal = false; 7949 if (!kEmitCompilerReadBarrier) { 7950 is_type_check_slow_path_fatal = 7951 (type_check_kind == TypeCheckKind::kExactCheck || 7952 type_check_kind == TypeCheckKind::kAbstractClassCheck || 7953 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 7954 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 7955 !instruction->CanThrowIntoCatchBlock(); 7956 } 7957 SlowPathCodeARMVIXL* type_check_slow_path = 7958 new (GetGraph()->GetArena()) TypeCheckSlowPathARMVIXL(instruction, 7959 is_type_check_slow_path_fatal); 7960 codegen_->AddSlowPath(type_check_slow_path); 7961 7962 vixl32::Label done; 7963 vixl32::Label* final_label = codegen_->GetFinalLabel(instruction, &done); 7964 // Avoid null check if we know obj is not null. 7965 if (instruction->MustDoNullCheck()) { 7966 __ CompareAndBranchIfZero(obj, final_label, /* far_target */ false); 7967 } 7968 7969 switch (type_check_kind) { 7970 case TypeCheckKind::kExactCheck: 7971 case TypeCheckKind::kArrayCheck: { 7972 // /* HeapReference<Class> */ temp = obj->klass_ 7973 GenerateReferenceLoadTwoRegisters(instruction, 7974 temp_loc, 7975 obj_loc, 7976 class_offset, 7977 maybe_temp2_loc, 7978 kWithoutReadBarrier); 7979 7980 __ Cmp(temp, cls); 7981 // Jump to slow path for throwing the exception or doing a 7982 // more involved array check. 7983 __ B(ne, type_check_slow_path->GetEntryLabel()); 7984 break; 7985 } 7986 7987 case TypeCheckKind::kAbstractClassCheck: { 7988 // /* HeapReference<Class> */ temp = obj->klass_ 7989 GenerateReferenceLoadTwoRegisters(instruction, 7990 temp_loc, 7991 obj_loc, 7992 class_offset, 7993 maybe_temp2_loc, 7994 kWithoutReadBarrier); 7995 7996 // If the class is abstract, we eagerly fetch the super class of the 7997 // object to avoid doing a comparison we know will fail. 7998 vixl32::Label loop; 7999 __ Bind(&loop); 8000 // /* HeapReference<Class> */ temp = temp->super_class_ 8001 GenerateReferenceLoadOneRegister(instruction, 8002 temp_loc, 8003 super_offset, 8004 maybe_temp2_loc, 8005 kWithoutReadBarrier); 8006 8007 // If the class reference currently in `temp` is null, jump to the slow path to throw the 8008 // exception. 8009 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 8010 8011 // Otherwise, compare the classes. 8012 __ Cmp(temp, cls); 8013 __ B(ne, &loop, /* far_target */ false); 8014 break; 8015 } 8016 8017 case TypeCheckKind::kClassHierarchyCheck: { 8018 // /* HeapReference<Class> */ temp = obj->klass_ 8019 GenerateReferenceLoadTwoRegisters(instruction, 8020 temp_loc, 8021 obj_loc, 8022 class_offset, 8023 maybe_temp2_loc, 8024 kWithoutReadBarrier); 8025 8026 // Walk over the class hierarchy to find a match. 8027 vixl32::Label loop; 8028 __ Bind(&loop); 8029 __ Cmp(temp, cls); 8030 __ B(eq, final_label, /* far_target */ false); 8031 8032 // /* HeapReference<Class> */ temp = temp->super_class_ 8033 GenerateReferenceLoadOneRegister(instruction, 8034 temp_loc, 8035 super_offset, 8036 maybe_temp2_loc, 8037 kWithoutReadBarrier); 8038 8039 // If the class reference currently in `temp` is null, jump to the slow path to throw the 8040 // exception. 8041 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 8042 // Otherwise, jump to the beginning of the loop. 8043 __ B(&loop); 8044 break; 8045 } 8046 8047 case TypeCheckKind::kArrayObjectCheck: { 8048 // /* HeapReference<Class> */ temp = obj->klass_ 8049 GenerateReferenceLoadTwoRegisters(instruction, 8050 temp_loc, 8051 obj_loc, 8052 class_offset, 8053 maybe_temp2_loc, 8054 kWithoutReadBarrier); 8055 8056 // Do an exact check. 8057 __ Cmp(temp, cls); 8058 __ B(eq, final_label, /* far_target */ false); 8059 8060 // Otherwise, we need to check that the object's class is a non-primitive array. 8061 // /* HeapReference<Class> */ temp = temp->component_type_ 8062 GenerateReferenceLoadOneRegister(instruction, 8063 temp_loc, 8064 component_offset, 8065 maybe_temp2_loc, 8066 kWithoutReadBarrier); 8067 // If the component type is null, jump to the slow path to throw the exception. 8068 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 8069 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 8070 // to further check that this component type is not a primitive type. 8071 GetAssembler()->LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 8072 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 8073 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 8074 break; 8075 } 8076 8077 case TypeCheckKind::kUnresolvedCheck: 8078 // We always go into the type check slow path for the unresolved check case. 8079 // We cannot directly call the CheckCast runtime entry point 8080 // without resorting to a type checking slow path here (i.e. by 8081 // calling InvokeRuntime directly), as it would require to 8082 // assign fixed registers for the inputs of this HInstanceOf 8083 // instruction (following the runtime calling convention), which 8084 // might be cluttered by the potential first read barrier 8085 // emission at the beginning of this method. 8086 8087 __ B(type_check_slow_path->GetEntryLabel()); 8088 break; 8089 8090 case TypeCheckKind::kInterfaceCheck: { 8091 // Avoid read barriers to improve performance of the fast path. We can not get false 8092 // positives by doing this. 8093 // /* HeapReference<Class> */ temp = obj->klass_ 8094 GenerateReferenceLoadTwoRegisters(instruction, 8095 temp_loc, 8096 obj_loc, 8097 class_offset, 8098 maybe_temp2_loc, 8099 kWithoutReadBarrier); 8100 8101 // /* HeapReference<Class> */ temp = temp->iftable_ 8102 GenerateReferenceLoadTwoRegisters(instruction, 8103 temp_loc, 8104 temp_loc, 8105 iftable_offset, 8106 maybe_temp2_loc, 8107 kWithoutReadBarrier); 8108 // Iftable is never null. 8109 __ Ldr(RegisterFrom(maybe_temp2_loc), MemOperand(temp, array_length_offset)); 8110 // Loop through the iftable and check if any class matches. 8111 vixl32::Label start_loop; 8112 __ Bind(&start_loop); 8113 __ CompareAndBranchIfZero(RegisterFrom(maybe_temp2_loc), 8114 type_check_slow_path->GetEntryLabel()); 8115 __ Ldr(RegisterFrom(maybe_temp3_loc), MemOperand(temp, object_array_data_offset)); 8116 GetAssembler()->MaybeUnpoisonHeapReference(RegisterFrom(maybe_temp3_loc)); 8117 // Go to next interface. 8118 __ Add(temp, temp, Operand::From(2 * kHeapReferenceSize)); 8119 __ Sub(RegisterFrom(maybe_temp2_loc), RegisterFrom(maybe_temp2_loc), 2); 8120 // Compare the classes and continue the loop if they do not match. 8121 __ Cmp(cls, RegisterFrom(maybe_temp3_loc)); 8122 __ B(ne, &start_loop, /* far_target */ false); 8123 break; 8124 } 8125 } 8126 if (done.IsReferenced()) { 8127 __ Bind(&done); 8128 } 8129 8130 __ Bind(type_check_slow_path->GetExitLabel()); 8131 } 8132 8133 void LocationsBuilderARMVIXL::VisitMonitorOperation(HMonitorOperation* instruction) { 8134 LocationSummary* locations = 8135 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 8136 InvokeRuntimeCallingConventionARMVIXL calling_convention; 8137 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 8138 } 8139 8140 void InstructionCodeGeneratorARMVIXL::VisitMonitorOperation(HMonitorOperation* instruction) { 8141 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 8142 instruction, 8143 instruction->GetDexPc()); 8144 if (instruction->IsEnter()) { 8145 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 8146 } else { 8147 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 8148 } 8149 } 8150 8151 void LocationsBuilderARMVIXL::VisitAnd(HAnd* instruction) { 8152 HandleBitwiseOperation(instruction, AND); 8153 } 8154 8155 void LocationsBuilderARMVIXL::VisitOr(HOr* instruction) { 8156 HandleBitwiseOperation(instruction, ORR); 8157 } 8158 8159 void LocationsBuilderARMVIXL::VisitXor(HXor* instruction) { 8160 HandleBitwiseOperation(instruction, EOR); 8161 } 8162 8163 void LocationsBuilderARMVIXL::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 8164 LocationSummary* locations = 8165 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 8166 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 8167 || instruction->GetResultType() == Primitive::kPrimLong); 8168 // Note: GVN reorders commutative operations to have the constant on the right hand side. 8169 locations->SetInAt(0, Location::RequiresRegister()); 8170 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 8171 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 8172 } 8173 8174 void InstructionCodeGeneratorARMVIXL::VisitAnd(HAnd* instruction) { 8175 HandleBitwiseOperation(instruction); 8176 } 8177 8178 void InstructionCodeGeneratorARMVIXL::VisitOr(HOr* instruction) { 8179 HandleBitwiseOperation(instruction); 8180 } 8181 8182 void InstructionCodeGeneratorARMVIXL::VisitXor(HXor* instruction) { 8183 HandleBitwiseOperation(instruction); 8184 } 8185 8186 void LocationsBuilderARMVIXL::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 8187 LocationSummary* locations = 8188 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 8189 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 8190 || instruction->GetResultType() == Primitive::kPrimLong); 8191 8192 locations->SetInAt(0, Location::RequiresRegister()); 8193 locations->SetInAt(1, Location::RequiresRegister()); 8194 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 8195 } 8196 8197 void InstructionCodeGeneratorARMVIXL::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 8198 LocationSummary* locations = instruction->GetLocations(); 8199 Location first = locations->InAt(0); 8200 Location second = locations->InAt(1); 8201 Location out = locations->Out(); 8202 8203 if (instruction->GetResultType() == Primitive::kPrimInt) { 8204 vixl32::Register first_reg = RegisterFrom(first); 8205 vixl32::Register second_reg = RegisterFrom(second); 8206 vixl32::Register out_reg = RegisterFrom(out); 8207 8208 switch (instruction->GetOpKind()) { 8209 case HInstruction::kAnd: 8210 __ Bic(out_reg, first_reg, second_reg); 8211 break; 8212 case HInstruction::kOr: 8213 __ Orn(out_reg, first_reg, second_reg); 8214 break; 8215 // There is no EON on arm. 8216 case HInstruction::kXor: 8217 default: 8218 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 8219 UNREACHABLE(); 8220 } 8221 return; 8222 8223 } else { 8224 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 8225 vixl32::Register first_low = LowRegisterFrom(first); 8226 vixl32::Register first_high = HighRegisterFrom(first); 8227 vixl32::Register second_low = LowRegisterFrom(second); 8228 vixl32::Register second_high = HighRegisterFrom(second); 8229 vixl32::Register out_low = LowRegisterFrom(out); 8230 vixl32::Register out_high = HighRegisterFrom(out); 8231 8232 switch (instruction->GetOpKind()) { 8233 case HInstruction::kAnd: 8234 __ Bic(out_low, first_low, second_low); 8235 __ Bic(out_high, first_high, second_high); 8236 break; 8237 case HInstruction::kOr: 8238 __ Orn(out_low, first_low, second_low); 8239 __ Orn(out_high, first_high, second_high); 8240 break; 8241 // There is no EON on arm. 8242 case HInstruction::kXor: 8243 default: 8244 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 8245 UNREACHABLE(); 8246 } 8247 } 8248 } 8249 8250 void LocationsBuilderARMVIXL::VisitDataProcWithShifterOp( 8251 HDataProcWithShifterOp* instruction) { 8252 DCHECK(instruction->GetType() == Primitive::kPrimInt || 8253 instruction->GetType() == Primitive::kPrimLong); 8254 LocationSummary* locations = 8255 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 8256 const bool overlap = instruction->GetType() == Primitive::kPrimLong && 8257 HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); 8258 8259 locations->SetInAt(0, Location::RequiresRegister()); 8260 locations->SetInAt(1, Location::RequiresRegister()); 8261 locations->SetOut(Location::RequiresRegister(), 8262 overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); 8263 } 8264 8265 void InstructionCodeGeneratorARMVIXL::VisitDataProcWithShifterOp( 8266 HDataProcWithShifterOp* instruction) { 8267 const LocationSummary* const locations = instruction->GetLocations(); 8268 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 8269 const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 8270 8271 if (instruction->GetType() == Primitive::kPrimInt) { 8272 DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); 8273 8274 const vixl32::Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong 8275 ? LowRegisterFrom(locations->InAt(1)) 8276 : InputRegisterAt(instruction, 1); 8277 8278 GenerateDataProcInstruction(kind, 8279 OutputRegister(instruction), 8280 InputRegisterAt(instruction, 0), 8281 Operand(second, 8282 ShiftFromOpKind(op_kind), 8283 instruction->GetShiftAmount()), 8284 codegen_); 8285 } else { 8286 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 8287 8288 if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { 8289 const vixl32::Register second = InputRegisterAt(instruction, 1); 8290 8291 DCHECK(!LowRegisterFrom(locations->Out()).Is(second)); 8292 GenerateDataProc(kind, 8293 locations->Out(), 8294 locations->InAt(0), 8295 second, 8296 Operand(second, ShiftType::ASR, 31), 8297 codegen_); 8298 } else { 8299 GenerateLongDataProc(instruction, codegen_); 8300 } 8301 } 8302 } 8303 8304 // TODO(VIXL): Remove optimizations in the helper when they are implemented in vixl. 8305 void InstructionCodeGeneratorARMVIXL::GenerateAndConst(vixl32::Register out, 8306 vixl32::Register first, 8307 uint32_t value) { 8308 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 8309 if (value == 0xffffffffu) { 8310 if (!out.Is(first)) { 8311 __ Mov(out, first); 8312 } 8313 return; 8314 } 8315 if (value == 0u) { 8316 __ Mov(out, 0); 8317 return; 8318 } 8319 if (GetAssembler()->ShifterOperandCanHold(AND, value)) { 8320 __ And(out, first, value); 8321 } else if (GetAssembler()->ShifterOperandCanHold(BIC, ~value)) { 8322 __ Bic(out, first, ~value); 8323 } else { 8324 DCHECK(IsPowerOfTwo(value + 1)); 8325 __ Ubfx(out, first, 0, WhichPowerOf2(value + 1)); 8326 } 8327 } 8328 8329 // TODO(VIXL): Remove optimizations in the helper when they are implemented in vixl. 8330 void InstructionCodeGeneratorARMVIXL::GenerateOrrConst(vixl32::Register out, 8331 vixl32::Register first, 8332 uint32_t value) { 8333 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 8334 if (value == 0u) { 8335 if (!out.Is(first)) { 8336 __ Mov(out, first); 8337 } 8338 return; 8339 } 8340 if (value == 0xffffffffu) { 8341 __ Mvn(out, 0); 8342 return; 8343 } 8344 if (GetAssembler()->ShifterOperandCanHold(ORR, value)) { 8345 __ Orr(out, first, value); 8346 } else { 8347 DCHECK(GetAssembler()->ShifterOperandCanHold(ORN, ~value)); 8348 __ Orn(out, first, ~value); 8349 } 8350 } 8351 8352 // TODO(VIXL): Remove optimizations in the helper when they are implemented in vixl. 8353 void InstructionCodeGeneratorARMVIXL::GenerateEorConst(vixl32::Register out, 8354 vixl32::Register first, 8355 uint32_t value) { 8356 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 8357 if (value == 0u) { 8358 if (!out.Is(first)) { 8359 __ Mov(out, first); 8360 } 8361 return; 8362 } 8363 __ Eor(out, first, value); 8364 } 8365 8366 void InstructionCodeGeneratorARMVIXL::GenerateAddLongConst(Location out, 8367 Location first, 8368 uint64_t value) { 8369 vixl32::Register out_low = LowRegisterFrom(out); 8370 vixl32::Register out_high = HighRegisterFrom(out); 8371 vixl32::Register first_low = LowRegisterFrom(first); 8372 vixl32::Register first_high = HighRegisterFrom(first); 8373 uint32_t value_low = Low32Bits(value); 8374 uint32_t value_high = High32Bits(value); 8375 if (value_low == 0u) { 8376 if (!out_low.Is(first_low)) { 8377 __ Mov(out_low, first_low); 8378 } 8379 __ Add(out_high, first_high, value_high); 8380 return; 8381 } 8382 __ Adds(out_low, first_low, value_low); 8383 if (GetAssembler()->ShifterOperandCanHold(ADC, value_high, kCcDontCare)) { 8384 __ Adc(out_high, first_high, value_high); 8385 } else if (GetAssembler()->ShifterOperandCanHold(SBC, ~value_high, kCcDontCare)) { 8386 __ Sbc(out_high, first_high, ~value_high); 8387 } else { 8388 LOG(FATAL) << "Unexpected constant " << value_high; 8389 UNREACHABLE(); 8390 } 8391 } 8392 8393 void InstructionCodeGeneratorARMVIXL::HandleBitwiseOperation(HBinaryOperation* instruction) { 8394 LocationSummary* locations = instruction->GetLocations(); 8395 Location first = locations->InAt(0); 8396 Location second = locations->InAt(1); 8397 Location out = locations->Out(); 8398 8399 if (second.IsConstant()) { 8400 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 8401 uint32_t value_low = Low32Bits(value); 8402 if (instruction->GetResultType() == Primitive::kPrimInt) { 8403 vixl32::Register first_reg = InputRegisterAt(instruction, 0); 8404 vixl32::Register out_reg = OutputRegister(instruction); 8405 if (instruction->IsAnd()) { 8406 GenerateAndConst(out_reg, first_reg, value_low); 8407 } else if (instruction->IsOr()) { 8408 GenerateOrrConst(out_reg, first_reg, value_low); 8409 } else { 8410 DCHECK(instruction->IsXor()); 8411 GenerateEorConst(out_reg, first_reg, value_low); 8412 } 8413 } else { 8414 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 8415 uint32_t value_high = High32Bits(value); 8416 vixl32::Register first_low = LowRegisterFrom(first); 8417 vixl32::Register first_high = HighRegisterFrom(first); 8418 vixl32::Register out_low = LowRegisterFrom(out); 8419 vixl32::Register out_high = HighRegisterFrom(out); 8420 if (instruction->IsAnd()) { 8421 GenerateAndConst(out_low, first_low, value_low); 8422 GenerateAndConst(out_high, first_high, value_high); 8423 } else if (instruction->IsOr()) { 8424 GenerateOrrConst(out_low, first_low, value_low); 8425 GenerateOrrConst(out_high, first_high, value_high); 8426 } else { 8427 DCHECK(instruction->IsXor()); 8428 GenerateEorConst(out_low, first_low, value_low); 8429 GenerateEorConst(out_high, first_high, value_high); 8430 } 8431 } 8432 return; 8433 } 8434 8435 if (instruction->GetResultType() == Primitive::kPrimInt) { 8436 vixl32::Register first_reg = InputRegisterAt(instruction, 0); 8437 vixl32::Register second_reg = InputRegisterAt(instruction, 1); 8438 vixl32::Register out_reg = OutputRegister(instruction); 8439 if (instruction->IsAnd()) { 8440 __ And(out_reg, first_reg, second_reg); 8441 } else if (instruction->IsOr()) { 8442 __ Orr(out_reg, first_reg, second_reg); 8443 } else { 8444 DCHECK(instruction->IsXor()); 8445 __ Eor(out_reg, first_reg, second_reg); 8446 } 8447 } else { 8448 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 8449 vixl32::Register first_low = LowRegisterFrom(first); 8450 vixl32::Register first_high = HighRegisterFrom(first); 8451 vixl32::Register second_low = LowRegisterFrom(second); 8452 vixl32::Register second_high = HighRegisterFrom(second); 8453 vixl32::Register out_low = LowRegisterFrom(out); 8454 vixl32::Register out_high = HighRegisterFrom(out); 8455 if (instruction->IsAnd()) { 8456 __ And(out_low, first_low, second_low); 8457 __ And(out_high, first_high, second_high); 8458 } else if (instruction->IsOr()) { 8459 __ Orr(out_low, first_low, second_low); 8460 __ Orr(out_high, first_high, second_high); 8461 } else { 8462 DCHECK(instruction->IsXor()); 8463 __ Eor(out_low, first_low, second_low); 8464 __ Eor(out_high, first_high, second_high); 8465 } 8466 } 8467 } 8468 8469 void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadOneRegister( 8470 HInstruction* instruction, 8471 Location out, 8472 uint32_t offset, 8473 Location maybe_temp, 8474 ReadBarrierOption read_barrier_option) { 8475 vixl32::Register out_reg = RegisterFrom(out); 8476 if (read_barrier_option == kWithReadBarrier) { 8477 CHECK(kEmitCompilerReadBarrier); 8478 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 8479 if (kUseBakerReadBarrier) { 8480 // Load with fast path based Baker's read barrier. 8481 // /* HeapReference<Object> */ out = *(out + offset) 8482 codegen_->GenerateFieldLoadWithBakerReadBarrier( 8483 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 8484 } else { 8485 // Load with slow path based read barrier. 8486 // Save the value of `out` into `maybe_temp` before overwriting it 8487 // in the following move operation, as we will need it for the 8488 // read barrier below. 8489 __ Mov(RegisterFrom(maybe_temp), out_reg); 8490 // /* HeapReference<Object> */ out = *(out + offset) 8491 GetAssembler()->LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 8492 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 8493 } 8494 } else { 8495 // Plain load with no read barrier. 8496 // /* HeapReference<Object> */ out = *(out + offset) 8497 GetAssembler()->LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 8498 GetAssembler()->MaybeUnpoisonHeapReference(out_reg); 8499 } 8500 } 8501 8502 void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadTwoRegisters( 8503 HInstruction* instruction, 8504 Location out, 8505 Location obj, 8506 uint32_t offset, 8507 Location maybe_temp, 8508 ReadBarrierOption read_barrier_option) { 8509 vixl32::Register out_reg = RegisterFrom(out); 8510 vixl32::Register obj_reg = RegisterFrom(obj); 8511 if (read_barrier_option == kWithReadBarrier) { 8512 CHECK(kEmitCompilerReadBarrier); 8513 if (kUseBakerReadBarrier) { 8514 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 8515 // Load with fast path based Baker's read barrier. 8516 // /* HeapReference<Object> */ out = *(obj + offset) 8517 codegen_->GenerateFieldLoadWithBakerReadBarrier( 8518 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 8519 } else { 8520 // Load with slow path based read barrier. 8521 // /* HeapReference<Object> */ out = *(obj + offset) 8522 GetAssembler()->LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 8523 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 8524 } 8525 } else { 8526 // Plain load with no read barrier. 8527 // /* HeapReference<Object> */ out = *(obj + offset) 8528 GetAssembler()->LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 8529 GetAssembler()->MaybeUnpoisonHeapReference(out_reg); 8530 } 8531 } 8532 8533 void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( 8534 HInstruction* instruction, 8535 Location root, 8536 vixl32::Register obj, 8537 uint32_t offset, 8538 ReadBarrierOption read_barrier_option) { 8539 vixl32::Register root_reg = RegisterFrom(root); 8540 if (read_barrier_option == kWithReadBarrier) { 8541 DCHECK(kEmitCompilerReadBarrier); 8542 if (kUseBakerReadBarrier) { 8543 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 8544 // Baker's read barrier are used. 8545 if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && 8546 !Runtime::Current()->UseJitCompilation()) { 8547 // Query `art::Thread::Current()->GetIsGcMarking()` (stored in 8548 // the Marking Register) to decide whether we need to enter 8549 // the slow path to mark the GC root. 8550 // 8551 // We use link-time generated thunks for the slow path. That thunk 8552 // checks the reference and jumps to the entrypoint if needed. 8553 // 8554 // lr = &return_address; 8555 // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. 8556 // if (mr) { // Thread::Current()->GetIsGcMarking() 8557 // goto gc_root_thunk<root_reg>(lr) 8558 // } 8559 // return_address: 8560 8561 UseScratchRegisterScope temps(GetVIXLAssembler()); 8562 ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction); 8563 bool narrow = CanEmitNarrowLdr(root_reg, obj, offset); 8564 uint32_t custom_data = linker::Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData( 8565 root_reg.GetCode(), narrow); 8566 vixl32::Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data); 8567 8568 vixl::EmissionCheckScope guard(GetVIXLAssembler(), 4 * vixl32::kMaxInstructionSizeInBytes); 8569 vixl32::Label return_address; 8570 EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); 8571 __ cmp(mr, Operand(0)); 8572 // Currently the offset is always within range. If that changes, 8573 // we shall have to split the load the same way as for fields. 8574 DCHECK_LT(offset, kReferenceLoadMinFarOffset); 8575 ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); 8576 __ ldr(EncodingSize(narrow ? Narrow : Wide), root_reg, MemOperand(obj, offset)); 8577 EmitPlaceholderBne(codegen_, bne_label); 8578 __ Bind(&return_address); 8579 DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), 8580 narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET 8581 : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_OFFSET); 8582 } else { 8583 // Query `art::Thread::Current()->GetIsGcMarking()` (stored in 8584 // the Marking Register) to decide whether we need to enter 8585 // the slow path to mark the GC root. 8586 // 8587 // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. 8588 // if (mr) { // Thread::Current()->GetIsGcMarking() 8589 // // Slow path. 8590 // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() 8591 // root = entrypoint(root); // root = ReadBarrier::Mark(root); // Entry point call. 8592 // } 8593 8594 // Slow path marking the GC root `root`. The entrypoint will 8595 // be loaded by the slow path code. 8596 SlowPathCodeARMVIXL* slow_path = 8597 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(instruction, root); 8598 codegen_->AddSlowPath(slow_path); 8599 8600 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 8601 GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset); 8602 static_assert( 8603 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 8604 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 8605 "have different sizes."); 8606 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 8607 "art::mirror::CompressedReference<mirror::Object> and int32_t " 8608 "have different sizes."); 8609 8610 __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); 8611 __ Bind(slow_path->GetExitLabel()); 8612 } 8613 } else { 8614 // GC root loaded through a slow path for read barriers other 8615 // than Baker's. 8616 // /* GcRoot<mirror::Object>* */ root = obj + offset 8617 __ Add(root_reg, obj, offset); 8618 // /* mirror::Object* */ root = root->Read() 8619 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 8620 } 8621 } else { 8622 // Plain GC root load with no read barrier. 8623 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 8624 GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset); 8625 // Note that GC roots are not affected by heap poisoning, thus we 8626 // do not have to unpoison `root_reg` here. 8627 } 8628 } 8629 8630 void CodeGeneratorARMVIXL::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) { 8631 DCHECK(kEmitCompilerReadBarrier); 8632 DCHECK(kUseBakerReadBarrier); 8633 if (kBakerReadBarrierLinkTimeThunksEnableForFields) { 8634 if (!Runtime::Current()->UseJitCompilation()) { 8635 locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister.GetCode())); 8636 } 8637 } 8638 } 8639 8640 void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 8641 Location ref, 8642 vixl32::Register obj, 8643 uint32_t offset, 8644 Location temp, 8645 bool needs_null_check) { 8646 DCHECK(kEmitCompilerReadBarrier); 8647 DCHECK(kUseBakerReadBarrier); 8648 8649 if (kBakerReadBarrierLinkTimeThunksEnableForFields && 8650 !Runtime::Current()->UseJitCompilation()) { 8651 // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the 8652 // Marking Register) to decide whether we need to enter the slow 8653 // path to mark the reference. Then, in the slow path, check the 8654 // gray bit in the lock word of the reference's holder (`obj`) to 8655 // decide whether to mark `ref` or not. 8656 // 8657 // We use link-time generated thunks for the slow path. That thunk checks 8658 // the holder and jumps to the entrypoint if needed. If the holder is not 8659 // gray, it creates a fake dependency and returns to the LDR instruction. 8660 // 8661 // lr = &gray_return_address; 8662 // if (mr) { // Thread::Current()->GetIsGcMarking() 8663 // goto field_thunk<holder_reg, base_reg>(lr) 8664 // } 8665 // not_gray_return_address: 8666 // // Original reference load. If the offset is too large to fit 8667 // // into LDR, we use an adjusted base register here. 8668 // HeapReference<mirror::Object> reference = *(obj+offset); 8669 // gray_return_address: 8670 8671 DCHECK_ALIGNED(offset, sizeof(mirror::HeapReference<mirror::Object>)); 8672 vixl32::Register ref_reg = RegisterFrom(ref, Primitive::kPrimNot); 8673 bool narrow = CanEmitNarrowLdr(ref_reg, obj, offset); 8674 vixl32::Register base = obj; 8675 if (offset >= kReferenceLoadMinFarOffset) { 8676 base = RegisterFrom(temp); 8677 DCHECK(!base.Is(kBakerCcEntrypointRegister)); 8678 static_assert(IsPowerOfTwo(kReferenceLoadMinFarOffset), "Expecting a power of 2."); 8679 __ Add(base, obj, Operand(offset & ~(kReferenceLoadMinFarOffset - 1u))); 8680 offset &= (kReferenceLoadMinFarOffset - 1u); 8681 // Use narrow LDR only for small offsets. Generating narrow encoding LDR for the large 8682 // offsets with `(offset & (kReferenceLoadMinFarOffset - 1u)) < 32u` would most likely 8683 // increase the overall code size when taking the generated thunks into account. 8684 DCHECK(!narrow); 8685 } 8686 UseScratchRegisterScope temps(GetVIXLAssembler()); 8687 ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction); 8688 uint32_t custom_data = linker::Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( 8689 base.GetCode(), obj.GetCode(), narrow); 8690 vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); 8691 8692 vixl::EmissionCheckScope guard( 8693 GetVIXLAssembler(), 8694 (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); 8695 vixl32::Label return_address; 8696 EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); 8697 __ cmp(mr, Operand(0)); 8698 EmitPlaceholderBne(this, bne_label); 8699 ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); 8700 __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset)); 8701 if (needs_null_check) { 8702 MaybeRecordImplicitNullCheck(instruction); 8703 } 8704 // Note: We need a specific width for the unpoisoning NEG. 8705 if (kPoisonHeapReferences) { 8706 if (narrow) { 8707 // The only 16-bit encoding is T1 which sets flags outside IT block (i.e. RSBS, not RSB). 8708 __ rsbs(EncodingSize(Narrow), ref_reg, ref_reg, Operand(0)); 8709 } else { 8710 __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0)); 8711 } 8712 } 8713 __ Bind(&return_address); 8714 DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), 8715 narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET 8716 : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET); 8717 return; 8718 } 8719 8720 // /* HeapReference<Object> */ ref = *(obj + offset) 8721 Location no_index = Location::NoLocation(); 8722 ScaleFactor no_scale_factor = TIMES_1; 8723 GenerateReferenceLoadWithBakerReadBarrier( 8724 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 8725 } 8726 8727 void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 8728 Location ref, 8729 vixl32::Register obj, 8730 uint32_t data_offset, 8731 Location index, 8732 Location temp, 8733 bool needs_null_check) { 8734 DCHECK(kEmitCompilerReadBarrier); 8735 DCHECK(kUseBakerReadBarrier); 8736 8737 static_assert( 8738 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 8739 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 8740 ScaleFactor scale_factor = TIMES_4; 8741 8742 if (kBakerReadBarrierLinkTimeThunksEnableForArrays && 8743 !Runtime::Current()->UseJitCompilation()) { 8744 // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the 8745 // Marking Register) to decide whether we need to enter the slow 8746 // path to mark the reference. Then, in the slow path, check the 8747 // gray bit in the lock word of the reference's holder (`obj`) to 8748 // decide whether to mark `ref` or not. 8749 // 8750 // We use link-time generated thunks for the slow path. That thunk checks 8751 // the holder and jumps to the entrypoint if needed. If the holder is not 8752 // gray, it creates a fake dependency and returns to the LDR instruction. 8753 // 8754 // lr = &gray_return_address; 8755 // if (mr) { // Thread::Current()->GetIsGcMarking() 8756 // goto array_thunk<base_reg>(lr) 8757 // } 8758 // not_gray_return_address: 8759 // // Original reference load. If the offset is too large to fit 8760 // // into LDR, we use an adjusted base register here. 8761 // HeapReference<mirror::Object> reference = data[index]; 8762 // gray_return_address: 8763 8764 DCHECK(index.IsValid()); 8765 vixl32::Register index_reg = RegisterFrom(index, Primitive::kPrimInt); 8766 vixl32::Register ref_reg = RegisterFrom(ref, Primitive::kPrimNot); 8767 vixl32::Register data_reg = RegisterFrom(temp, Primitive::kPrimInt); // Raw pointer. 8768 DCHECK(!data_reg.Is(kBakerCcEntrypointRegister)); 8769 8770 UseScratchRegisterScope temps(GetVIXLAssembler()); 8771 ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction); 8772 uint32_t custom_data = 8773 linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg.GetCode()); 8774 vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); 8775 8776 __ Add(data_reg, obj, Operand(data_offset)); 8777 vixl::EmissionCheckScope guard( 8778 GetVIXLAssembler(), 8779 (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); 8780 vixl32::Label return_address; 8781 EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); 8782 __ cmp(mr, Operand(0)); 8783 EmitPlaceholderBne(this, bne_label); 8784 ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); 8785 __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor)); 8786 DCHECK(!needs_null_check); // The thunk cannot handle the null check. 8787 // Note: We need a Wide NEG for the unpoisoning. 8788 if (kPoisonHeapReferences) { 8789 __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0)); 8790 } 8791 __ Bind(&return_address); 8792 DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), 8793 BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); 8794 return; 8795 } 8796 8797 // /* HeapReference<Object> */ ref = 8798 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 8799 GenerateReferenceLoadWithBakerReadBarrier( 8800 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 8801 } 8802 8803 void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 8804 Location ref, 8805 vixl32::Register obj, 8806 uint32_t offset, 8807 Location index, 8808 ScaleFactor scale_factor, 8809 Location temp, 8810 bool needs_null_check) { 8811 DCHECK(kEmitCompilerReadBarrier); 8812 DCHECK(kUseBakerReadBarrier); 8813 8814 // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the 8815 // Marking Register) to decide whether we need to enter the slow 8816 // path to mark the reference. Then, in the slow path, check the 8817 // gray bit in the lock word of the reference's holder (`obj`) to 8818 // decide whether to mark `ref` or not. 8819 // 8820 // if (mr) { // Thread::Current()->GetIsGcMarking() 8821 // // Slow path. 8822 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 8823 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 8824 // HeapReference<mirror::Object> ref = *src; // Original reference load. 8825 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 8826 // if (is_gray) { 8827 // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() 8828 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 8829 // } 8830 // } else { 8831 // HeapReference<mirror::Object> ref = *src; // Original reference load. 8832 // } 8833 8834 vixl32::Register temp_reg = RegisterFrom(temp); 8835 8836 // Slow path marking the object `ref` when the GC is marking. The 8837 // entrypoint will be loaded by the slow path code. 8838 SlowPathCodeARMVIXL* slow_path = 8839 new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARMVIXL( 8840 instruction, ref, obj, offset, index, scale_factor, needs_null_check, temp_reg); 8841 AddSlowPath(slow_path); 8842 8843 __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); 8844 // Fast path: the GC is not marking: just load the reference. 8845 GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); 8846 __ Bind(slow_path->GetExitLabel()); 8847 } 8848 8849 void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, 8850 Location ref, 8851 vixl32::Register obj, 8852 Location field_offset, 8853 Location temp, 8854 bool needs_null_check, 8855 vixl32::Register temp2) { 8856 DCHECK(kEmitCompilerReadBarrier); 8857 DCHECK(kUseBakerReadBarrier); 8858 8859 // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the 8860 // Marking Register) to decide whether we need to enter the slow 8861 // path to update the reference field within `obj`. Then, in the 8862 // slow path, check the gray bit in the lock word of the reference's 8863 // holder (`obj`) to decide whether to mark `ref` and update the 8864 // field or not. 8865 // 8866 // if (mr) { // Thread::Current()->GetIsGcMarking() 8867 // // Slow path. 8868 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 8869 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 8870 // HeapReference<mirror::Object> ref = *(obj + field_offset); // Reference load. 8871 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 8872 // if (is_gray) { 8873 // old_ref = ref; 8874 // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() 8875 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 8876 // compareAndSwapObject(obj, field_offset, old_ref, ref); 8877 // } 8878 // } 8879 8880 vixl32::Register temp_reg = RegisterFrom(temp); 8881 8882 // Slow path updating the object reference at address `obj + field_offset` 8883 // when the GC is marking. The entrypoint will be loaded by the slow path code. 8884 SlowPathCodeARMVIXL* slow_path = 8885 new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL( 8886 instruction, 8887 ref, 8888 obj, 8889 /* offset */ 0u, 8890 /* index */ field_offset, 8891 /* scale_factor */ ScaleFactor::TIMES_1, 8892 needs_null_check, 8893 temp_reg, 8894 temp2); 8895 AddSlowPath(slow_path); 8896 8897 __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); 8898 // Fast path: the GC is not marking: nothing to do (the field is 8899 // up-to-date, and we don't need to load the reference). 8900 __ Bind(slow_path->GetExitLabel()); 8901 } 8902 8903 void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction, 8904 Location ref, 8905 vixl::aarch32::Register obj, 8906 uint32_t offset, 8907 Location index, 8908 ScaleFactor scale_factor, 8909 bool needs_null_check) { 8910 Primitive::Type type = Primitive::kPrimNot; 8911 vixl32::Register ref_reg = RegisterFrom(ref, type); 8912 8913 // If needed, vixl::EmissionCheckScope guards are used to ensure 8914 // that no pools are emitted between the load (macro) instruction 8915 // and MaybeRecordImplicitNullCheck. 8916 8917 if (index.IsValid()) { 8918 // Load types involving an "index": ArrayGet, 8919 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8920 // intrinsics. 8921 // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) 8922 if (index.IsConstant()) { 8923 size_t computed_offset = 8924 (Int32ConstantFrom(index) << scale_factor) + offset; 8925 vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); 8926 GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 8927 if (needs_null_check) { 8928 MaybeRecordImplicitNullCheck(instruction); 8929 } 8930 } else { 8931 // Handle the special case of the 8932 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8933 // intrinsics, which use a register pair as index ("long 8934 // offset"), of which only the low part contains data. 8935 vixl32::Register index_reg = index.IsRegisterPair() 8936 ? LowRegisterFrom(index) 8937 : RegisterFrom(index); 8938 UseScratchRegisterScope temps(GetVIXLAssembler()); 8939 vixl32::Register temp = temps.Acquire(); 8940 __ Add(temp, obj, Operand(index_reg, ShiftType::LSL, scale_factor)); 8941 { 8942 vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); 8943 GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, temp, offset); 8944 if (needs_null_check) { 8945 MaybeRecordImplicitNullCheck(instruction); 8946 } 8947 } 8948 } 8949 } else { 8950 // /* HeapReference<mirror::Object> */ ref = *(obj + offset) 8951 vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); 8952 GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, offset); 8953 if (needs_null_check) { 8954 MaybeRecordImplicitNullCheck(instruction); 8955 } 8956 } 8957 8958 // Object* ref = ref_addr->AsMirrorPtr() 8959 GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); 8960 } 8961 8962 void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction, 8963 Location out, 8964 Location ref, 8965 Location obj, 8966 uint32_t offset, 8967 Location index) { 8968 DCHECK(kEmitCompilerReadBarrier); 8969 8970 // Insert a slow path based read barrier *after* the reference load. 8971 // 8972 // If heap poisoning is enabled, the unpoisoning of the loaded 8973 // reference will be carried out by the runtime within the slow 8974 // path. 8975 // 8976 // Note that `ref` currently does not get unpoisoned (when heap 8977 // poisoning is enabled), which is alright as the `ref` argument is 8978 // not used by the artReadBarrierSlow entry point. 8979 // 8980 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 8981 SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) 8982 ReadBarrierForHeapReferenceSlowPathARMVIXL(instruction, out, ref, obj, offset, index); 8983 AddSlowPath(slow_path); 8984 8985 __ B(slow_path->GetEntryLabel()); 8986 __ Bind(slow_path->GetExitLabel()); 8987 } 8988 8989 void CodeGeneratorARMVIXL::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 8990 Location out, 8991 Location ref, 8992 Location obj, 8993 uint32_t offset, 8994 Location index) { 8995 if (kEmitCompilerReadBarrier) { 8996 // Baker's read barriers shall be handled by the fast path 8997 // (CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier). 8998 DCHECK(!kUseBakerReadBarrier); 8999 // If heap poisoning is enabled, unpoisoning will be taken care of 9000 // by the runtime within the slow path. 9001 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 9002 } else if (kPoisonHeapReferences) { 9003 GetAssembler()->UnpoisonHeapReference(RegisterFrom(out)); 9004 } 9005 } 9006 9007 void CodeGeneratorARMVIXL::GenerateReadBarrierForRootSlow(HInstruction* instruction, 9008 Location out, 9009 Location root) { 9010 DCHECK(kEmitCompilerReadBarrier); 9011 9012 // Insert a slow path based read barrier *after* the GC root load. 9013 // 9014 // Note that GC roots are not affected by heap poisoning, so we do 9015 // not need to do anything special for this here. 9016 SlowPathCodeARMVIXL* slow_path = 9017 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARMVIXL(instruction, out, root); 9018 AddSlowPath(slow_path); 9019 9020 __ B(slow_path->GetEntryLabel()); 9021 __ Bind(slow_path->GetExitLabel()); 9022 } 9023 9024 // Check if the desired_dispatch_info is supported. If it is, return it, 9025 // otherwise return a fall-back info that should be used instead. 9026 HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARMVIXL::GetSupportedInvokeStaticOrDirectDispatch( 9027 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 9028 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 9029 return desired_dispatch_info; 9030 } 9031 9032 vixl32::Register CodeGeneratorARMVIXL::GetInvokeStaticOrDirectExtraParameter( 9033 HInvokeStaticOrDirect* invoke, vixl32::Register temp) { 9034 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 9035 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 9036 if (!invoke->GetLocations()->Intrinsified()) { 9037 return RegisterFrom(location); 9038 } 9039 // For intrinsics we allow any location, so it may be on the stack. 9040 if (!location.IsRegister()) { 9041 GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, location.GetStackIndex()); 9042 return temp; 9043 } 9044 // For register locations, check if the register was saved. If so, get it from the stack. 9045 // Note: There is a chance that the register was saved but not overwritten, so we could 9046 // save one load. However, since this is just an intrinsic slow path we prefer this 9047 // simple and more robust approach rather that trying to determine if that's the case. 9048 SlowPathCode* slow_path = GetCurrentSlowPath(); 9049 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(RegisterFrom(location).GetCode())) { 9050 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(RegisterFrom(location).GetCode()); 9051 GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, stack_offset); 9052 return temp; 9053 } 9054 return RegisterFrom(location); 9055 } 9056 9057 void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( 9058 HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { 9059 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 9060 switch (invoke->GetMethodLoadKind()) { 9061 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 9062 uint32_t offset = 9063 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 9064 // temp = thread->string_init_entrypoint 9065 GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, offset); 9066 break; 9067 } 9068 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 9069 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 9070 break; 9071 case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: { 9072 DCHECK(GetCompilerOptions().IsBootImage()); 9073 PcRelativePatchInfo* labels = NewPcRelativeMethodPatch(invoke->GetTargetMethod()); 9074 vixl32::Register temp_reg = RegisterFrom(temp); 9075 EmitMovwMovtPlaceholder(labels, temp_reg); 9076 break; 9077 } 9078 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 9079 __ Mov(RegisterFrom(temp), Operand::From(invoke->GetMethodAddress())); 9080 break; 9081 case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { 9082 PcRelativePatchInfo* labels = NewMethodBssEntryPatch( 9083 MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); 9084 vixl32::Register temp_reg = RegisterFrom(temp); 9085 EmitMovwMovtPlaceholder(labels, temp_reg); 9086 GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset*/ 0); 9087 break; 9088 } 9089 case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { 9090 GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); 9091 return; // No code pointer retrieval; the runtime performs the call directly. 9092 } 9093 } 9094 9095 switch (invoke->GetCodePtrLocation()) { 9096 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 9097 { 9098 // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. 9099 ExactAssemblyScope aas(GetVIXLAssembler(), 9100 vixl32::k32BitT32InstructionSizeInBytes, 9101 CodeBufferCheckScope::kMaximumSize); 9102 __ bl(GetFrameEntryLabel()); 9103 RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); 9104 } 9105 break; 9106 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 9107 // LR = callee_method->entry_point_from_quick_compiled_code_ 9108 GetAssembler()->LoadFromOffset( 9109 kLoadWord, 9110 lr, 9111 RegisterFrom(callee_method), 9112 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 9113 { 9114 // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. 9115 // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. 9116 ExactAssemblyScope aas(GetVIXLAssembler(), 9117 vixl32::k16BitT32InstructionSizeInBytes, 9118 CodeBufferCheckScope::kExactSize); 9119 // LR() 9120 __ blx(lr); 9121 RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); 9122 } 9123 break; 9124 } 9125 9126 DCHECK(!IsLeafMethod()); 9127 } 9128 9129 void CodeGeneratorARMVIXL::GenerateVirtualCall( 9130 HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { 9131 vixl32::Register temp = RegisterFrom(temp_location); 9132 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 9133 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 9134 9135 // Use the calling convention instead of the location of the receiver, as 9136 // intrinsics may have put the receiver in a different register. In the intrinsics 9137 // slow path, the arguments have been moved to the right place, so here we are 9138 // guaranteed that the receiver is the first register of the calling convention. 9139 InvokeDexCallingConventionARMVIXL calling_convention; 9140 vixl32::Register receiver = calling_convention.GetRegisterAt(0); 9141 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 9142 { 9143 // Make sure the pc is recorded immediately after the `ldr` instruction. 9144 ExactAssemblyScope aas(GetVIXLAssembler(), 9145 vixl32::kMaxInstructionSizeInBytes, 9146 CodeBufferCheckScope::kMaximumSize); 9147 // /* HeapReference<Class> */ temp = receiver->klass_ 9148 __ ldr(temp, MemOperand(receiver, class_offset)); 9149 MaybeRecordImplicitNullCheck(invoke); 9150 } 9151 // Instead of simply (possibly) unpoisoning `temp` here, we should 9152 // emit a read barrier for the previous class reference load. 9153 // However this is not required in practice, as this is an 9154 // intermediate/temporary reference and because the current 9155 // concurrent copying collector keeps the from-space memory 9156 // intact/accessible until the end of the marking phase (the 9157 // concurrent copying collector may not in the future). 9158 GetAssembler()->MaybeUnpoisonHeapReference(temp); 9159 9160 // temp = temp->GetMethodAt(method_offset); 9161 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 9162 kArmPointerSize).Int32Value(); 9163 GetAssembler()->LoadFromOffset(kLoadWord, temp, temp, method_offset); 9164 // LR = temp->GetEntryPoint(); 9165 GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, entry_point); 9166 { 9167 // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. 9168 // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. 9169 ExactAssemblyScope aas(GetVIXLAssembler(), 9170 vixl32::k16BitT32InstructionSizeInBytes, 9171 CodeBufferCheckScope::kExactSize); 9172 // LR(); 9173 __ blx(lr); 9174 RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); 9175 } 9176 } 9177 9178 CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeMethodPatch( 9179 MethodReference target_method) { 9180 return NewPcRelativePatch(*target_method.dex_file, 9181 target_method.dex_method_index, 9182 &pc_relative_method_patches_); 9183 } 9184 9185 CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewMethodBssEntryPatch( 9186 MethodReference target_method) { 9187 return NewPcRelativePatch(*target_method.dex_file, 9188 target_method.dex_method_index, 9189 &method_bss_entry_patches_); 9190 } 9191 9192 CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeTypePatch( 9193 const DexFile& dex_file, dex::TypeIndex type_index) { 9194 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 9195 } 9196 9197 CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewTypeBssEntryPatch( 9198 const DexFile& dex_file, dex::TypeIndex type_index) { 9199 return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); 9200 } 9201 9202 CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeStringPatch( 9203 const DexFile& dex_file, dex::StringIndex string_index) { 9204 return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); 9205 } 9206 9207 CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativePatch( 9208 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 9209 patches->emplace_back(dex_file, offset_or_index); 9210 return &patches->back(); 9211 } 9212 9213 vixl::aarch32::Label* CodeGeneratorARMVIXL::NewBakerReadBarrierPatch(uint32_t custom_data) { 9214 baker_read_barrier_patches_.emplace_back(custom_data); 9215 return &baker_read_barrier_patches_.back().label; 9216 } 9217 9218 VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageAddressLiteral(uint32_t address) { 9219 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); 9220 } 9221 9222 VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateJitStringLiteral( 9223 const DexFile& dex_file, 9224 dex::StringIndex string_index, 9225 Handle<mirror::String> handle) { 9226 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), 9227 reinterpret_cast64<uint64_t>(handle.GetReference())); 9228 return jit_string_patches_.GetOrCreate( 9229 StringReference(&dex_file, string_index), 9230 [this]() { 9231 return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); 9232 }); 9233 } 9234 9235 VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateJitClassLiteral(const DexFile& dex_file, 9236 dex::TypeIndex type_index, 9237 Handle<mirror::Class> handle) { 9238 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), 9239 reinterpret_cast64<uint64_t>(handle.GetReference())); 9240 return jit_class_patches_.GetOrCreate( 9241 TypeReference(&dex_file, type_index), 9242 [this]() { 9243 return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); 9244 }); 9245 } 9246 9247 template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 9248 inline void CodeGeneratorARMVIXL::EmitPcRelativeLinkerPatches( 9249 const ArenaDeque<PcRelativePatchInfo>& infos, 9250 ArenaVector<LinkerPatch>* linker_patches) { 9251 for (const PcRelativePatchInfo& info : infos) { 9252 const DexFile& dex_file = info.target_dex_file; 9253 size_t offset_or_index = info.offset_or_index; 9254 DCHECK(info.add_pc_label.IsBound()); 9255 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.GetLocation()); 9256 // Add MOVW patch. 9257 DCHECK(info.movw_label.IsBound()); 9258 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.GetLocation()); 9259 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 9260 // Add MOVT patch. 9261 DCHECK(info.movt_label.IsBound()); 9262 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.GetLocation()); 9263 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 9264 } 9265 } 9266 9267 void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 9268 DCHECK(linker_patches->empty()); 9269 size_t size = 9270 /* MOVW+MOVT for each entry */ 2u * pc_relative_method_patches_.size() + 9271 /* MOVW+MOVT for each entry */ 2u * method_bss_entry_patches_.size() + 9272 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 9273 /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + 9274 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 9275 baker_read_barrier_patches_.size(); 9276 linker_patches->reserve(size); 9277 if (GetCompilerOptions().IsBootImage()) { 9278 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, 9279 linker_patches); 9280 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 9281 linker_patches); 9282 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 9283 linker_patches); 9284 } else { 9285 DCHECK(pc_relative_method_patches_.empty()); 9286 DCHECK(pc_relative_type_patches_.empty()); 9287 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 9288 linker_patches); 9289 } 9290 EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, 9291 linker_patches); 9292 EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, 9293 linker_patches); 9294 for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { 9295 linker_patches->push_back(LinkerPatch::BakerReadBarrierBranchPatch(info.label.GetLocation(), 9296 info.custom_data)); 9297 } 9298 DCHECK_EQ(size, linker_patches->size()); 9299 } 9300 9301 VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateUint32Literal( 9302 uint32_t value, 9303 Uint32ToLiteralMap* map) { 9304 return map->GetOrCreate( 9305 value, 9306 [this, value]() { 9307 return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ value); 9308 }); 9309 } 9310 9311 void LocationsBuilderARMVIXL::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 9312 LocationSummary* locations = 9313 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 9314 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 9315 Location::RequiresRegister()); 9316 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 9317 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 9318 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 9319 } 9320 9321 void InstructionCodeGeneratorARMVIXL::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 9322 vixl32::Register res = OutputRegister(instr); 9323 vixl32::Register accumulator = 9324 InputRegisterAt(instr, HMultiplyAccumulate::kInputAccumulatorIndex); 9325 vixl32::Register mul_left = 9326 InputRegisterAt(instr, HMultiplyAccumulate::kInputMulLeftIndex); 9327 vixl32::Register mul_right = 9328 InputRegisterAt(instr, HMultiplyAccumulate::kInputMulRightIndex); 9329 9330 if (instr->GetOpKind() == HInstruction::kAdd) { 9331 __ Mla(res, mul_left, mul_right, accumulator); 9332 } else { 9333 __ Mls(res, mul_left, mul_right, accumulator); 9334 } 9335 } 9336 9337 void LocationsBuilderARMVIXL::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 9338 // Nothing to do, this should be removed during prepare for register allocator. 9339 LOG(FATAL) << "Unreachable"; 9340 } 9341 9342 void InstructionCodeGeneratorARMVIXL::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 9343 // Nothing to do, this should be removed during prepare for register allocator. 9344 LOG(FATAL) << "Unreachable"; 9345 } 9346 9347 // Simple implementation of packed switch - generate cascaded compare/jumps. 9348 void LocationsBuilderARMVIXL::VisitPackedSwitch(HPackedSwitch* switch_instr) { 9349 LocationSummary* locations = 9350 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 9351 locations->SetInAt(0, Location::RequiresRegister()); 9352 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 9353 codegen_->GetAssembler()->GetVIXLAssembler()->IsUsingT32()) { 9354 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 9355 if (switch_instr->GetStartValue() != 0) { 9356 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 9357 } 9358 } 9359 } 9360 9361 // TODO(VIXL): Investigate and reach the parity with old arm codegen. 9362 void InstructionCodeGeneratorARMVIXL::VisitPackedSwitch(HPackedSwitch* switch_instr) { 9363 int32_t lower_bound = switch_instr->GetStartValue(); 9364 uint32_t num_entries = switch_instr->GetNumEntries(); 9365 LocationSummary* locations = switch_instr->GetLocations(); 9366 vixl32::Register value_reg = InputRegisterAt(switch_instr, 0); 9367 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 9368 9369 if (num_entries <= kPackedSwitchCompareJumpThreshold || 9370 !codegen_->GetAssembler()->GetVIXLAssembler()->IsUsingT32()) { 9371 // Create a series of compare/jumps. 9372 UseScratchRegisterScope temps(GetVIXLAssembler()); 9373 vixl32::Register temp_reg = temps.Acquire(); 9374 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 9375 // the immediate, because IP is used as the destination register. For the other 9376 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 9377 // and they can be encoded in the instruction without making use of IP register. 9378 __ Adds(temp_reg, value_reg, -lower_bound); 9379 9380 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 9381 // Jump to successors[0] if value == lower_bound. 9382 __ B(eq, codegen_->GetLabelOf(successors[0])); 9383 int32_t last_index = 0; 9384 for (; num_entries - last_index > 2; last_index += 2) { 9385 __ Adds(temp_reg, temp_reg, -2); 9386 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 9387 __ B(lo, codegen_->GetLabelOf(successors[last_index + 1])); 9388 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 9389 __ B(eq, codegen_->GetLabelOf(successors[last_index + 2])); 9390 } 9391 if (num_entries - last_index == 2) { 9392 // The last missing case_value. 9393 __ Cmp(temp_reg, 1); 9394 __ B(eq, codegen_->GetLabelOf(successors[last_index + 1])); 9395 } 9396 9397 // And the default for any other value. 9398 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 9399 __ B(codegen_->GetLabelOf(default_block)); 9400 } 9401 } else { 9402 // Create a table lookup. 9403 vixl32::Register table_base = RegisterFrom(locations->GetTemp(0)); 9404 9405 JumpTableARMVIXL* jump_table = codegen_->CreateJumpTable(switch_instr); 9406 9407 // Remove the bias. 9408 vixl32::Register key_reg; 9409 if (lower_bound != 0) { 9410 key_reg = RegisterFrom(locations->GetTemp(1)); 9411 __ Sub(key_reg, value_reg, lower_bound); 9412 } else { 9413 key_reg = value_reg; 9414 } 9415 9416 // Check whether the value is in the table, jump to default block if not. 9417 __ Cmp(key_reg, num_entries - 1); 9418 __ B(hi, codegen_->GetLabelOf(default_block)); 9419 9420 UseScratchRegisterScope temps(GetVIXLAssembler()); 9421 vixl32::Register jump_offset = temps.Acquire(); 9422 9423 // Load jump offset from the table. 9424 { 9425 const size_t jump_size = switch_instr->GetNumEntries() * sizeof(int32_t); 9426 ExactAssemblyScope aas(GetVIXLAssembler(), 9427 (vixl32::kMaxInstructionSizeInBytes * 4) + jump_size, 9428 CodeBufferCheckScope::kMaximumSize); 9429 __ adr(table_base, jump_table->GetTableStartLabel()); 9430 __ ldr(jump_offset, MemOperand(table_base, key_reg, vixl32::LSL, 2)); 9431 9432 // Jump to target block by branching to table_base(pc related) + offset. 9433 vixl32::Register target_address = table_base; 9434 __ add(target_address, table_base, jump_offset); 9435 __ bx(target_address); 9436 9437 jump_table->EmitTable(codegen_); 9438 } 9439 } 9440 } 9441 9442 // Copy the result of a call into the given target. 9443 void CodeGeneratorARMVIXL::MoveFromReturnRegister(Location trg, Primitive::Type type) { 9444 if (!trg.IsValid()) { 9445 DCHECK_EQ(type, Primitive::kPrimVoid); 9446 return; 9447 } 9448 9449 DCHECK_NE(type, Primitive::kPrimVoid); 9450 9451 Location return_loc = InvokeDexCallingConventionVisitorARMVIXL().GetReturnLocation(type); 9452 if (return_loc.Equals(trg)) { 9453 return; 9454 } 9455 9456 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 9457 // with the last branch. 9458 if (type == Primitive::kPrimLong) { 9459 TODO_VIXL32(FATAL); 9460 } else if (type == Primitive::kPrimDouble) { 9461 TODO_VIXL32(FATAL); 9462 } else { 9463 // Let the parallel move resolver take care of all of this. 9464 HParallelMove parallel_move(GetGraph()->GetArena()); 9465 parallel_move.AddMove(return_loc, trg, type, nullptr); 9466 GetMoveResolver()->EmitNativeCode(¶llel_move); 9467 } 9468 } 9469 9470 void LocationsBuilderARMVIXL::VisitClassTableGet(HClassTableGet* instruction) { 9471 LocationSummary* locations = 9472 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 9473 locations->SetInAt(0, Location::RequiresRegister()); 9474 locations->SetOut(Location::RequiresRegister()); 9475 } 9476 9477 void InstructionCodeGeneratorARMVIXL::VisitClassTableGet(HClassTableGet* instruction) { 9478 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 9479 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 9480 instruction->GetIndex(), kArmPointerSize).SizeValue(); 9481 GetAssembler()->LoadFromOffset(kLoadWord, 9482 OutputRegister(instruction), 9483 InputRegisterAt(instruction, 0), 9484 method_offset); 9485 } else { 9486 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 9487 instruction->GetIndex(), kArmPointerSize)); 9488 GetAssembler()->LoadFromOffset(kLoadWord, 9489 OutputRegister(instruction), 9490 InputRegisterAt(instruction, 0), 9491 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 9492 GetAssembler()->LoadFromOffset(kLoadWord, 9493 OutputRegister(instruction), 9494 OutputRegister(instruction), 9495 method_offset); 9496 } 9497 } 9498 9499 static void PatchJitRootUse(uint8_t* code, 9500 const uint8_t* roots_data, 9501 VIXLUInt32Literal* literal, 9502 uint64_t index_in_table) { 9503 DCHECK(literal->IsBound()); 9504 uint32_t literal_offset = literal->GetLocation(); 9505 uintptr_t address = 9506 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 9507 uint8_t* data = code + literal_offset; 9508 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 9509 } 9510 9511 void CodeGeneratorARMVIXL::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 9512 for (const auto& entry : jit_string_patches_) { 9513 const StringReference& string_reference = entry.first; 9514 VIXLUInt32Literal* table_entry_literal = entry.second; 9515 const auto it = jit_string_roots_.find(string_reference); 9516 DCHECK(it != jit_string_roots_.end()); 9517 uint64_t index_in_table = it->second; 9518 PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table); 9519 } 9520 for (const auto& entry : jit_class_patches_) { 9521 const TypeReference& type_reference = entry.first; 9522 VIXLUInt32Literal* table_entry_literal = entry.second; 9523 const auto it = jit_class_roots_.find(type_reference); 9524 DCHECK(it != jit_class_roots_.end()); 9525 uint64_t index_in_table = it->second; 9526 PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table); 9527 } 9528 } 9529 9530 void CodeGeneratorARMVIXL::EmitMovwMovtPlaceholder( 9531 CodeGeneratorARMVIXL::PcRelativePatchInfo* labels, 9532 vixl32::Register out) { 9533 ExactAssemblyScope aas(GetVIXLAssembler(), 9534 3 * vixl32::kMaxInstructionSizeInBytes, 9535 CodeBufferCheckScope::kMaximumSize); 9536 // TODO(VIXL): Think about using mov instead of movw. 9537 __ bind(&labels->movw_label); 9538 __ movw(out, /* placeholder */ 0u); 9539 __ bind(&labels->movt_label); 9540 __ movt(out, /* placeholder */ 0u); 9541 __ bind(&labels->add_pc_label); 9542 __ add(out, out, pc); 9543 } 9544 9545 #undef __ 9546 #undef QUICK_ENTRY_POINT 9547 #undef TODO_VIXL32 9548 9549 } // namespace arm 9550 } // namespace art 9551