1 // Copyright 2014 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/code-generator.h" 6 7 #include "src/arm64/frames-arm64.h" 8 #include "src/arm64/macro-assembler-arm64.h" 9 #include "src/compilation-info.h" 10 #include "src/compiler/code-generator-impl.h" 11 #include "src/compiler/gap-resolver.h" 12 #include "src/compiler/node-matchers.h" 13 #include "src/compiler/osr.h" 14 15 namespace v8 { 16 namespace internal { 17 namespace compiler { 18 19 #define __ masm()-> 20 21 22 // Adds Arm64-specific methods to convert InstructionOperands. 23 class Arm64OperandConverter final : public InstructionOperandConverter { 24 public: 25 Arm64OperandConverter(CodeGenerator* gen, Instruction* instr) 26 : InstructionOperandConverter(gen, instr) {} 27 28 DoubleRegister InputFloat32Register(size_t index) { 29 return InputDoubleRegister(index).S(); 30 } 31 32 DoubleRegister InputFloat64Register(size_t index) { 33 return InputDoubleRegister(index); 34 } 35 36 CPURegister InputFloat32OrZeroRegister(size_t index) { 37 if (instr_->InputAt(index)->IsImmediate()) { 38 DCHECK(bit_cast<int32_t>(InputFloat32(index)) == 0); 39 return wzr; 40 } 41 DCHECK(instr_->InputAt(index)->IsFPRegister()); 42 return InputDoubleRegister(index).S(); 43 } 44 45 CPURegister InputFloat64OrZeroRegister(size_t index) { 46 if (instr_->InputAt(index)->IsImmediate()) { 47 DCHECK(bit_cast<int64_t>(InputDouble(index)) == 0); 48 return xzr; 49 } 50 DCHECK(instr_->InputAt(index)->IsDoubleRegister()); 51 return InputDoubleRegister(index); 52 } 53 54 size_t OutputCount() { return instr_->OutputCount(); } 55 56 DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); } 57 58 DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); } 59 60 Register InputRegister32(size_t index) { 61 return ToRegister(instr_->InputAt(index)).W(); 62 } 63 64 Register InputOrZeroRegister32(size_t index) { 65 DCHECK(instr_->InputAt(index)->IsRegister() || 66 (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0))); 67 if (instr_->InputAt(index)->IsImmediate()) { 68 return wzr; 69 } 70 return InputRegister32(index); 71 } 72 73 Register InputRegister64(size_t index) { return InputRegister(index); } 74 75 Register InputOrZeroRegister64(size_t index) { 76 DCHECK(instr_->InputAt(index)->IsRegister() || 77 (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0))); 78 if (instr_->InputAt(index)->IsImmediate()) { 79 return xzr; 80 } 81 return InputRegister64(index); 82 } 83 84 Operand InputImmediate(size_t index) { 85 return ToImmediate(instr_->InputAt(index)); 86 } 87 88 Operand InputOperand(size_t index) { 89 return ToOperand(instr_->InputAt(index)); 90 } 91 92 Operand InputOperand64(size_t index) { return InputOperand(index); } 93 94 Operand InputOperand32(size_t index) { 95 return ToOperand32(instr_->InputAt(index)); 96 } 97 98 Register OutputRegister64() { return OutputRegister(); } 99 100 Register OutputRegister32() { return ToRegister(instr_->Output()).W(); } 101 102 Operand InputOperand2_32(size_t index) { 103 switch (AddressingModeField::decode(instr_->opcode())) { 104 case kMode_None: 105 return InputOperand32(index); 106 case kMode_Operand2_R_LSL_I: 107 return Operand(InputRegister32(index), LSL, InputInt5(index + 1)); 108 case kMode_Operand2_R_LSR_I: 109 return Operand(InputRegister32(index), LSR, InputInt5(index + 1)); 110 case kMode_Operand2_R_ASR_I: 111 return Operand(InputRegister32(index), ASR, InputInt5(index + 1)); 112 case kMode_Operand2_R_ROR_I: 113 return Operand(InputRegister32(index), ROR, InputInt5(index + 1)); 114 case kMode_Operand2_R_UXTB: 115 return Operand(InputRegister32(index), UXTB); 116 case kMode_Operand2_R_UXTH: 117 return Operand(InputRegister32(index), UXTH); 118 case kMode_Operand2_R_SXTB: 119 return Operand(InputRegister32(index), SXTB); 120 case kMode_Operand2_R_SXTH: 121 return Operand(InputRegister32(index), SXTH); 122 case kMode_Operand2_R_SXTW: 123 return Operand(InputRegister32(index), SXTW); 124 case kMode_MRI: 125 case kMode_MRR: 126 break; 127 } 128 UNREACHABLE(); 129 return Operand(-1); 130 } 131 132 Operand InputOperand2_64(size_t index) { 133 switch (AddressingModeField::decode(instr_->opcode())) { 134 case kMode_None: 135 return InputOperand64(index); 136 case kMode_Operand2_R_LSL_I: 137 return Operand(InputRegister64(index), LSL, InputInt6(index + 1)); 138 case kMode_Operand2_R_LSR_I: 139 return Operand(InputRegister64(index), LSR, InputInt6(index + 1)); 140 case kMode_Operand2_R_ASR_I: 141 return Operand(InputRegister64(index), ASR, InputInt6(index + 1)); 142 case kMode_Operand2_R_ROR_I: 143 return Operand(InputRegister64(index), ROR, InputInt6(index + 1)); 144 case kMode_Operand2_R_UXTB: 145 return Operand(InputRegister64(index), UXTB); 146 case kMode_Operand2_R_UXTH: 147 return Operand(InputRegister64(index), UXTH); 148 case kMode_Operand2_R_SXTB: 149 return Operand(InputRegister64(index), SXTB); 150 case kMode_Operand2_R_SXTH: 151 return Operand(InputRegister64(index), SXTH); 152 case kMode_Operand2_R_SXTW: 153 return Operand(InputRegister64(index), SXTW); 154 case kMode_MRI: 155 case kMode_MRR: 156 break; 157 } 158 UNREACHABLE(); 159 return Operand(-1); 160 } 161 162 MemOperand MemoryOperand(size_t* first_index) { 163 const size_t index = *first_index; 164 switch (AddressingModeField::decode(instr_->opcode())) { 165 case kMode_None: 166 case kMode_Operand2_R_LSR_I: 167 case kMode_Operand2_R_ASR_I: 168 case kMode_Operand2_R_ROR_I: 169 case kMode_Operand2_R_UXTB: 170 case kMode_Operand2_R_UXTH: 171 case kMode_Operand2_R_SXTB: 172 case kMode_Operand2_R_SXTH: 173 case kMode_Operand2_R_SXTW: 174 break; 175 case kMode_Operand2_R_LSL_I: 176 *first_index += 3; 177 return MemOperand(InputRegister(index + 0), InputRegister(index + 1), 178 LSL, InputInt32(index + 2)); 179 case kMode_MRI: 180 *first_index += 2; 181 return MemOperand(InputRegister(index + 0), InputInt32(index + 1)); 182 case kMode_MRR: 183 *first_index += 2; 184 return MemOperand(InputRegister(index + 0), InputRegister(index + 1)); 185 } 186 UNREACHABLE(); 187 return MemOperand(no_reg); 188 } 189 190 MemOperand MemoryOperand(size_t first_index = 0) { 191 return MemoryOperand(&first_index); 192 } 193 194 Operand ToOperand(InstructionOperand* op) { 195 if (op->IsRegister()) { 196 return Operand(ToRegister(op)); 197 } 198 return ToImmediate(op); 199 } 200 201 Operand ToOperand32(InstructionOperand* op) { 202 if (op->IsRegister()) { 203 return Operand(ToRegister(op).W()); 204 } 205 return ToImmediate(op); 206 } 207 208 Operand ToImmediate(InstructionOperand* operand) { 209 Constant constant = ToConstant(operand); 210 switch (constant.type()) { 211 case Constant::kInt32: 212 if (RelocInfo::IsWasmSizeReference(constant.rmode())) { 213 return Operand(constant.ToInt32(), constant.rmode()); 214 } else { 215 return Operand(constant.ToInt32()); 216 } 217 case Constant::kInt64: 218 if (RelocInfo::IsWasmPtrReference(constant.rmode())) { 219 return Operand(constant.ToInt64(), constant.rmode()); 220 } else { 221 DCHECK(!RelocInfo::IsWasmSizeReference(constant.rmode())); 222 return Operand(constant.ToInt64()); 223 } 224 case Constant::kFloat32: 225 return Operand( 226 isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED)); 227 case Constant::kFloat64: 228 return Operand( 229 isolate()->factory()->NewNumber(constant.ToFloat64(), TENURED)); 230 case Constant::kExternalReference: 231 return Operand(constant.ToExternalReference()); 232 case Constant::kHeapObject: 233 return Operand(constant.ToHeapObject()); 234 case Constant::kRpoNumber: 235 UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64. 236 break; 237 } 238 UNREACHABLE(); 239 return Operand(-1); 240 } 241 242 MemOperand ToMemOperand(InstructionOperand* op, MacroAssembler* masm) const { 243 DCHECK_NOT_NULL(op); 244 DCHECK(op->IsStackSlot() || op->IsFPStackSlot()); 245 return SlotToMemOperand(AllocatedOperand::cast(op)->index(), masm); 246 } 247 248 MemOperand SlotToMemOperand(int slot, MacroAssembler* masm) const { 249 FrameOffset offset = frame_access_state()->GetFrameOffset(slot); 250 if (offset.from_frame_pointer()) { 251 int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset(); 252 // Convert FP-offsets to SP-offsets if it results in better code. 253 if (Assembler::IsImmLSUnscaled(from_sp) || 254 Assembler::IsImmLSScaled(from_sp, LSDoubleWord)) { 255 offset = FrameOffset::FromStackPointer(from_sp); 256 } 257 } 258 return MemOperand(offset.from_stack_pointer() ? masm->StackPointer() : fp, 259 offset.offset()); 260 } 261 }; 262 263 264 namespace { 265 266 class OutOfLineLoadNaN32 final : public OutOfLineCode { 267 public: 268 OutOfLineLoadNaN32(CodeGenerator* gen, DoubleRegister result) 269 : OutOfLineCode(gen), result_(result) {} 270 271 void Generate() final { 272 __ Fmov(result_, std::numeric_limits<float>::quiet_NaN()); 273 } 274 275 private: 276 DoubleRegister const result_; 277 }; 278 279 280 class OutOfLineLoadNaN64 final : public OutOfLineCode { 281 public: 282 OutOfLineLoadNaN64(CodeGenerator* gen, DoubleRegister result) 283 : OutOfLineCode(gen), result_(result) {} 284 285 void Generate() final { 286 __ Fmov(result_, std::numeric_limits<double>::quiet_NaN()); 287 } 288 289 private: 290 DoubleRegister const result_; 291 }; 292 293 294 class OutOfLineLoadZero final : public OutOfLineCode { 295 public: 296 OutOfLineLoadZero(CodeGenerator* gen, Register result) 297 : OutOfLineCode(gen), result_(result) {} 298 299 void Generate() final { __ Mov(result_, 0); } 300 301 private: 302 Register const result_; 303 }; 304 305 306 class OutOfLineRecordWrite final : public OutOfLineCode { 307 public: 308 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index, 309 Register value, Register scratch0, Register scratch1, 310 RecordWriteMode mode, 311 UnwindingInfoWriter* unwinding_info_writer) 312 : OutOfLineCode(gen), 313 object_(object), 314 index_(index), 315 value_(value), 316 scratch0_(scratch0), 317 scratch1_(scratch1), 318 mode_(mode), 319 must_save_lr_(!gen->frame_access_state()->has_frame()), 320 unwinding_info_writer_(unwinding_info_writer) {} 321 322 void Generate() final { 323 if (mode_ > RecordWriteMode::kValueIsPointer) { 324 __ JumpIfSmi(value_, exit()); 325 } 326 __ CheckPageFlagClear(value_, scratch0_, 327 MemoryChunk::kPointersToHereAreInterestingMask, 328 exit()); 329 RememberedSetAction const remembered_set_action = 330 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET 331 : OMIT_REMEMBERED_SET; 332 SaveFPRegsMode const save_fp_mode = 333 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; 334 if (must_save_lr_) { 335 // We need to save and restore lr if the frame was elided. 336 __ Push(lr); 337 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(), 338 __ StackPointer()); 339 } 340 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_, 341 remembered_set_action, save_fp_mode); 342 __ Add(scratch1_, object_, index_); 343 __ CallStub(&stub); 344 if (must_save_lr_) { 345 __ Pop(lr); 346 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset()); 347 } 348 } 349 350 private: 351 Register const object_; 352 Operand const index_; 353 Register const value_; 354 Register const scratch0_; 355 Register const scratch1_; 356 RecordWriteMode const mode_; 357 bool must_save_lr_; 358 UnwindingInfoWriter* const unwinding_info_writer_; 359 }; 360 361 362 Condition FlagsConditionToCondition(FlagsCondition condition) { 363 switch (condition) { 364 case kEqual: 365 return eq; 366 case kNotEqual: 367 return ne; 368 case kSignedLessThan: 369 return lt; 370 case kSignedGreaterThanOrEqual: 371 return ge; 372 case kSignedLessThanOrEqual: 373 return le; 374 case kSignedGreaterThan: 375 return gt; 376 case kUnsignedLessThan: 377 return lo; 378 case kUnsignedGreaterThanOrEqual: 379 return hs; 380 case kUnsignedLessThanOrEqual: 381 return ls; 382 case kUnsignedGreaterThan: 383 return hi; 384 case kFloatLessThanOrUnordered: 385 return lt; 386 case kFloatGreaterThanOrEqual: 387 return ge; 388 case kFloatLessThanOrEqual: 389 return ls; 390 case kFloatGreaterThanOrUnordered: 391 return hi; 392 case kFloatLessThan: 393 return lo; 394 case kFloatGreaterThanOrEqualOrUnordered: 395 return hs; 396 case kFloatLessThanOrEqualOrUnordered: 397 return le; 398 case kFloatGreaterThan: 399 return gt; 400 case kOverflow: 401 return vs; 402 case kNotOverflow: 403 return vc; 404 case kUnorderedEqual: 405 case kUnorderedNotEqual: 406 break; 407 case kPositiveOrZero: 408 return pl; 409 case kNegative: 410 return mi; 411 } 412 UNREACHABLE(); 413 return nv; 414 } 415 416 } // namespace 417 418 #define ASSEMBLE_BOUNDS_CHECK(offset, length, out_of_bounds) \ 419 do { \ 420 if (length.IsImmediate() && \ 421 base::bits::IsPowerOfTwo64(length.ImmediateValue())) { \ 422 __ Tst(offset, ~(length.ImmediateValue() - 1)); \ 423 __ B(ne, out_of_bounds); \ 424 } else { \ 425 __ Cmp(offset, length); \ 426 __ B(hs, out_of_bounds); \ 427 } \ 428 } while (0) 429 430 #define ASSEMBLE_CHECKED_LOAD_FLOAT(width) \ 431 do { \ 432 auto result = i.OutputFloat##width##Register(); \ 433 auto buffer = i.InputRegister(0); \ 434 auto offset = i.InputRegister32(1); \ 435 auto length = i.InputOperand32(2); \ 436 auto ool = new (zone()) OutOfLineLoadNaN##width(this, result); \ 437 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \ 438 __ Ldr(result, MemOperand(buffer, offset, UXTW)); \ 439 __ Bind(ool->exit()); \ 440 } while (0) 441 442 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \ 443 do { \ 444 auto result = i.OutputRegister32(); \ 445 auto buffer = i.InputRegister(0); \ 446 auto offset = i.InputRegister32(1); \ 447 auto length = i.InputOperand32(2); \ 448 auto ool = new (zone()) OutOfLineLoadZero(this, result); \ 449 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \ 450 __ asm_instr(result, MemOperand(buffer, offset, UXTW)); \ 451 __ Bind(ool->exit()); \ 452 } while (0) 453 454 #define ASSEMBLE_CHECKED_LOAD_INTEGER_64(asm_instr) \ 455 do { \ 456 auto result = i.OutputRegister(); \ 457 auto buffer = i.InputRegister(0); \ 458 auto offset = i.InputRegister32(1); \ 459 auto length = i.InputOperand32(2); \ 460 auto ool = new (zone()) OutOfLineLoadZero(this, result); \ 461 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \ 462 __ asm_instr(result, MemOperand(buffer, offset, UXTW)); \ 463 __ Bind(ool->exit()); \ 464 } while (0) 465 466 #define ASSEMBLE_CHECKED_STORE_FLOAT(width) \ 467 do { \ 468 auto buffer = i.InputRegister(0); \ 469 auto offset = i.InputRegister32(1); \ 470 auto length = i.InputOperand32(2); \ 471 auto value = i.InputFloat##width##OrZeroRegister(3); \ 472 Label done; \ 473 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \ 474 __ Str(value, MemOperand(buffer, offset, UXTW)); \ 475 __ Bind(&done); \ 476 } while (0) 477 478 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \ 479 do { \ 480 auto buffer = i.InputRegister(0); \ 481 auto offset = i.InputRegister32(1); \ 482 auto length = i.InputOperand32(2); \ 483 auto value = i.InputOrZeroRegister32(3); \ 484 Label done; \ 485 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \ 486 __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \ 487 __ Bind(&done); \ 488 } while (0) 489 490 #define ASSEMBLE_CHECKED_STORE_INTEGER_64(asm_instr) \ 491 do { \ 492 auto buffer = i.InputRegister(0); \ 493 auto offset = i.InputRegister32(1); \ 494 auto length = i.InputOperand32(2); \ 495 auto value = i.InputOrZeroRegister64(3); \ 496 Label done; \ 497 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \ 498 __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \ 499 __ Bind(&done); \ 500 } while (0) 501 502 #define ASSEMBLE_SHIFT(asm_instr, width) \ 503 do { \ 504 if (instr->InputAt(1)->IsRegister()) { \ 505 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \ 506 i.InputRegister##width(1)); \ 507 } else { \ 508 uint32_t imm = \ 509 static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \ 510 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \ 511 imm % (width)); \ 512 } \ 513 } while (0) 514 515 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \ 516 do { \ 517 __ asm_instr(i.OutputRegister(), \ 518 MemOperand(i.InputRegister(0), i.InputRegister(1))); \ 519 __ Dmb(InnerShareable, BarrierAll); \ 520 } while (0) 521 522 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \ 523 do { \ 524 __ Dmb(InnerShareable, BarrierAll); \ 525 __ asm_instr(i.InputRegister(2), \ 526 MemOperand(i.InputRegister(0), i.InputRegister(1))); \ 527 __ Dmb(InnerShareable, BarrierAll); \ 528 } while (0) 529 530 #define ASSEMBLE_IEEE754_BINOP(name) \ 531 do { \ 532 FrameScope scope(masm(), StackFrame::MANUAL); \ 533 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 534 0, 2); \ 535 } while (0) 536 537 #define ASSEMBLE_IEEE754_UNOP(name) \ 538 do { \ 539 FrameScope scope(masm(), StackFrame::MANUAL); \ 540 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 541 0, 1); \ 542 } while (0) 543 544 void CodeGenerator::AssembleDeconstructFrame() { 545 const CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 546 if (descriptor->IsCFunctionCall() || descriptor->UseNativeStack()) { 547 __ Mov(csp, fp); 548 } else { 549 __ Mov(jssp, fp); 550 } 551 __ Pop(fp, lr); 552 553 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset()); 554 } 555 556 void CodeGenerator::AssemblePrepareTailCall() { 557 if (frame_access_state()->has_frame()) { 558 __ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset)); 559 __ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset)); 560 } 561 frame_access_state()->SetFrameAccessToSP(); 562 } 563 564 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg, 565 Register scratch1, 566 Register scratch2, 567 Register scratch3) { 568 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3)); 569 Label done; 570 571 // Check if current frame is an arguments adaptor frame. 572 __ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset)); 573 __ Cmp(scratch1, 574 Operand(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR))); 575 __ B(ne, &done); 576 577 // Load arguments count from current arguments adaptor frame (note, it 578 // does not include receiver). 579 Register caller_args_count_reg = scratch1; 580 __ Ldr(caller_args_count_reg, 581 MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset)); 582 __ SmiUntag(caller_args_count_reg); 583 584 ParameterCount callee_args_count(args_reg); 585 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2, 586 scratch3); 587 __ bind(&done); 588 } 589 590 namespace { 591 592 void AdjustStackPointerForTailCall(MacroAssembler* masm, 593 FrameAccessState* state, 594 int new_slot_above_sp, 595 bool allow_shrinkage = true) { 596 int current_sp_offset = state->GetSPToFPSlotCount() + 597 StandardFrameConstants::kFixedSlotCountAboveFp; 598 int stack_slot_delta = new_slot_above_sp - current_sp_offset; 599 if (stack_slot_delta > 0) { 600 masm->Claim(stack_slot_delta); 601 state->IncreaseSPDelta(stack_slot_delta); 602 } else if (allow_shrinkage && stack_slot_delta < 0) { 603 masm->Drop(-stack_slot_delta); 604 state->IncreaseSPDelta(stack_slot_delta); 605 } 606 } 607 608 } // namespace 609 610 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, 611 int first_unused_stack_slot) { 612 AdjustStackPointerForTailCall(masm(), frame_access_state(), 613 first_unused_stack_slot, false); 614 } 615 616 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr, 617 int first_unused_stack_slot) { 618 AdjustStackPointerForTailCall(masm(), frame_access_state(), 619 first_unused_stack_slot); 620 } 621 622 // Assembles an instruction after register allocation, producing machine code. 623 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( 624 Instruction* instr) { 625 Arm64OperandConverter i(this, instr); 626 InstructionCode opcode = instr->opcode(); 627 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode); 628 switch (arch_opcode) { 629 case kArchCallCodeObject: { 630 EnsureSpaceForLazyDeopt(); 631 if (instr->InputAt(0)->IsImmediate()) { 632 __ Call(Handle<Code>::cast(i.InputHeapObject(0)), 633 RelocInfo::CODE_TARGET); 634 } else { 635 Register target = i.InputRegister(0); 636 __ Add(target, target, Code::kHeaderSize - kHeapObjectTag); 637 __ Call(target); 638 } 639 RecordCallPosition(instr); 640 // TODO(titzer): this is ugly. JSSP should be a caller-save register 641 // in this case, but it is not possible to express in the register 642 // allocator. 643 CallDescriptor::Flags flags(MiscField::decode(opcode)); 644 if (flags & CallDescriptor::kRestoreJSSP) { 645 __ Ldr(jssp, MemOperand(csp)); 646 __ Mov(csp, jssp); 647 } 648 if (flags & CallDescriptor::kRestoreCSP) { 649 __ Mov(csp, jssp); 650 __ AssertCspAligned(); 651 } 652 frame_access_state()->ClearSPDelta(); 653 break; 654 } 655 case kArchTailCallCodeObjectFromJSFunction: 656 case kArchTailCallCodeObject: { 657 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) { 658 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 659 i.TempRegister(0), i.TempRegister(1), 660 i.TempRegister(2)); 661 } 662 if (instr->InputAt(0)->IsImmediate()) { 663 __ Jump(Handle<Code>::cast(i.InputHeapObject(0)), 664 RelocInfo::CODE_TARGET); 665 } else { 666 Register target = i.InputRegister(0); 667 __ Add(target, target, Code::kHeaderSize - kHeapObjectTag); 668 __ Jump(target); 669 } 670 unwinding_info_writer_.MarkBlockWillExit(); 671 frame_access_state()->ClearSPDelta(); 672 frame_access_state()->SetFrameAccessToDefault(); 673 break; 674 } 675 case kArchTailCallAddress: { 676 CHECK(!instr->InputAt(0)->IsImmediate()); 677 __ Jump(i.InputRegister(0)); 678 unwinding_info_writer_.MarkBlockWillExit(); 679 frame_access_state()->ClearSPDelta(); 680 frame_access_state()->SetFrameAccessToDefault(); 681 break; 682 } 683 case kArchCallJSFunction: { 684 EnsureSpaceForLazyDeopt(); 685 Register func = i.InputRegister(0); 686 if (FLAG_debug_code) { 687 // Check the function's context matches the context argument. 688 UseScratchRegisterScope scope(masm()); 689 Register temp = scope.AcquireX(); 690 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset)); 691 __ cmp(cp, temp); 692 __ Assert(eq, kWrongFunctionContext); 693 } 694 __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset)); 695 __ Call(x10); 696 RecordCallPosition(instr); 697 // TODO(titzer): this is ugly. JSSP should be a caller-save register 698 // in this case, but it is not possible to express in the register 699 // allocator. 700 CallDescriptor::Flags flags(MiscField::decode(opcode)); 701 if (flags & CallDescriptor::kRestoreJSSP) { 702 __ Ldr(jssp, MemOperand(csp)); 703 __ Mov(csp, jssp); 704 } 705 if (flags & CallDescriptor::kRestoreCSP) { 706 __ Mov(csp, jssp); 707 __ AssertCspAligned(); 708 } 709 frame_access_state()->ClearSPDelta(); 710 break; 711 } 712 case kArchTailCallJSFunctionFromJSFunction: { 713 Register func = i.InputRegister(0); 714 if (FLAG_debug_code) { 715 // Check the function's context matches the context argument. 716 UseScratchRegisterScope scope(masm()); 717 Register temp = scope.AcquireX(); 718 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset)); 719 __ cmp(cp, temp); 720 __ Assert(eq, kWrongFunctionContext); 721 } 722 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 723 i.TempRegister(0), i.TempRegister(1), 724 i.TempRegister(2)); 725 __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset)); 726 __ Jump(x10); 727 frame_access_state()->ClearSPDelta(); 728 frame_access_state()->SetFrameAccessToDefault(); 729 break; 730 } 731 case kArchPrepareCallCFunction: 732 // We don't need kArchPrepareCallCFunction on arm64 as the instruction 733 // selector already perform a Claim to reserve space on the stack and 734 // guarantee correct alignment of stack pointer. 735 UNREACHABLE(); 736 break; 737 case kArchPrepareTailCall: 738 AssemblePrepareTailCall(); 739 break; 740 case kArchCallCFunction: { 741 int const num_parameters = MiscField::decode(instr->opcode()); 742 if (instr->InputAt(0)->IsImmediate()) { 743 ExternalReference ref = i.InputExternalReference(0); 744 __ CallCFunction(ref, num_parameters, 0); 745 } else { 746 Register func = i.InputRegister(0); 747 __ CallCFunction(func, num_parameters, 0); 748 } 749 // CallCFunction only supports register arguments so we never need to call 750 // frame()->ClearOutgoingParameterSlots() here. 751 DCHECK(frame_access_state()->sp_delta() == 0); 752 break; 753 } 754 case kArchJmp: 755 AssembleArchJump(i.InputRpo(0)); 756 break; 757 case kArchTableSwitch: 758 AssembleArchTableSwitch(instr); 759 break; 760 case kArchLookupSwitch: 761 AssembleArchLookupSwitch(instr); 762 break; 763 case kArchDebugBreak: 764 __ Debug("kArchDebugBreak", 0, BREAK); 765 break; 766 case kArchComment: { 767 Address comment_string = i.InputExternalReference(0).address(); 768 __ RecordComment(reinterpret_cast<const char*>(comment_string)); 769 break; 770 } 771 case kArchNop: 772 case kArchThrowTerminator: 773 // don't emit code for nops. 774 break; 775 case kArchDeoptimize: { 776 int deopt_state_id = 777 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); 778 CodeGenResult result = 779 AssembleDeoptimizerCall(deopt_state_id, current_source_position_); 780 if (result != kSuccess) return result; 781 break; 782 } 783 case kArchRet: 784 AssembleReturn(instr->InputAt(0)); 785 break; 786 case kArchStackPointer: 787 __ mov(i.OutputRegister(), masm()->StackPointer()); 788 break; 789 case kArchFramePointer: 790 __ mov(i.OutputRegister(), fp); 791 break; 792 case kArchParentFramePointer: 793 if (frame_access_state()->has_frame()) { 794 __ ldr(i.OutputRegister(), MemOperand(fp, 0)); 795 } else { 796 __ mov(i.OutputRegister(), fp); 797 } 798 break; 799 case kArchTruncateDoubleToI: 800 __ TruncateDoubleToI(i.OutputRegister(), i.InputDoubleRegister(0)); 801 break; 802 case kArchStoreWithWriteBarrier: { 803 RecordWriteMode mode = 804 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode())); 805 AddressingMode addressing_mode = 806 AddressingModeField::decode(instr->opcode()); 807 Register object = i.InputRegister(0); 808 Operand index(0); 809 if (addressing_mode == kMode_MRI) { 810 index = Operand(i.InputInt64(1)); 811 } else { 812 DCHECK_EQ(addressing_mode, kMode_MRR); 813 index = Operand(i.InputRegister(1)); 814 } 815 Register value = i.InputRegister(2); 816 Register scratch0 = i.TempRegister(0); 817 Register scratch1 = i.TempRegister(1); 818 auto ool = new (zone()) 819 OutOfLineRecordWrite(this, object, index, value, scratch0, scratch1, 820 mode, &unwinding_info_writer_); 821 __ Str(value, MemOperand(object, index)); 822 __ CheckPageFlagSet(object, scratch0, 823 MemoryChunk::kPointersFromHereAreInterestingMask, 824 ool->entry()); 825 __ Bind(ool->exit()); 826 break; 827 } 828 case kArchStackSlot: { 829 FrameOffset offset = 830 frame_access_state()->GetFrameOffset(i.InputInt32(0)); 831 Register base; 832 if (offset.from_stack_pointer()) { 833 base = __ StackPointer(); 834 } else { 835 base = fp; 836 } 837 __ Add(i.OutputRegister(0), base, Operand(offset.offset())); 838 break; 839 } 840 case kIeee754Float64Acos: 841 ASSEMBLE_IEEE754_UNOP(acos); 842 break; 843 case kIeee754Float64Acosh: 844 ASSEMBLE_IEEE754_UNOP(acosh); 845 break; 846 case kIeee754Float64Asin: 847 ASSEMBLE_IEEE754_UNOP(asin); 848 break; 849 case kIeee754Float64Asinh: 850 ASSEMBLE_IEEE754_UNOP(asinh); 851 break; 852 case kIeee754Float64Atan: 853 ASSEMBLE_IEEE754_UNOP(atan); 854 break; 855 case kIeee754Float64Atanh: 856 ASSEMBLE_IEEE754_UNOP(atanh); 857 break; 858 case kIeee754Float64Atan2: 859 ASSEMBLE_IEEE754_BINOP(atan2); 860 break; 861 case kIeee754Float64Cos: 862 ASSEMBLE_IEEE754_UNOP(cos); 863 break; 864 case kIeee754Float64Cosh: 865 ASSEMBLE_IEEE754_UNOP(cosh); 866 break; 867 case kIeee754Float64Cbrt: 868 ASSEMBLE_IEEE754_UNOP(cbrt); 869 break; 870 case kIeee754Float64Exp: 871 ASSEMBLE_IEEE754_UNOP(exp); 872 break; 873 case kIeee754Float64Expm1: 874 ASSEMBLE_IEEE754_UNOP(expm1); 875 break; 876 case kIeee754Float64Log: 877 ASSEMBLE_IEEE754_UNOP(log); 878 break; 879 case kIeee754Float64Log1p: 880 ASSEMBLE_IEEE754_UNOP(log1p); 881 break; 882 case kIeee754Float64Log2: 883 ASSEMBLE_IEEE754_UNOP(log2); 884 break; 885 case kIeee754Float64Log10: 886 ASSEMBLE_IEEE754_UNOP(log10); 887 break; 888 case kIeee754Float64Pow: { 889 MathPowStub stub(isolate(), MathPowStub::DOUBLE); 890 __ CallStub(&stub); 891 break; 892 } 893 case kIeee754Float64Sin: 894 ASSEMBLE_IEEE754_UNOP(sin); 895 break; 896 case kIeee754Float64Sinh: 897 ASSEMBLE_IEEE754_UNOP(sinh); 898 break; 899 case kIeee754Float64Tan: 900 ASSEMBLE_IEEE754_UNOP(tan); 901 break; 902 case kIeee754Float64Tanh: 903 ASSEMBLE_IEEE754_UNOP(tanh); 904 break; 905 case kArm64Float32RoundDown: 906 __ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0)); 907 break; 908 case kArm64Float64RoundDown: 909 __ Frintm(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 910 break; 911 case kArm64Float32RoundUp: 912 __ Frintp(i.OutputFloat32Register(), i.InputFloat32Register(0)); 913 break; 914 case kArm64Float64RoundUp: 915 __ Frintp(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 916 break; 917 case kArm64Float64RoundTiesAway: 918 __ Frinta(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 919 break; 920 case kArm64Float32RoundTruncate: 921 __ Frintz(i.OutputFloat32Register(), i.InputFloat32Register(0)); 922 break; 923 case kArm64Float64RoundTruncate: 924 __ Frintz(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 925 break; 926 case kArm64Float32RoundTiesEven: 927 __ Frintn(i.OutputFloat32Register(), i.InputFloat32Register(0)); 928 break; 929 case kArm64Float64RoundTiesEven: 930 __ Frintn(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 931 break; 932 case kArm64Add: 933 if (FlagsModeField::decode(opcode) != kFlags_none) { 934 __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0), 935 i.InputOperand2_64(1)); 936 } else { 937 __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0), 938 i.InputOperand2_64(1)); 939 } 940 break; 941 case kArm64Add32: 942 if (FlagsModeField::decode(opcode) != kFlags_none) { 943 __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0), 944 i.InputOperand2_32(1)); 945 } else { 946 __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0), 947 i.InputOperand2_32(1)); 948 } 949 break; 950 case kArm64And: 951 if (FlagsModeField::decode(opcode) != kFlags_none) { 952 // The ands instruction only sets N and Z, so only the following 953 // conditions make sense. 954 DCHECK(FlagsConditionField::decode(opcode) == kEqual || 955 FlagsConditionField::decode(opcode) == kNotEqual || 956 FlagsConditionField::decode(opcode) == kPositiveOrZero || 957 FlagsConditionField::decode(opcode) == kNegative); 958 __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0), 959 i.InputOperand2_64(1)); 960 } else { 961 __ And(i.OutputRegister(), i.InputOrZeroRegister64(0), 962 i.InputOperand2_64(1)); 963 } 964 break; 965 case kArm64And32: 966 if (FlagsModeField::decode(opcode) != kFlags_none) { 967 // The ands instruction only sets N and Z, so only the following 968 // conditions make sense. 969 DCHECK(FlagsConditionField::decode(opcode) == kEqual || 970 FlagsConditionField::decode(opcode) == kNotEqual || 971 FlagsConditionField::decode(opcode) == kPositiveOrZero || 972 FlagsConditionField::decode(opcode) == kNegative); 973 __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0), 974 i.InputOperand2_32(1)); 975 } else { 976 __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0), 977 i.InputOperand2_32(1)); 978 } 979 break; 980 case kArm64Bic: 981 __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0), 982 i.InputOperand2_64(1)); 983 break; 984 case kArm64Bic32: 985 __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0), 986 i.InputOperand2_32(1)); 987 break; 988 case kArm64Mul: 989 __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 990 break; 991 case kArm64Mul32: 992 __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 993 break; 994 case kArm64Smull: 995 __ Smull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1)); 996 break; 997 case kArm64Umull: 998 __ Umull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1)); 999 break; 1000 case kArm64Madd: 1001 __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1), 1002 i.InputRegister(2)); 1003 break; 1004 case kArm64Madd32: 1005 __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1), 1006 i.InputRegister32(2)); 1007 break; 1008 case kArm64Msub: 1009 __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1), 1010 i.InputRegister(2)); 1011 break; 1012 case kArm64Msub32: 1013 __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1), 1014 i.InputRegister32(2)); 1015 break; 1016 case kArm64Mneg: 1017 __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 1018 break; 1019 case kArm64Mneg32: 1020 __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 1021 break; 1022 case kArm64Idiv: 1023 __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 1024 break; 1025 case kArm64Idiv32: 1026 __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 1027 break; 1028 case kArm64Udiv: 1029 __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 1030 break; 1031 case kArm64Udiv32: 1032 __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 1033 break; 1034 case kArm64Imod: { 1035 UseScratchRegisterScope scope(masm()); 1036 Register temp = scope.AcquireX(); 1037 __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1)); 1038 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0)); 1039 break; 1040 } 1041 case kArm64Imod32: { 1042 UseScratchRegisterScope scope(masm()); 1043 Register temp = scope.AcquireW(); 1044 __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1)); 1045 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1), 1046 i.InputRegister32(0)); 1047 break; 1048 } 1049 case kArm64Umod: { 1050 UseScratchRegisterScope scope(masm()); 1051 Register temp = scope.AcquireX(); 1052 __ Udiv(temp, i.InputRegister(0), i.InputRegister(1)); 1053 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0)); 1054 break; 1055 } 1056 case kArm64Umod32: { 1057 UseScratchRegisterScope scope(masm()); 1058 Register temp = scope.AcquireW(); 1059 __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1)); 1060 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1), 1061 i.InputRegister32(0)); 1062 break; 1063 } 1064 case kArm64Not: 1065 __ Mvn(i.OutputRegister(), i.InputOperand(0)); 1066 break; 1067 case kArm64Not32: 1068 __ Mvn(i.OutputRegister32(), i.InputOperand32(0)); 1069 break; 1070 case kArm64Or: 1071 __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0), 1072 i.InputOperand2_64(1)); 1073 break; 1074 case kArm64Or32: 1075 __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1076 i.InputOperand2_32(1)); 1077 break; 1078 case kArm64Orn: 1079 __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0), 1080 i.InputOperand2_64(1)); 1081 break; 1082 case kArm64Orn32: 1083 __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1084 i.InputOperand2_32(1)); 1085 break; 1086 case kArm64Eor: 1087 __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0), 1088 i.InputOperand2_64(1)); 1089 break; 1090 case kArm64Eor32: 1091 __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1092 i.InputOperand2_32(1)); 1093 break; 1094 case kArm64Eon: 1095 __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0), 1096 i.InputOperand2_64(1)); 1097 break; 1098 case kArm64Eon32: 1099 __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1100 i.InputOperand2_32(1)); 1101 break; 1102 case kArm64Sub: 1103 if (FlagsModeField::decode(opcode) != kFlags_none) { 1104 __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0), 1105 i.InputOperand2_64(1)); 1106 } else { 1107 __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0), 1108 i.InputOperand2_64(1)); 1109 } 1110 break; 1111 case kArm64Sub32: 1112 if (FlagsModeField::decode(opcode) != kFlags_none) { 1113 __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1114 i.InputOperand2_32(1)); 1115 } else { 1116 __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1117 i.InputOperand2_32(1)); 1118 } 1119 break; 1120 case kArm64Lsl: 1121 ASSEMBLE_SHIFT(Lsl, 64); 1122 break; 1123 case kArm64Lsl32: 1124 ASSEMBLE_SHIFT(Lsl, 32); 1125 break; 1126 case kArm64Lsr: 1127 ASSEMBLE_SHIFT(Lsr, 64); 1128 break; 1129 case kArm64Lsr32: 1130 ASSEMBLE_SHIFT(Lsr, 32); 1131 break; 1132 case kArm64Asr: 1133 ASSEMBLE_SHIFT(Asr, 64); 1134 break; 1135 case kArm64Asr32: 1136 ASSEMBLE_SHIFT(Asr, 32); 1137 break; 1138 case kArm64Ror: 1139 ASSEMBLE_SHIFT(Ror, 64); 1140 break; 1141 case kArm64Ror32: 1142 ASSEMBLE_SHIFT(Ror, 32); 1143 break; 1144 case kArm64Mov32: 1145 __ Mov(i.OutputRegister32(), i.InputRegister32(0)); 1146 break; 1147 case kArm64Sxtb32: 1148 __ Sxtb(i.OutputRegister32(), i.InputRegister32(0)); 1149 break; 1150 case kArm64Sxth32: 1151 __ Sxth(i.OutputRegister32(), i.InputRegister32(0)); 1152 break; 1153 case kArm64Sxtw: 1154 __ Sxtw(i.OutputRegister(), i.InputRegister32(0)); 1155 break; 1156 case kArm64Sbfx32: 1157 __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1), 1158 i.InputInt5(2)); 1159 break; 1160 case kArm64Ubfx: 1161 __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1), 1162 i.InputInt6(2)); 1163 break; 1164 case kArm64Ubfx32: 1165 __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1), 1166 i.InputInt5(2)); 1167 break; 1168 case kArm64Ubfiz32: 1169 __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1), 1170 i.InputInt5(2)); 1171 break; 1172 case kArm64Bfi: 1173 __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2), 1174 i.InputInt6(3)); 1175 break; 1176 case kArm64TestAndBranch32: 1177 case kArm64TestAndBranch: 1178 // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch. 1179 break; 1180 case kArm64CompareAndBranch32: 1181 case kArm64CompareAndBranch: 1182 // Pseudo instruction turned into cbz/cbnz in AssembleArchBranch. 1183 break; 1184 case kArm64ClaimCSP: { 1185 int count = RoundUp(i.InputInt32(0), 2); 1186 Register prev = __ StackPointer(); 1187 if (prev.Is(jssp)) { 1188 // TODO(titzer): make this a macro-assembler method. 1189 // Align the CSP and store the previous JSSP on the stack. 1190 UseScratchRegisterScope scope(masm()); 1191 Register tmp = scope.AcquireX(); 1192 1193 int sp_alignment = __ ActivationFrameAlignment(); 1194 __ Sub(tmp, jssp, kPointerSize); 1195 __ And(tmp, tmp, Operand(~static_cast<uint64_t>(sp_alignment - 1))); 1196 __ Mov(csp, tmp); 1197 __ Str(jssp, MemOperand(csp)); 1198 if (count > 0) { 1199 __ SetStackPointer(csp); 1200 __ Claim(count); 1201 __ SetStackPointer(prev); 1202 } 1203 } else { 1204 __ AssertCspAligned(); 1205 if (count > 0) { 1206 __ Claim(count); 1207 frame_access_state()->IncreaseSPDelta(count); 1208 } 1209 } 1210 break; 1211 } 1212 case kArm64ClaimJSSP: { 1213 int count = i.InputInt32(0); 1214 if (csp.Is(__ StackPointer())) { 1215 // No JSSP is set up. Compute it from the CSP. 1216 __ AssertCspAligned(); 1217 if (count > 0) { 1218 int even = RoundUp(count, 2); 1219 __ Sub(jssp, csp, count * kPointerSize); 1220 __ Sub(csp, csp, even * kPointerSize); // Must always be aligned. 1221 frame_access_state()->IncreaseSPDelta(even); 1222 } else { 1223 __ Mov(jssp, csp); 1224 } 1225 } else { 1226 // JSSP is the current stack pointer, just use regular Claim(). 1227 __ Claim(count); 1228 frame_access_state()->IncreaseSPDelta(count); 1229 } 1230 break; 1231 } 1232 case kArm64PokeCSP: // fall through 1233 case kArm64PokeJSSP: { 1234 Register prev = __ StackPointer(); 1235 __ SetStackPointer(arch_opcode == kArm64PokeCSP ? csp : jssp); 1236 Operand operand(i.InputInt32(1) * kPointerSize); 1237 if (instr->InputAt(0)->IsFPRegister()) { 1238 __ Poke(i.InputFloat64Register(0), operand); 1239 } else { 1240 __ Poke(i.InputRegister(0), operand); 1241 } 1242 __ SetStackPointer(prev); 1243 break; 1244 } 1245 case kArm64PokePair: { 1246 int slot = i.InputInt32(2) - 1; 1247 if (instr->InputAt(0)->IsFPRegister()) { 1248 __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0), 1249 slot * kPointerSize); 1250 } else { 1251 __ PokePair(i.InputRegister(1), i.InputRegister(0), 1252 slot * kPointerSize); 1253 } 1254 break; 1255 } 1256 case kArm64Clz: 1257 __ Clz(i.OutputRegister64(), i.InputRegister64(0)); 1258 break; 1259 case kArm64Clz32: 1260 __ Clz(i.OutputRegister32(), i.InputRegister32(0)); 1261 break; 1262 case kArm64Rbit: 1263 __ Rbit(i.OutputRegister64(), i.InputRegister64(0)); 1264 break; 1265 case kArm64Rbit32: 1266 __ Rbit(i.OutputRegister32(), i.InputRegister32(0)); 1267 break; 1268 case kArm64Cmp: 1269 __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1)); 1270 break; 1271 case kArm64Cmp32: 1272 __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1)); 1273 break; 1274 case kArm64Cmn: 1275 __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1)); 1276 break; 1277 case kArm64Cmn32: 1278 __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1)); 1279 break; 1280 case kArm64Tst: 1281 __ Tst(i.InputOrZeroRegister64(0), i.InputOperand(1)); 1282 break; 1283 case kArm64Tst32: 1284 __ Tst(i.InputOrZeroRegister32(0), i.InputOperand32(1)); 1285 break; 1286 case kArm64Float32Cmp: 1287 if (instr->InputAt(1)->IsFPRegister()) { 1288 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1)); 1289 } else { 1290 DCHECK(instr->InputAt(1)->IsImmediate()); 1291 // 0.0 is the only immediate supported by fcmp instructions. 1292 DCHECK(i.InputFloat32(1) == 0.0f); 1293 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1)); 1294 } 1295 break; 1296 case kArm64Float32Add: 1297 __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0), 1298 i.InputFloat32Register(1)); 1299 break; 1300 case kArm64Float32Sub: 1301 __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0), 1302 i.InputFloat32Register(1)); 1303 break; 1304 case kArm64Float32Mul: 1305 __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0), 1306 i.InputFloat32Register(1)); 1307 break; 1308 case kArm64Float32Div: 1309 __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0), 1310 i.InputFloat32Register(1)); 1311 break; 1312 case kArm64Float32Abs: 1313 __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0)); 1314 break; 1315 case kArm64Float32Neg: 1316 __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0)); 1317 break; 1318 case kArm64Float32Sqrt: 1319 __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0)); 1320 break; 1321 case kArm64Float64Cmp: 1322 if (instr->InputAt(1)->IsFPRegister()) { 1323 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1324 } else { 1325 DCHECK(instr->InputAt(1)->IsImmediate()); 1326 // 0.0 is the only immediate supported by fcmp instructions. 1327 DCHECK(i.InputDouble(1) == 0.0); 1328 __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1)); 1329 } 1330 break; 1331 case kArm64Float64Add: 1332 __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1333 i.InputDoubleRegister(1)); 1334 break; 1335 case kArm64Float64Sub: 1336 __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1337 i.InputDoubleRegister(1)); 1338 break; 1339 case kArm64Float64Mul: 1340 __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1341 i.InputDoubleRegister(1)); 1342 break; 1343 case kArm64Float64Div: 1344 __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1345 i.InputDoubleRegister(1)); 1346 break; 1347 case kArm64Float64Mod: { 1348 // TODO(dcarney): implement directly. See note in lithium-codegen-arm64.cc 1349 FrameScope scope(masm(), StackFrame::MANUAL); 1350 DCHECK(d0.is(i.InputDoubleRegister(0))); 1351 DCHECK(d1.is(i.InputDoubleRegister(1))); 1352 DCHECK(d0.is(i.OutputDoubleRegister())); 1353 // TODO(dcarney): make sure this saves all relevant registers. 1354 __ CallCFunction(ExternalReference::mod_two_doubles_operation(isolate()), 1355 0, 2); 1356 break; 1357 } 1358 case kArm64Float32Max: { 1359 __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0), 1360 i.InputFloat32Register(1)); 1361 break; 1362 } 1363 case kArm64Float64Max: { 1364 __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1365 i.InputDoubleRegister(1)); 1366 break; 1367 } 1368 case kArm64Float32Min: { 1369 __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0), 1370 i.InputFloat32Register(1)); 1371 break; 1372 } 1373 case kArm64Float64Min: { 1374 __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1375 i.InputDoubleRegister(1)); 1376 break; 1377 } 1378 case kArm64Float64Abs: 1379 __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1380 break; 1381 case kArm64Float64Neg: 1382 __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1383 break; 1384 case kArm64Float64Sqrt: 1385 __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1386 break; 1387 case kArm64Float32ToFloat64: 1388 __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S()); 1389 break; 1390 case kArm64Float64ToFloat32: 1391 __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0)); 1392 break; 1393 case kArm64Float32ToInt32: 1394 __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0)); 1395 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead, 1396 // because INT32_MIN allows easier out-of-bounds detection. 1397 __ Cmn(i.OutputRegister32(), 1); 1398 __ Csinc(i.OutputRegister32(), i.OutputRegister32(), i.OutputRegister32(), 1399 vc); 1400 break; 1401 case kArm64Float64ToInt32: 1402 __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0)); 1403 break; 1404 case kArm64Float32ToUint32: 1405 __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0)); 1406 // Avoid UINT32_MAX as an overflow indicator and use 0 instead, 1407 // because 0 allows easier out-of-bounds detection. 1408 __ Cmn(i.OutputRegister32(), 1); 1409 __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0)); 1410 break; 1411 case kArm64Float64ToUint32: 1412 __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0)); 1413 break; 1414 case kArm64Float32ToInt64: 1415 __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0)); 1416 if (i.OutputCount() > 1) { 1417 __ Mov(i.OutputRegister(1), 1); 1418 Label done; 1419 __ Cmp(i.OutputRegister(0), 1); 1420 __ Ccmp(i.OutputRegister(0), -1, VFlag, vc); 1421 __ Fccmp(i.InputFloat32Register(0), i.InputFloat32Register(0), VFlag, 1422 vc); 1423 __ B(vc, &done); 1424 __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN)); 1425 __ Cset(i.OutputRegister(1), eq); 1426 __ Bind(&done); 1427 } 1428 break; 1429 case kArm64Float64ToInt64: 1430 __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0)); 1431 if (i.OutputCount() > 1) { 1432 __ Mov(i.OutputRegister(1), 1); 1433 Label done; 1434 __ Cmp(i.OutputRegister(0), 1); 1435 __ Ccmp(i.OutputRegister(0), -1, VFlag, vc); 1436 __ Fccmp(i.InputDoubleRegister(0), i.InputDoubleRegister(0), VFlag, vc); 1437 __ B(vc, &done); 1438 __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN)); 1439 __ Cset(i.OutputRegister(1), eq); 1440 __ Bind(&done); 1441 } 1442 break; 1443 case kArm64Float32ToUint64: 1444 __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0)); 1445 if (i.OutputCount() > 1) { 1446 __ Fcmp(i.InputFloat32Register(0), -1.0); 1447 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt); 1448 __ Cset(i.OutputRegister(1), ne); 1449 } 1450 break; 1451 case kArm64Float64ToUint64: 1452 __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0)); 1453 if (i.OutputCount() > 1) { 1454 __ Fcmp(i.InputDoubleRegister(0), -1.0); 1455 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt); 1456 __ Cset(i.OutputRegister(1), ne); 1457 } 1458 break; 1459 case kArm64Int32ToFloat32: 1460 __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0)); 1461 break; 1462 case kArm64Int32ToFloat64: 1463 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0)); 1464 break; 1465 case kArm64Int64ToFloat32: 1466 __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0)); 1467 break; 1468 case kArm64Int64ToFloat64: 1469 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0)); 1470 break; 1471 case kArm64Uint32ToFloat32: 1472 __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0)); 1473 break; 1474 case kArm64Uint32ToFloat64: 1475 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0)); 1476 break; 1477 case kArm64Uint64ToFloat32: 1478 __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0)); 1479 break; 1480 case kArm64Uint64ToFloat64: 1481 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0)); 1482 break; 1483 case kArm64Float64ExtractLowWord32: 1484 __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0)); 1485 break; 1486 case kArm64Float64ExtractHighWord32: 1487 // TODO(arm64): This should use MOV (to general) when NEON is supported. 1488 __ Fmov(i.OutputRegister(), i.InputFloat64Register(0)); 1489 __ Lsr(i.OutputRegister(), i.OutputRegister(), 32); 1490 break; 1491 case kArm64Float64InsertLowWord32: { 1492 // TODO(arm64): This should use MOV (from general) when NEON is supported. 1493 UseScratchRegisterScope scope(masm()); 1494 Register tmp = scope.AcquireX(); 1495 __ Fmov(tmp, i.InputFloat64Register(0)); 1496 __ Bfi(tmp, i.InputRegister(1), 0, 32); 1497 __ Fmov(i.OutputFloat64Register(), tmp); 1498 break; 1499 } 1500 case kArm64Float64InsertHighWord32: { 1501 // TODO(arm64): This should use MOV (from general) when NEON is supported. 1502 UseScratchRegisterScope scope(masm()); 1503 Register tmp = scope.AcquireX(); 1504 __ Fmov(tmp.W(), i.InputFloat32Register(0)); 1505 __ Bfi(tmp, i.InputRegister(1), 32, 32); 1506 __ Fmov(i.OutputFloat64Register(), tmp); 1507 break; 1508 } 1509 case kArm64Float64MoveU64: 1510 __ Fmov(i.OutputFloat64Register(), i.InputRegister(0)); 1511 break; 1512 case kArm64Float64SilenceNaN: 1513 __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1514 break; 1515 case kArm64U64MoveFloat64: 1516 __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0)); 1517 break; 1518 case kArm64Ldrb: 1519 __ Ldrb(i.OutputRegister(), i.MemoryOperand()); 1520 break; 1521 case kArm64Ldrsb: 1522 __ Ldrsb(i.OutputRegister(), i.MemoryOperand()); 1523 break; 1524 case kArm64Strb: 1525 __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1)); 1526 break; 1527 case kArm64Ldrh: 1528 __ Ldrh(i.OutputRegister(), i.MemoryOperand()); 1529 break; 1530 case kArm64Ldrsh: 1531 __ Ldrsh(i.OutputRegister(), i.MemoryOperand()); 1532 break; 1533 case kArm64Strh: 1534 __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1)); 1535 break; 1536 case kArm64Ldrsw: 1537 __ Ldrsw(i.OutputRegister(), i.MemoryOperand()); 1538 break; 1539 case kArm64LdrW: 1540 __ Ldr(i.OutputRegister32(), i.MemoryOperand()); 1541 break; 1542 case kArm64StrW: 1543 __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1)); 1544 break; 1545 case kArm64Ldr: 1546 __ Ldr(i.OutputRegister(), i.MemoryOperand()); 1547 break; 1548 case kArm64Str: 1549 __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1)); 1550 break; 1551 case kArm64LdrS: 1552 __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand()); 1553 break; 1554 case kArm64StrS: 1555 __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1)); 1556 break; 1557 case kArm64LdrD: 1558 __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand()); 1559 break; 1560 case kArm64StrD: 1561 __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1)); 1562 break; 1563 case kCheckedLoadInt8: 1564 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsb); 1565 break; 1566 case kCheckedLoadUint8: 1567 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrb); 1568 break; 1569 case kCheckedLoadInt16: 1570 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsh); 1571 break; 1572 case kCheckedLoadUint16: 1573 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrh); 1574 break; 1575 case kCheckedLoadWord32: 1576 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldr); 1577 break; 1578 case kCheckedLoadWord64: 1579 ASSEMBLE_CHECKED_LOAD_INTEGER_64(Ldr); 1580 break; 1581 case kCheckedLoadFloat32: 1582 ASSEMBLE_CHECKED_LOAD_FLOAT(32); 1583 break; 1584 case kCheckedLoadFloat64: 1585 ASSEMBLE_CHECKED_LOAD_FLOAT(64); 1586 break; 1587 case kCheckedStoreWord8: 1588 ASSEMBLE_CHECKED_STORE_INTEGER(Strb); 1589 break; 1590 case kCheckedStoreWord16: 1591 ASSEMBLE_CHECKED_STORE_INTEGER(Strh); 1592 break; 1593 case kCheckedStoreWord32: 1594 ASSEMBLE_CHECKED_STORE_INTEGER(Str); 1595 break; 1596 case kCheckedStoreWord64: 1597 ASSEMBLE_CHECKED_STORE_INTEGER_64(Str); 1598 break; 1599 case kCheckedStoreFloat32: 1600 ASSEMBLE_CHECKED_STORE_FLOAT(32); 1601 break; 1602 case kCheckedStoreFloat64: 1603 ASSEMBLE_CHECKED_STORE_FLOAT(64); 1604 break; 1605 case kAtomicLoadInt8: 1606 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb); 1607 break; 1608 case kAtomicLoadUint8: 1609 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb); 1610 break; 1611 case kAtomicLoadInt16: 1612 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh); 1613 break; 1614 case kAtomicLoadUint16: 1615 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh); 1616 break; 1617 case kAtomicLoadWord32: 1618 __ Ldr(i.OutputRegister32(), 1619 MemOperand(i.InputRegister(0), i.InputRegister(1))); 1620 __ Dmb(InnerShareable, BarrierAll); 1621 break; 1622 case kAtomicStoreWord8: 1623 ASSEMBLE_ATOMIC_STORE_INTEGER(Strb); 1624 break; 1625 case kAtomicStoreWord16: 1626 ASSEMBLE_ATOMIC_STORE_INTEGER(Strh); 1627 break; 1628 case kAtomicStoreWord32: 1629 __ Dmb(InnerShareable, BarrierAll); 1630 __ Str(i.InputRegister32(2), 1631 MemOperand(i.InputRegister(0), i.InputRegister(1))); 1632 __ Dmb(InnerShareable, BarrierAll); 1633 break; 1634 } 1635 return kSuccess; 1636 } // NOLINT(readability/fn_size) 1637 1638 1639 // Assemble branches after this instruction. 1640 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { 1641 Arm64OperandConverter i(this, instr); 1642 Label* tlabel = branch->true_label; 1643 Label* flabel = branch->false_label; 1644 FlagsCondition condition = branch->condition; 1645 ArchOpcode opcode = instr->arch_opcode(); 1646 1647 if (opcode == kArm64CompareAndBranch32) { 1648 switch (condition) { 1649 case kEqual: 1650 __ Cbz(i.InputRegister32(0), tlabel); 1651 break; 1652 case kNotEqual: 1653 __ Cbnz(i.InputRegister32(0), tlabel); 1654 break; 1655 default: 1656 UNREACHABLE(); 1657 } 1658 } else if (opcode == kArm64CompareAndBranch) { 1659 switch (condition) { 1660 case kEqual: 1661 __ Cbz(i.InputRegister64(0), tlabel); 1662 break; 1663 case kNotEqual: 1664 __ Cbnz(i.InputRegister64(0), tlabel); 1665 break; 1666 default: 1667 UNREACHABLE(); 1668 } 1669 } else if (opcode == kArm64TestAndBranch32) { 1670 switch (condition) { 1671 case kEqual: 1672 __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel); 1673 break; 1674 case kNotEqual: 1675 __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel); 1676 break; 1677 default: 1678 UNREACHABLE(); 1679 } 1680 } else if (opcode == kArm64TestAndBranch) { 1681 switch (condition) { 1682 case kEqual: 1683 __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel); 1684 break; 1685 case kNotEqual: 1686 __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel); 1687 break; 1688 default: 1689 UNREACHABLE(); 1690 } 1691 } else { 1692 Condition cc = FlagsConditionToCondition(condition); 1693 __ B(cc, tlabel); 1694 } 1695 if (!branch->fallthru) __ B(flabel); // no fallthru to flabel. 1696 } 1697 1698 1699 void CodeGenerator::AssembleArchJump(RpoNumber target) { 1700 if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target)); 1701 } 1702 1703 void CodeGenerator::AssembleArchTrap(Instruction* instr, 1704 FlagsCondition condition) { 1705 class OutOfLineTrap final : public OutOfLineCode { 1706 public: 1707 OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr) 1708 : OutOfLineCode(gen), 1709 frame_elided_(frame_elided), 1710 instr_(instr), 1711 gen_(gen) {} 1712 void Generate() final { 1713 Arm64OperandConverter i(gen_, instr_); 1714 Builtins::Name trap_id = 1715 static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1)); 1716 bool old_has_frame = __ has_frame(); 1717 if (frame_elided_) { 1718 __ set_has_frame(true); 1719 __ EnterFrame(StackFrame::WASM_COMPILED); 1720 } 1721 GenerateCallToTrap(trap_id); 1722 if (frame_elided_) { 1723 __ set_has_frame(old_has_frame); 1724 } 1725 } 1726 1727 private: 1728 void GenerateCallToTrap(Builtins::Name trap_id) { 1729 if (trap_id == Builtins::builtin_count) { 1730 // We cannot test calls to the runtime in cctest/test-run-wasm. 1731 // Therefore we emit a call to C here instead of a call to the runtime. 1732 __ CallCFunction( 1733 ExternalReference::wasm_call_trap_callback_for_testing(isolate()), 1734 0); 1735 __ LeaveFrame(StackFrame::WASM_COMPILED); 1736 __ Ret(); 1737 } else { 1738 DCHECK(csp.Is(__ StackPointer())); 1739 // Initialize the jssp because it is required for the runtime call. 1740 __ Mov(jssp, csp); 1741 gen_->AssembleSourcePosition(instr_); 1742 __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()), 1743 RelocInfo::CODE_TARGET); 1744 ReferenceMap* reference_map = 1745 new (gen_->zone()) ReferenceMap(gen_->zone()); 1746 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0, 1747 Safepoint::kNoLazyDeopt); 1748 if (FLAG_debug_code) { 1749 // The trap code should never return. 1750 __ Brk(0); 1751 } 1752 } 1753 } 1754 bool frame_elided_; 1755 Instruction* instr_; 1756 CodeGenerator* gen_; 1757 }; 1758 bool frame_elided = !frame_access_state()->has_frame(); 1759 auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr); 1760 Label* tlabel = ool->entry(); 1761 Condition cc = FlagsConditionToCondition(condition); 1762 __ B(cc, tlabel); 1763 } 1764 1765 // Assemble boolean materializations after this instruction. 1766 void CodeGenerator::AssembleArchBoolean(Instruction* instr, 1767 FlagsCondition condition) { 1768 Arm64OperandConverter i(this, instr); 1769 1770 // Materialize a full 64-bit 1 or 0 value. The result register is always the 1771 // last output of the instruction. 1772 DCHECK_NE(0u, instr->OutputCount()); 1773 Register reg = i.OutputRegister(instr->OutputCount() - 1); 1774 Condition cc = FlagsConditionToCondition(condition); 1775 __ Cset(reg, cc); 1776 } 1777 1778 1779 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) { 1780 Arm64OperandConverter i(this, instr); 1781 Register input = i.InputRegister32(0); 1782 for (size_t index = 2; index < instr->InputCount(); index += 2) { 1783 __ Cmp(input, i.InputInt32(index + 0)); 1784 __ B(eq, GetLabel(i.InputRpo(index + 1))); 1785 } 1786 AssembleArchJump(i.InputRpo(1)); 1787 } 1788 1789 1790 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) { 1791 Arm64OperandConverter i(this, instr); 1792 UseScratchRegisterScope scope(masm()); 1793 Register input = i.InputRegister32(0); 1794 Register temp = scope.AcquireX(); 1795 size_t const case_count = instr->InputCount() - 2; 1796 Label table; 1797 __ Cmp(input, case_count); 1798 __ B(hs, GetLabel(i.InputRpo(1))); 1799 __ Adr(temp, &table); 1800 __ Add(temp, temp, Operand(input, UXTW, 2)); 1801 __ Br(temp); 1802 __ StartBlockPools(); 1803 __ Bind(&table); 1804 for (size_t index = 0; index < case_count; ++index) { 1805 __ B(GetLabel(i.InputRpo(index + 2))); 1806 } 1807 __ EndBlockPools(); 1808 } 1809 1810 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall( 1811 int deoptimization_id, SourcePosition pos) { 1812 DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id); 1813 DeoptimizeReason deoptimization_reason = 1814 GetDeoptimizationReason(deoptimization_id); 1815 Deoptimizer::BailoutType bailout_type = 1816 deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT 1817 : Deoptimizer::EAGER; 1818 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry( 1819 isolate(), deoptimization_id, bailout_type); 1820 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts; 1821 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id); 1822 __ Call(deopt_entry, RelocInfo::RUNTIME_ENTRY); 1823 return kSuccess; 1824 } 1825 1826 void CodeGenerator::FinishFrame(Frame* frame) { 1827 frame->AlignFrame(16); 1828 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1829 1830 if (descriptor->UseNativeStack() || descriptor->IsCFunctionCall()) { 1831 __ SetStackPointer(csp); 1832 } else { 1833 __ SetStackPointer(jssp); 1834 } 1835 1836 // Save FP registers. 1837 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 1838 descriptor->CalleeSavedFPRegisters()); 1839 int saved_count = saves_fp.Count(); 1840 if (saved_count != 0) { 1841 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list()); 1842 frame->AllocateSavedCalleeRegisterSlots(saved_count * 1843 (kDoubleSize / kPointerSize)); 1844 } 1845 1846 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1847 descriptor->CalleeSavedRegisters()); 1848 saved_count = saves.Count(); 1849 if (saved_count != 0) { 1850 frame->AllocateSavedCalleeRegisterSlots(saved_count); 1851 } 1852 } 1853 1854 void CodeGenerator::AssembleConstructFrame() { 1855 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1856 if (descriptor->UseNativeStack()) { 1857 __ AssertCspAligned(); 1858 } 1859 1860 int fixed_frame_size = descriptor->CalculateFixedFrameSize(); 1861 int shrink_slots = 1862 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize(); 1863 1864 if (frame_access_state()->has_frame()) { 1865 // Link the frame 1866 if (descriptor->IsJSFunctionCall()) { 1867 DCHECK(!descriptor->UseNativeStack()); 1868 __ Prologue(this->info()->GeneratePreagedPrologue()); 1869 } else { 1870 __ Push(lr, fp); 1871 __ Mov(fp, masm_.StackPointer()); 1872 } 1873 if (!info()->GeneratePreagedPrologue()) { 1874 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset()); 1875 } 1876 1877 // Create OSR entry if applicable 1878 if (info()->is_osr()) { 1879 // TurboFan OSR-compiled functions cannot be entered directly. 1880 __ Abort(kShouldNotDirectlyEnterOsrFunction); 1881 1882 // Unoptimized code jumps directly to this entrypoint while the 1883 // unoptimized 1884 // frame is still on the stack. Optimized code uses OSR values directly 1885 // from 1886 // the unoptimized frame. Thus, all that needs to be done is to allocate 1887 // the 1888 // remaining stack slots. 1889 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --"); 1890 osr_pc_offset_ = __ pc_offset(); 1891 shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots(); 1892 } 1893 // Build remainder of frame, including accounting for and filling-in 1894 // frame-specific header information, e.g. claiming the extra slot that 1895 // other platforms explicitly push for STUB frames and frames recording 1896 // their argument count. 1897 __ Claim(shrink_slots + (fixed_frame_size & 1)); 1898 if (descriptor->PushArgumentCount()) { 1899 __ Str(kJavaScriptCallArgCountRegister, 1900 MemOperand(fp, OptimizedBuiltinFrameConstants::kArgCOffset)); 1901 } 1902 bool is_stub_frame = 1903 !descriptor->IsJSFunctionCall() && !descriptor->IsCFunctionCall(); 1904 if (is_stub_frame) { 1905 UseScratchRegisterScope temps(masm()); 1906 Register temp = temps.AcquireX(); 1907 __ Mov(temp, StackFrame::TypeToMarker(info()->GetOutputStackFrameType())); 1908 __ Str(temp, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset)); 1909 } 1910 } 1911 1912 // Save FP registers. 1913 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 1914 descriptor->CalleeSavedFPRegisters()); 1915 int saved_count = saves_fp.Count(); 1916 if (saved_count != 0) { 1917 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list()); 1918 __ PushCPURegList(saves_fp); 1919 } 1920 // Save registers. 1921 // TODO(palfia): TF save list is not in sync with 1922 // CPURegList::GetCalleeSaved(): x30 is missing. 1923 // DCHECK(saves.list() == CPURegList::GetCalleeSaved().list()); 1924 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1925 descriptor->CalleeSavedRegisters()); 1926 saved_count = saves.Count(); 1927 if (saved_count != 0) { 1928 __ PushCPURegList(saves); 1929 } 1930 } 1931 1932 void CodeGenerator::AssembleReturn(InstructionOperand* pop) { 1933 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1934 1935 // Restore registers. 1936 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1937 descriptor->CalleeSavedRegisters()); 1938 if (saves.Count() != 0) { 1939 __ PopCPURegList(saves); 1940 } 1941 1942 // Restore fp registers. 1943 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 1944 descriptor->CalleeSavedFPRegisters()); 1945 if (saves_fp.Count() != 0) { 1946 __ PopCPURegList(saves_fp); 1947 } 1948 1949 unwinding_info_writer_.MarkBlockWillExit(); 1950 1951 Arm64OperandConverter g(this, nullptr); 1952 int pop_count = static_cast<int>(descriptor->StackParameterCount()); 1953 if (descriptor->IsCFunctionCall()) { 1954 AssembleDeconstructFrame(); 1955 } else if (frame_access_state()->has_frame()) { 1956 // Canonicalize JSFunction return sites for now unless they have an variable 1957 // number of stack slot pops. 1958 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) { 1959 if (return_label_.is_bound()) { 1960 __ B(&return_label_); 1961 return; 1962 } else { 1963 __ Bind(&return_label_); 1964 AssembleDeconstructFrame(); 1965 if (descriptor->UseNativeStack()) { 1966 pop_count += (pop_count & 1); // align 1967 } 1968 } 1969 } else { 1970 AssembleDeconstructFrame(); 1971 if (descriptor->UseNativeStack()) { 1972 pop_count += (pop_count & 1); // align 1973 } 1974 } 1975 } else if (descriptor->UseNativeStack()) { 1976 pop_count += (pop_count & 1); // align 1977 } 1978 1979 if (pop->IsImmediate()) { 1980 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type()); 1981 pop_count += g.ToConstant(pop).ToInt32(); 1982 __ Drop(pop_count); 1983 } else { 1984 Register pop_reg = g.ToRegister(pop); 1985 __ Add(pop_reg, pop_reg, pop_count); 1986 __ Drop(pop_reg); 1987 } 1988 1989 if (descriptor->UseNativeStack()) { 1990 __ AssertCspAligned(); 1991 } 1992 __ Ret(); 1993 } 1994 1995 1996 void CodeGenerator::AssembleMove(InstructionOperand* source, 1997 InstructionOperand* destination) { 1998 Arm64OperandConverter g(this, nullptr); 1999 // Dispatch on the source and destination operand kinds. Not all 2000 // combinations are possible. 2001 if (source->IsRegister()) { 2002 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 2003 Register src = g.ToRegister(source); 2004 if (destination->IsRegister()) { 2005 __ Mov(g.ToRegister(destination), src); 2006 } else { 2007 __ Str(src, g.ToMemOperand(destination, masm())); 2008 } 2009 } else if (source->IsStackSlot()) { 2010 MemOperand src = g.ToMemOperand(source, masm()); 2011 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 2012 if (destination->IsRegister()) { 2013 __ Ldr(g.ToRegister(destination), src); 2014 } else { 2015 UseScratchRegisterScope scope(masm()); 2016 Register temp = scope.AcquireX(); 2017 __ Ldr(temp, src); 2018 __ Str(temp, g.ToMemOperand(destination, masm())); 2019 } 2020 } else if (source->IsConstant()) { 2021 Constant src = g.ToConstant(ConstantOperand::cast(source)); 2022 if (destination->IsRegister() || destination->IsStackSlot()) { 2023 UseScratchRegisterScope scope(masm()); 2024 Register dst = destination->IsRegister() ? g.ToRegister(destination) 2025 : scope.AcquireX(); 2026 if (src.type() == Constant::kHeapObject) { 2027 Handle<HeapObject> src_object = src.ToHeapObject(); 2028 Heap::RootListIndex index; 2029 if (IsMaterializableFromRoot(src_object, &index)) { 2030 __ LoadRoot(dst, index); 2031 } else { 2032 __ LoadObject(dst, src_object); 2033 } 2034 } else { 2035 __ Mov(dst, g.ToImmediate(source)); 2036 } 2037 if (destination->IsStackSlot()) { 2038 __ Str(dst, g.ToMemOperand(destination, masm())); 2039 } 2040 } else if (src.type() == Constant::kFloat32) { 2041 if (destination->IsFPRegister()) { 2042 FPRegister dst = g.ToDoubleRegister(destination).S(); 2043 __ Fmov(dst, src.ToFloat32()); 2044 } else { 2045 DCHECK(destination->IsFPStackSlot()); 2046 if (bit_cast<int32_t>(src.ToFloat32()) == 0) { 2047 __ Str(wzr, g.ToMemOperand(destination, masm())); 2048 } else { 2049 UseScratchRegisterScope scope(masm()); 2050 FPRegister temp = scope.AcquireS(); 2051 __ Fmov(temp, src.ToFloat32()); 2052 __ Str(temp, g.ToMemOperand(destination, masm())); 2053 } 2054 } 2055 } else { 2056 DCHECK_EQ(Constant::kFloat64, src.type()); 2057 if (destination->IsFPRegister()) { 2058 FPRegister dst = g.ToDoubleRegister(destination); 2059 __ Fmov(dst, src.ToFloat64()); 2060 } else { 2061 DCHECK(destination->IsFPStackSlot()); 2062 if (bit_cast<int64_t>(src.ToFloat64()) == 0) { 2063 __ Str(xzr, g.ToMemOperand(destination, masm())); 2064 } else { 2065 UseScratchRegisterScope scope(masm()); 2066 FPRegister temp = scope.AcquireD(); 2067 __ Fmov(temp, src.ToFloat64()); 2068 __ Str(temp, g.ToMemOperand(destination, masm())); 2069 } 2070 } 2071 } 2072 } else if (source->IsFPRegister()) { 2073 FPRegister src = g.ToDoubleRegister(source); 2074 if (destination->IsFPRegister()) { 2075 FPRegister dst = g.ToDoubleRegister(destination); 2076 __ Fmov(dst, src); 2077 } else { 2078 DCHECK(destination->IsFPStackSlot()); 2079 __ Str(src, g.ToMemOperand(destination, masm())); 2080 } 2081 } else if (source->IsFPStackSlot()) { 2082 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot()); 2083 MemOperand src = g.ToMemOperand(source, masm()); 2084 if (destination->IsFPRegister()) { 2085 __ Ldr(g.ToDoubleRegister(destination), src); 2086 } else { 2087 UseScratchRegisterScope scope(masm()); 2088 FPRegister temp = scope.AcquireD(); 2089 __ Ldr(temp, src); 2090 __ Str(temp, g.ToMemOperand(destination, masm())); 2091 } 2092 } else { 2093 UNREACHABLE(); 2094 } 2095 } 2096 2097 2098 void CodeGenerator::AssembleSwap(InstructionOperand* source, 2099 InstructionOperand* destination) { 2100 Arm64OperandConverter g(this, nullptr); 2101 // Dispatch on the source and destination operand kinds. Not all 2102 // combinations are possible. 2103 if (source->IsRegister()) { 2104 // Register-register. 2105 UseScratchRegisterScope scope(masm()); 2106 Register temp = scope.AcquireX(); 2107 Register src = g.ToRegister(source); 2108 if (destination->IsRegister()) { 2109 Register dst = g.ToRegister(destination); 2110 __ Mov(temp, src); 2111 __ Mov(src, dst); 2112 __ Mov(dst, temp); 2113 } else { 2114 DCHECK(destination->IsStackSlot()); 2115 MemOperand dst = g.ToMemOperand(destination, masm()); 2116 __ Mov(temp, src); 2117 __ Ldr(src, dst); 2118 __ Str(temp, dst); 2119 } 2120 } else if (source->IsStackSlot() || source->IsFPStackSlot()) { 2121 UseScratchRegisterScope scope(masm()); 2122 DoubleRegister temp_0 = scope.AcquireD(); 2123 DoubleRegister temp_1 = scope.AcquireD(); 2124 MemOperand src = g.ToMemOperand(source, masm()); 2125 MemOperand dst = g.ToMemOperand(destination, masm()); 2126 __ Ldr(temp_0, src); 2127 __ Ldr(temp_1, dst); 2128 __ Str(temp_0, dst); 2129 __ Str(temp_1, src); 2130 } else if (source->IsFPRegister()) { 2131 UseScratchRegisterScope scope(masm()); 2132 FPRegister temp = scope.AcquireD(); 2133 FPRegister src = g.ToDoubleRegister(source); 2134 if (destination->IsFPRegister()) { 2135 FPRegister dst = g.ToDoubleRegister(destination); 2136 __ Fmov(temp, src); 2137 __ Fmov(src, dst); 2138 __ Fmov(dst, temp); 2139 } else { 2140 DCHECK(destination->IsFPStackSlot()); 2141 MemOperand dst = g.ToMemOperand(destination, masm()); 2142 __ Fmov(temp, src); 2143 __ Ldr(src, dst); 2144 __ Str(temp, dst); 2145 } 2146 } else { 2147 // No other combinations are possible. 2148 UNREACHABLE(); 2149 } 2150 } 2151 2152 2153 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) { 2154 // On 64-bit ARM we emit the jump tables inline. 2155 UNREACHABLE(); 2156 } 2157 2158 2159 void CodeGenerator::EnsureSpaceForLazyDeopt() { 2160 if (!info()->ShouldEnsureSpaceForLazyDeopt()) { 2161 return; 2162 } 2163 2164 int space_needed = Deoptimizer::patch_size(); 2165 // Ensure that we have enough space after the previous lazy-bailout 2166 // instruction for patching the code here. 2167 intptr_t current_pc = masm()->pc_offset(); 2168 2169 if (current_pc < (last_lazy_deopt_pc_ + space_needed)) { 2170 intptr_t padding_size = last_lazy_deopt_pc_ + space_needed - current_pc; 2171 DCHECK((padding_size % kInstructionSize) == 0); 2172 InstructionAccurateScope instruction_accurate( 2173 masm(), padding_size / kInstructionSize); 2174 2175 while (padding_size > 0) { 2176 __ nop(); 2177 padding_size -= kInstructionSize; 2178 } 2179 } 2180 } 2181 2182 #undef __ 2183 2184 } // namespace compiler 2185 } // namespace internal 2186 } // namespace v8 2187
