1 // Copyright 2014 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/code-generator.h" 6 7 #include "src/arm64/frames-arm64.h" 8 #include "src/arm64/macro-assembler-arm64.h" 9 #include "src/ast/scopes.h" 10 #include "src/compiler/code-generator-impl.h" 11 #include "src/compiler/gap-resolver.h" 12 #include "src/compiler/node-matchers.h" 13 #include "src/compiler/osr.h" 14 15 namespace v8 { 16 namespace internal { 17 namespace compiler { 18 19 #define __ masm()-> 20 21 22 // Adds Arm64-specific methods to convert InstructionOperands. 23 class Arm64OperandConverter final : public InstructionOperandConverter { 24 public: 25 Arm64OperandConverter(CodeGenerator* gen, Instruction* instr) 26 : InstructionOperandConverter(gen, instr) {} 27 28 DoubleRegister InputFloat32Register(size_t index) { 29 return InputDoubleRegister(index).S(); 30 } 31 32 DoubleRegister InputFloat64Register(size_t index) { 33 return InputDoubleRegister(index); 34 } 35 36 CPURegister InputFloat32OrZeroRegister(size_t index) { 37 if (instr_->InputAt(index)->IsImmediate()) { 38 DCHECK(bit_cast<int32_t>(InputFloat32(index)) == 0); 39 return wzr; 40 } 41 DCHECK(instr_->InputAt(index)->IsFPRegister()); 42 return InputDoubleRegister(index).S(); 43 } 44 45 CPURegister InputFloat64OrZeroRegister(size_t index) { 46 if (instr_->InputAt(index)->IsImmediate()) { 47 DCHECK(bit_cast<int64_t>(InputDouble(index)) == 0); 48 return xzr; 49 } 50 DCHECK(instr_->InputAt(index)->IsDoubleRegister()); 51 return InputDoubleRegister(index); 52 } 53 54 size_t OutputCount() { return instr_->OutputCount(); } 55 56 DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); } 57 58 DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); } 59 60 Register InputRegister32(size_t index) { 61 return ToRegister(instr_->InputAt(index)).W(); 62 } 63 64 Register InputOrZeroRegister32(size_t index) { 65 DCHECK(instr_->InputAt(index)->IsRegister() || 66 (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0))); 67 if (instr_->InputAt(index)->IsImmediate()) { 68 return wzr; 69 } 70 return InputRegister32(index); 71 } 72 73 Register InputRegister64(size_t index) { return InputRegister(index); } 74 75 Register InputOrZeroRegister64(size_t index) { 76 DCHECK(instr_->InputAt(index)->IsRegister() || 77 (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0))); 78 if (instr_->InputAt(index)->IsImmediate()) { 79 return xzr; 80 } 81 return InputRegister64(index); 82 } 83 84 Operand InputImmediate(size_t index) { 85 return ToImmediate(instr_->InputAt(index)); 86 } 87 88 Operand InputOperand(size_t index) { 89 return ToOperand(instr_->InputAt(index)); 90 } 91 92 Operand InputOperand64(size_t index) { return InputOperand(index); } 93 94 Operand InputOperand32(size_t index) { 95 return ToOperand32(instr_->InputAt(index)); 96 } 97 98 Register OutputRegister64() { return OutputRegister(); } 99 100 Register OutputRegister32() { return ToRegister(instr_->Output()).W(); } 101 102 Operand InputOperand2_32(size_t index) { 103 switch (AddressingModeField::decode(instr_->opcode())) { 104 case kMode_None: 105 return InputOperand32(index); 106 case kMode_Operand2_R_LSL_I: 107 return Operand(InputRegister32(index), LSL, InputInt5(index + 1)); 108 case kMode_Operand2_R_LSR_I: 109 return Operand(InputRegister32(index), LSR, InputInt5(index + 1)); 110 case kMode_Operand2_R_ASR_I: 111 return Operand(InputRegister32(index), ASR, InputInt5(index + 1)); 112 case kMode_Operand2_R_ROR_I: 113 return Operand(InputRegister32(index), ROR, InputInt5(index + 1)); 114 case kMode_Operand2_R_UXTB: 115 return Operand(InputRegister32(index), UXTB); 116 case kMode_Operand2_R_UXTH: 117 return Operand(InputRegister32(index), UXTH); 118 case kMode_Operand2_R_SXTB: 119 return Operand(InputRegister32(index), SXTB); 120 case kMode_Operand2_R_SXTH: 121 return Operand(InputRegister32(index), SXTH); 122 case kMode_MRI: 123 case kMode_MRR: 124 break; 125 } 126 UNREACHABLE(); 127 return Operand(-1); 128 } 129 130 Operand InputOperand2_64(size_t index) { 131 switch (AddressingModeField::decode(instr_->opcode())) { 132 case kMode_None: 133 return InputOperand64(index); 134 case kMode_Operand2_R_LSL_I: 135 return Operand(InputRegister64(index), LSL, InputInt6(index + 1)); 136 case kMode_Operand2_R_LSR_I: 137 return Operand(InputRegister64(index), LSR, InputInt6(index + 1)); 138 case kMode_Operand2_R_ASR_I: 139 return Operand(InputRegister64(index), ASR, InputInt6(index + 1)); 140 case kMode_Operand2_R_ROR_I: 141 return Operand(InputRegister64(index), ROR, InputInt6(index + 1)); 142 case kMode_Operand2_R_UXTB: 143 return Operand(InputRegister64(index), UXTB); 144 case kMode_Operand2_R_UXTH: 145 return Operand(InputRegister64(index), UXTH); 146 case kMode_Operand2_R_SXTB: 147 return Operand(InputRegister64(index), SXTB); 148 case kMode_Operand2_R_SXTH: 149 return Operand(InputRegister64(index), SXTH); 150 case kMode_MRI: 151 case kMode_MRR: 152 break; 153 } 154 UNREACHABLE(); 155 return Operand(-1); 156 } 157 158 MemOperand MemoryOperand(size_t* first_index) { 159 const size_t index = *first_index; 160 switch (AddressingModeField::decode(instr_->opcode())) { 161 case kMode_None: 162 case kMode_Operand2_R_LSR_I: 163 case kMode_Operand2_R_ASR_I: 164 case kMode_Operand2_R_ROR_I: 165 case kMode_Operand2_R_UXTB: 166 case kMode_Operand2_R_UXTH: 167 case kMode_Operand2_R_SXTB: 168 case kMode_Operand2_R_SXTH: 169 break; 170 case kMode_Operand2_R_LSL_I: 171 *first_index += 3; 172 return MemOperand(InputRegister(index + 0), InputRegister(index + 1), 173 LSL, InputInt32(index + 2)); 174 case kMode_MRI: 175 *first_index += 2; 176 return MemOperand(InputRegister(index + 0), InputInt32(index + 1)); 177 case kMode_MRR: 178 *first_index += 2; 179 return MemOperand(InputRegister(index + 0), InputRegister(index + 1)); 180 } 181 UNREACHABLE(); 182 return MemOperand(no_reg); 183 } 184 185 MemOperand MemoryOperand(size_t first_index = 0) { 186 return MemoryOperand(&first_index); 187 } 188 189 Operand ToOperand(InstructionOperand* op) { 190 if (op->IsRegister()) { 191 return Operand(ToRegister(op)); 192 } 193 return ToImmediate(op); 194 } 195 196 Operand ToOperand32(InstructionOperand* op) { 197 if (op->IsRegister()) { 198 return Operand(ToRegister(op).W()); 199 } 200 return ToImmediate(op); 201 } 202 203 Operand ToImmediate(InstructionOperand* operand) { 204 Constant constant = ToConstant(operand); 205 switch (constant.type()) { 206 case Constant::kInt32: 207 if (constant.rmode() == RelocInfo::WASM_MEMORY_SIZE_REFERENCE) { 208 return Operand(constant.ToInt32(), constant.rmode()); 209 } else { 210 return Operand(constant.ToInt32()); 211 } 212 case Constant::kInt64: 213 if (constant.rmode() == RelocInfo::WASM_MEMORY_REFERENCE || 214 constant.rmode() == RelocInfo::WASM_GLOBAL_REFERENCE) { 215 return Operand(constant.ToInt64(), constant.rmode()); 216 } else { 217 DCHECK(constant.rmode() != RelocInfo::WASM_MEMORY_SIZE_REFERENCE); 218 return Operand(constant.ToInt64()); 219 } 220 case Constant::kFloat32: 221 return Operand( 222 isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED)); 223 case Constant::kFloat64: 224 return Operand( 225 isolate()->factory()->NewNumber(constant.ToFloat64(), TENURED)); 226 case Constant::kExternalReference: 227 return Operand(constant.ToExternalReference()); 228 case Constant::kHeapObject: 229 return Operand(constant.ToHeapObject()); 230 case Constant::kRpoNumber: 231 UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64. 232 break; 233 } 234 UNREACHABLE(); 235 return Operand(-1); 236 } 237 238 MemOperand ToMemOperand(InstructionOperand* op, MacroAssembler* masm) const { 239 DCHECK_NOT_NULL(op); 240 DCHECK(op->IsStackSlot() || op->IsFPStackSlot()); 241 return SlotToMemOperand(AllocatedOperand::cast(op)->index(), masm); 242 } 243 244 MemOperand SlotToMemOperand(int slot, MacroAssembler* masm) const { 245 FrameOffset offset = frame_access_state()->GetFrameOffset(slot); 246 if (offset.from_frame_pointer()) { 247 int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset(); 248 // Convert FP-offsets to SP-offsets if it results in better code. 249 if (Assembler::IsImmLSUnscaled(from_sp) || 250 Assembler::IsImmLSScaled(from_sp, LSDoubleWord)) { 251 offset = FrameOffset::FromStackPointer(from_sp); 252 } 253 } 254 return MemOperand(offset.from_stack_pointer() ? masm->StackPointer() : fp, 255 offset.offset()); 256 } 257 }; 258 259 260 namespace { 261 262 class OutOfLineLoadNaN32 final : public OutOfLineCode { 263 public: 264 OutOfLineLoadNaN32(CodeGenerator* gen, DoubleRegister result) 265 : OutOfLineCode(gen), result_(result) {} 266 267 void Generate() final { 268 __ Fmov(result_, std::numeric_limits<float>::quiet_NaN()); 269 } 270 271 private: 272 DoubleRegister const result_; 273 }; 274 275 276 class OutOfLineLoadNaN64 final : public OutOfLineCode { 277 public: 278 OutOfLineLoadNaN64(CodeGenerator* gen, DoubleRegister result) 279 : OutOfLineCode(gen), result_(result) {} 280 281 void Generate() final { 282 __ Fmov(result_, std::numeric_limits<double>::quiet_NaN()); 283 } 284 285 private: 286 DoubleRegister const result_; 287 }; 288 289 290 class OutOfLineLoadZero final : public OutOfLineCode { 291 public: 292 OutOfLineLoadZero(CodeGenerator* gen, Register result) 293 : OutOfLineCode(gen), result_(result) {} 294 295 void Generate() final { __ Mov(result_, 0); } 296 297 private: 298 Register const result_; 299 }; 300 301 302 class OutOfLineRecordWrite final : public OutOfLineCode { 303 public: 304 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index, 305 Register value, Register scratch0, Register scratch1, 306 RecordWriteMode mode) 307 : OutOfLineCode(gen), 308 object_(object), 309 index_(index), 310 value_(value), 311 scratch0_(scratch0), 312 scratch1_(scratch1), 313 mode_(mode), 314 must_save_lr_(!gen->frame_access_state()->has_frame()) {} 315 316 void Generate() final { 317 if (mode_ > RecordWriteMode::kValueIsPointer) { 318 __ JumpIfSmi(value_, exit()); 319 } 320 __ CheckPageFlagClear(value_, scratch0_, 321 MemoryChunk::kPointersToHereAreInterestingMask, 322 exit()); 323 RememberedSetAction const remembered_set_action = 324 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET 325 : OMIT_REMEMBERED_SET; 326 SaveFPRegsMode const save_fp_mode = 327 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; 328 if (must_save_lr_) { 329 // We need to save and restore lr if the frame was elided. 330 __ Push(lr); 331 } 332 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_, 333 remembered_set_action, save_fp_mode); 334 __ Add(scratch1_, object_, index_); 335 __ CallStub(&stub); 336 if (must_save_lr_) { 337 __ Pop(lr); 338 } 339 } 340 341 private: 342 Register const object_; 343 Operand const index_; 344 Register const value_; 345 Register const scratch0_; 346 Register const scratch1_; 347 RecordWriteMode const mode_; 348 bool must_save_lr_; 349 }; 350 351 352 Condition FlagsConditionToCondition(FlagsCondition condition) { 353 switch (condition) { 354 case kEqual: 355 return eq; 356 case kNotEqual: 357 return ne; 358 case kSignedLessThan: 359 return lt; 360 case kSignedGreaterThanOrEqual: 361 return ge; 362 case kSignedLessThanOrEqual: 363 return le; 364 case kSignedGreaterThan: 365 return gt; 366 case kUnsignedLessThan: 367 return lo; 368 case kUnsignedGreaterThanOrEqual: 369 return hs; 370 case kUnsignedLessThanOrEqual: 371 return ls; 372 case kUnsignedGreaterThan: 373 return hi; 374 case kFloatLessThanOrUnordered: 375 return lt; 376 case kFloatGreaterThanOrEqual: 377 return ge; 378 case kFloatLessThanOrEqual: 379 return ls; 380 case kFloatGreaterThanOrUnordered: 381 return hi; 382 case kFloatLessThan: 383 return lo; 384 case kFloatGreaterThanOrEqualOrUnordered: 385 return hs; 386 case kFloatLessThanOrEqualOrUnordered: 387 return le; 388 case kFloatGreaterThan: 389 return gt; 390 case kOverflow: 391 return vs; 392 case kNotOverflow: 393 return vc; 394 case kUnorderedEqual: 395 case kUnorderedNotEqual: 396 break; 397 } 398 UNREACHABLE(); 399 return nv; 400 } 401 402 } // namespace 403 404 #define ASSEMBLE_BOUNDS_CHECK(offset, length, out_of_bounds) \ 405 do { \ 406 if (length.IsImmediate() && \ 407 base::bits::IsPowerOfTwo64(length.ImmediateValue())) { \ 408 __ Tst(offset, ~(length.ImmediateValue() - 1)); \ 409 __ B(ne, out_of_bounds); \ 410 } else { \ 411 __ Cmp(offset, length); \ 412 __ B(hs, out_of_bounds); \ 413 } \ 414 } while (0) 415 416 #define ASSEMBLE_CHECKED_LOAD_FLOAT(width) \ 417 do { \ 418 auto result = i.OutputFloat##width##Register(); \ 419 auto buffer = i.InputRegister(0); \ 420 auto offset = i.InputRegister32(1); \ 421 auto length = i.InputOperand32(2); \ 422 auto ool = new (zone()) OutOfLineLoadNaN##width(this, result); \ 423 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \ 424 __ Ldr(result, MemOperand(buffer, offset, UXTW)); \ 425 __ Bind(ool->exit()); \ 426 } while (0) 427 428 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \ 429 do { \ 430 auto result = i.OutputRegister32(); \ 431 auto buffer = i.InputRegister(0); \ 432 auto offset = i.InputRegister32(1); \ 433 auto length = i.InputOperand32(2); \ 434 auto ool = new (zone()) OutOfLineLoadZero(this, result); \ 435 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \ 436 __ asm_instr(result, MemOperand(buffer, offset, UXTW)); \ 437 __ Bind(ool->exit()); \ 438 } while (0) 439 440 #define ASSEMBLE_CHECKED_LOAD_INTEGER_64(asm_instr) \ 441 do { \ 442 auto result = i.OutputRegister(); \ 443 auto buffer = i.InputRegister(0); \ 444 auto offset = i.InputRegister32(1); \ 445 auto length = i.InputOperand32(2); \ 446 auto ool = new (zone()) OutOfLineLoadZero(this, result); \ 447 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \ 448 __ asm_instr(result, MemOperand(buffer, offset, UXTW)); \ 449 __ Bind(ool->exit()); \ 450 } while (0) 451 452 #define ASSEMBLE_CHECKED_STORE_FLOAT(width) \ 453 do { \ 454 auto buffer = i.InputRegister(0); \ 455 auto offset = i.InputRegister32(1); \ 456 auto length = i.InputOperand32(2); \ 457 auto value = i.InputFloat##width##OrZeroRegister(3); \ 458 Label done; \ 459 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \ 460 __ Str(value, MemOperand(buffer, offset, UXTW)); \ 461 __ Bind(&done); \ 462 } while (0) 463 464 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \ 465 do { \ 466 auto buffer = i.InputRegister(0); \ 467 auto offset = i.InputRegister32(1); \ 468 auto length = i.InputOperand32(2); \ 469 auto value = i.InputOrZeroRegister32(3); \ 470 Label done; \ 471 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \ 472 __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \ 473 __ Bind(&done); \ 474 } while (0) 475 476 #define ASSEMBLE_CHECKED_STORE_INTEGER_64(asm_instr) \ 477 do { \ 478 auto buffer = i.InputRegister(0); \ 479 auto offset = i.InputRegister32(1); \ 480 auto length = i.InputOperand32(2); \ 481 auto value = i.InputOrZeroRegister64(3); \ 482 Label done; \ 483 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \ 484 __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \ 485 __ Bind(&done); \ 486 } while (0) 487 488 #define ASSEMBLE_SHIFT(asm_instr, width) \ 489 do { \ 490 if (instr->InputAt(1)->IsRegister()) { \ 491 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \ 492 i.InputRegister##width(1)); \ 493 } else { \ 494 uint32_t imm = \ 495 static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \ 496 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \ 497 imm % (width)); \ 498 } \ 499 } while (0) 500 501 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \ 502 do { \ 503 __ asm_instr(i.OutputRegister(), \ 504 MemOperand(i.InputRegister(0), i.InputRegister(1))); \ 505 __ Dmb(InnerShareable, BarrierAll); \ 506 } while (0) 507 508 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \ 509 do { \ 510 __ Dmb(InnerShareable, BarrierAll); \ 511 __ asm_instr(i.InputRegister(2), \ 512 MemOperand(i.InputRegister(0), i.InputRegister(1))); \ 513 __ Dmb(InnerShareable, BarrierAll); \ 514 } while (0) 515 516 #define ASSEMBLE_IEEE754_BINOP(name) \ 517 do { \ 518 FrameScope scope(masm(), StackFrame::MANUAL); \ 519 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 520 0, 2); \ 521 } while (0) 522 523 #define ASSEMBLE_IEEE754_UNOP(name) \ 524 do { \ 525 FrameScope scope(masm(), StackFrame::MANUAL); \ 526 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 527 0, 1); \ 528 } while (0) 529 530 void CodeGenerator::AssembleDeconstructFrame() { 531 const CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 532 if (descriptor->IsCFunctionCall() || descriptor->UseNativeStack()) { 533 __ Mov(csp, fp); 534 } else { 535 __ Mov(jssp, fp); 536 } 537 __ Pop(fp, lr); 538 } 539 540 void CodeGenerator::AssembleDeconstructActivationRecord(int stack_param_delta) { 541 int sp_slot_delta = TailCallFrameStackSlotDelta(stack_param_delta); 542 if (sp_slot_delta > 0) { 543 __ Drop(sp_slot_delta); 544 } 545 frame_access_state()->SetFrameAccessToDefault(); 546 } 547 548 549 void CodeGenerator::AssemblePrepareTailCall(int stack_param_delta) { 550 int sp_slot_delta = TailCallFrameStackSlotDelta(stack_param_delta); 551 if (sp_slot_delta < 0) { 552 __ Claim(-sp_slot_delta); 553 frame_access_state()->IncreaseSPDelta(-sp_slot_delta); 554 } 555 if (frame_access_state()->has_frame()) { 556 __ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset)); 557 __ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset)); 558 } 559 frame_access_state()->SetFrameAccessToSP(); 560 } 561 562 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg, 563 Register scratch1, 564 Register scratch2, 565 Register scratch3) { 566 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3)); 567 Label done; 568 569 // Check if current frame is an arguments adaptor frame. 570 __ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset)); 571 __ Cmp(scratch1, Operand(Smi::FromInt(StackFrame::ARGUMENTS_ADAPTOR))); 572 __ B(ne, &done); 573 574 // Load arguments count from current arguments adaptor frame (note, it 575 // does not include receiver). 576 Register caller_args_count_reg = scratch1; 577 __ Ldr(caller_args_count_reg, 578 MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset)); 579 __ SmiUntag(caller_args_count_reg); 580 581 ParameterCount callee_args_count(args_reg); 582 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2, 583 scratch3); 584 __ bind(&done); 585 } 586 587 // Assembles an instruction after register allocation, producing machine code. 588 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( 589 Instruction* instr) { 590 Arm64OperandConverter i(this, instr); 591 InstructionCode opcode = instr->opcode(); 592 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode); 593 switch (arch_opcode) { 594 case kArchCallCodeObject: { 595 EnsureSpaceForLazyDeopt(); 596 if (instr->InputAt(0)->IsImmediate()) { 597 __ Call(Handle<Code>::cast(i.InputHeapObject(0)), 598 RelocInfo::CODE_TARGET); 599 } else { 600 Register target = i.InputRegister(0); 601 __ Add(target, target, Code::kHeaderSize - kHeapObjectTag); 602 __ Call(target); 603 } 604 RecordCallPosition(instr); 605 // TODO(titzer): this is ugly. JSSP should be a caller-save register 606 // in this case, but it is not possible to express in the register 607 // allocator. 608 CallDescriptor::Flags flags(MiscField::decode(opcode)); 609 if (flags & CallDescriptor::kRestoreJSSP) { 610 __ Ldr(jssp, MemOperand(csp)); 611 __ Mov(csp, jssp); 612 } 613 if (flags & CallDescriptor::kRestoreCSP) { 614 __ Mov(csp, jssp); 615 __ AssertCspAligned(); 616 } 617 frame_access_state()->ClearSPDelta(); 618 break; 619 } 620 case kArchTailCallCodeObjectFromJSFunction: 621 case kArchTailCallCodeObject: { 622 int stack_param_delta = i.InputInt32(instr->InputCount() - 1); 623 AssembleDeconstructActivationRecord(stack_param_delta); 624 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) { 625 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 626 i.TempRegister(0), i.TempRegister(1), 627 i.TempRegister(2)); 628 } 629 if (instr->InputAt(0)->IsImmediate()) { 630 __ Jump(Handle<Code>::cast(i.InputHeapObject(0)), 631 RelocInfo::CODE_TARGET); 632 } else { 633 Register target = i.InputRegister(0); 634 __ Add(target, target, Code::kHeaderSize - kHeapObjectTag); 635 __ Jump(target); 636 } 637 frame_access_state()->ClearSPDelta(); 638 break; 639 } 640 case kArchTailCallAddress: { 641 int stack_param_delta = i.InputInt32(instr->InputCount() - 1); 642 AssembleDeconstructActivationRecord(stack_param_delta); 643 CHECK(!instr->InputAt(0)->IsImmediate()); 644 __ Jump(i.InputRegister(0)); 645 frame_access_state()->ClearSPDelta(); 646 break; 647 } 648 case kArchCallJSFunction: { 649 EnsureSpaceForLazyDeopt(); 650 Register func = i.InputRegister(0); 651 if (FLAG_debug_code) { 652 // Check the function's context matches the context argument. 653 UseScratchRegisterScope scope(masm()); 654 Register temp = scope.AcquireX(); 655 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset)); 656 __ cmp(cp, temp); 657 __ Assert(eq, kWrongFunctionContext); 658 } 659 __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset)); 660 __ Call(x10); 661 RecordCallPosition(instr); 662 // TODO(titzer): this is ugly. JSSP should be a caller-save register 663 // in this case, but it is not possible to express in the register 664 // allocator. 665 CallDescriptor::Flags flags(MiscField::decode(opcode)); 666 if (flags & CallDescriptor::kRestoreJSSP) { 667 __ Ldr(jssp, MemOperand(csp)); 668 __ Mov(csp, jssp); 669 } 670 if (flags & CallDescriptor::kRestoreCSP) { 671 __ Mov(csp, jssp); 672 __ AssertCspAligned(); 673 } 674 frame_access_state()->ClearSPDelta(); 675 break; 676 } 677 case kArchTailCallJSFunctionFromJSFunction: 678 case kArchTailCallJSFunction: { 679 Register func = i.InputRegister(0); 680 if (FLAG_debug_code) { 681 // Check the function's context matches the context argument. 682 UseScratchRegisterScope scope(masm()); 683 Register temp = scope.AcquireX(); 684 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset)); 685 __ cmp(cp, temp); 686 __ Assert(eq, kWrongFunctionContext); 687 } 688 int stack_param_delta = i.InputInt32(instr->InputCount() - 1); 689 AssembleDeconstructActivationRecord(stack_param_delta); 690 if (arch_opcode == kArchTailCallJSFunctionFromJSFunction) { 691 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 692 i.TempRegister(0), i.TempRegister(1), 693 i.TempRegister(2)); 694 } 695 __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset)); 696 __ Jump(x10); 697 frame_access_state()->ClearSPDelta(); 698 break; 699 } 700 case kArchPrepareCallCFunction: 701 // We don't need kArchPrepareCallCFunction on arm64 as the instruction 702 // selector already perform a Claim to reserve space on the stack and 703 // guarantee correct alignment of stack pointer. 704 UNREACHABLE(); 705 break; 706 case kArchPrepareTailCall: 707 AssemblePrepareTailCall(i.InputInt32(instr->InputCount() - 1)); 708 break; 709 case kArchCallCFunction: { 710 int const num_parameters = MiscField::decode(instr->opcode()); 711 if (instr->InputAt(0)->IsImmediate()) { 712 ExternalReference ref = i.InputExternalReference(0); 713 __ CallCFunction(ref, num_parameters, 0); 714 } else { 715 Register func = i.InputRegister(0); 716 __ CallCFunction(func, num_parameters, 0); 717 } 718 // CallCFunction only supports register arguments so we never need to call 719 // frame()->ClearOutgoingParameterSlots() here. 720 DCHECK(frame_access_state()->sp_delta() == 0); 721 break; 722 } 723 case kArchJmp: 724 AssembleArchJump(i.InputRpo(0)); 725 break; 726 case kArchTableSwitch: 727 AssembleArchTableSwitch(instr); 728 break; 729 case kArchLookupSwitch: 730 AssembleArchLookupSwitch(instr); 731 break; 732 case kArchDebugBreak: 733 __ Debug("kArchDebugBreak", 0, BREAK); 734 break; 735 case kArchComment: { 736 Address comment_string = i.InputExternalReference(0).address(); 737 __ RecordComment(reinterpret_cast<const char*>(comment_string)); 738 break; 739 } 740 case kArchNop: 741 case kArchThrowTerminator: 742 // don't emit code for nops. 743 break; 744 case kArchDeoptimize: { 745 int deopt_state_id = 746 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); 747 Deoptimizer::BailoutType bailout_type = 748 Deoptimizer::BailoutType(MiscField::decode(instr->opcode())); 749 CodeGenResult result = 750 AssembleDeoptimizerCall(deopt_state_id, bailout_type); 751 if (result != kSuccess) return result; 752 break; 753 } 754 case kArchRet: 755 AssembleReturn(); 756 break; 757 case kArchStackPointer: 758 __ mov(i.OutputRegister(), masm()->StackPointer()); 759 break; 760 case kArchFramePointer: 761 __ mov(i.OutputRegister(), fp); 762 break; 763 case kArchParentFramePointer: 764 if (frame_access_state()->has_frame()) { 765 __ ldr(i.OutputRegister(), MemOperand(fp, 0)); 766 } else { 767 __ mov(i.OutputRegister(), fp); 768 } 769 break; 770 case kArchTruncateDoubleToI: 771 __ TruncateDoubleToI(i.OutputRegister(), i.InputDoubleRegister(0)); 772 break; 773 case kArchStoreWithWriteBarrier: { 774 RecordWriteMode mode = 775 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode())); 776 AddressingMode addressing_mode = 777 AddressingModeField::decode(instr->opcode()); 778 Register object = i.InputRegister(0); 779 Operand index(0); 780 if (addressing_mode == kMode_MRI) { 781 index = Operand(i.InputInt64(1)); 782 } else { 783 DCHECK_EQ(addressing_mode, kMode_MRR); 784 index = Operand(i.InputRegister(1)); 785 } 786 Register value = i.InputRegister(2); 787 Register scratch0 = i.TempRegister(0); 788 Register scratch1 = i.TempRegister(1); 789 auto ool = new (zone()) OutOfLineRecordWrite(this, object, index, value, 790 scratch0, scratch1, mode); 791 __ Str(value, MemOperand(object, index)); 792 __ CheckPageFlagSet(object, scratch0, 793 MemoryChunk::kPointersFromHereAreInterestingMask, 794 ool->entry()); 795 __ Bind(ool->exit()); 796 break; 797 } 798 case kArchStackSlot: { 799 FrameOffset offset = 800 frame_access_state()->GetFrameOffset(i.InputInt32(0)); 801 Register base; 802 if (offset.from_stack_pointer()) { 803 base = __ StackPointer(); 804 } else { 805 base = fp; 806 } 807 __ Add(i.OutputRegister(0), base, Operand(offset.offset())); 808 break; 809 } 810 case kIeee754Float64Atan: 811 ASSEMBLE_IEEE754_UNOP(atan); 812 break; 813 case kIeee754Float64Atan2: 814 ASSEMBLE_IEEE754_BINOP(atan2); 815 break; 816 case kIeee754Float64Cos: 817 ASSEMBLE_IEEE754_UNOP(cos); 818 break; 819 case kIeee754Float64Cbrt: 820 ASSEMBLE_IEEE754_UNOP(cbrt); 821 break; 822 case kIeee754Float64Exp: 823 ASSEMBLE_IEEE754_UNOP(exp); 824 break; 825 case kIeee754Float64Expm1: 826 ASSEMBLE_IEEE754_UNOP(expm1); 827 break; 828 case kIeee754Float64Atanh: 829 ASSEMBLE_IEEE754_UNOP(atanh); 830 break; 831 case kIeee754Float64Log: 832 ASSEMBLE_IEEE754_UNOP(log); 833 break; 834 case kIeee754Float64Log1p: 835 ASSEMBLE_IEEE754_UNOP(log1p); 836 break; 837 case kIeee754Float64Log2: 838 ASSEMBLE_IEEE754_UNOP(log2); 839 break; 840 case kIeee754Float64Log10: 841 ASSEMBLE_IEEE754_UNOP(log10); 842 break; 843 case kIeee754Float64Sin: 844 ASSEMBLE_IEEE754_UNOP(sin); 845 break; 846 case kIeee754Float64Tan: 847 ASSEMBLE_IEEE754_UNOP(tan); 848 break; 849 case kArm64Float32RoundDown: 850 __ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0)); 851 break; 852 case kArm64Float64RoundDown: 853 __ Frintm(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 854 break; 855 case kArm64Float32RoundUp: 856 __ Frintp(i.OutputFloat32Register(), i.InputFloat32Register(0)); 857 break; 858 case kArm64Float64RoundUp: 859 __ Frintp(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 860 break; 861 case kArm64Float64RoundTiesAway: 862 __ Frinta(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 863 break; 864 case kArm64Float32RoundTruncate: 865 __ Frintz(i.OutputFloat32Register(), i.InputFloat32Register(0)); 866 break; 867 case kArm64Float64RoundTruncate: 868 __ Frintz(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 869 break; 870 case kArm64Float32RoundTiesEven: 871 __ Frintn(i.OutputFloat32Register(), i.InputFloat32Register(0)); 872 break; 873 case kArm64Float64RoundTiesEven: 874 __ Frintn(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 875 break; 876 case kArm64Add: 877 if (FlagsModeField::decode(opcode) != kFlags_none) { 878 __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0), 879 i.InputOperand2_64(1)); 880 } else { 881 __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0), 882 i.InputOperand2_64(1)); 883 } 884 break; 885 case kArm64Add32: 886 if (FlagsModeField::decode(opcode) != kFlags_none) { 887 __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0), 888 i.InputOperand2_32(1)); 889 } else { 890 __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0), 891 i.InputOperand2_32(1)); 892 } 893 break; 894 case kArm64And: 895 __ And(i.OutputRegister(), i.InputOrZeroRegister64(0), 896 i.InputOperand2_64(1)); 897 break; 898 case kArm64And32: 899 __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0), 900 i.InputOperand2_32(1)); 901 break; 902 case kArm64Bic: 903 __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0), 904 i.InputOperand2_64(1)); 905 break; 906 case kArm64Bic32: 907 __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0), 908 i.InputOperand2_32(1)); 909 break; 910 case kArm64Mul: 911 __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 912 break; 913 case kArm64Mul32: 914 __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 915 break; 916 case kArm64Smull: 917 __ Smull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1)); 918 break; 919 case kArm64Umull: 920 __ Umull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1)); 921 break; 922 case kArm64Madd: 923 __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1), 924 i.InputRegister(2)); 925 break; 926 case kArm64Madd32: 927 __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1), 928 i.InputRegister32(2)); 929 break; 930 case kArm64Msub: 931 __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1), 932 i.InputRegister(2)); 933 break; 934 case kArm64Msub32: 935 __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1), 936 i.InputRegister32(2)); 937 break; 938 case kArm64Mneg: 939 __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 940 break; 941 case kArm64Mneg32: 942 __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 943 break; 944 case kArm64Idiv: 945 __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 946 break; 947 case kArm64Idiv32: 948 __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 949 break; 950 case kArm64Udiv: 951 __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); 952 break; 953 case kArm64Udiv32: 954 __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1)); 955 break; 956 case kArm64Imod: { 957 UseScratchRegisterScope scope(masm()); 958 Register temp = scope.AcquireX(); 959 __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1)); 960 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0)); 961 break; 962 } 963 case kArm64Imod32: { 964 UseScratchRegisterScope scope(masm()); 965 Register temp = scope.AcquireW(); 966 __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1)); 967 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1), 968 i.InputRegister32(0)); 969 break; 970 } 971 case kArm64Umod: { 972 UseScratchRegisterScope scope(masm()); 973 Register temp = scope.AcquireX(); 974 __ Udiv(temp, i.InputRegister(0), i.InputRegister(1)); 975 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0)); 976 break; 977 } 978 case kArm64Umod32: { 979 UseScratchRegisterScope scope(masm()); 980 Register temp = scope.AcquireW(); 981 __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1)); 982 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1), 983 i.InputRegister32(0)); 984 break; 985 } 986 case kArm64Not: 987 __ Mvn(i.OutputRegister(), i.InputOperand(0)); 988 break; 989 case kArm64Not32: 990 __ Mvn(i.OutputRegister32(), i.InputOperand32(0)); 991 break; 992 case kArm64Or: 993 __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0), 994 i.InputOperand2_64(1)); 995 break; 996 case kArm64Or32: 997 __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0), 998 i.InputOperand2_32(1)); 999 break; 1000 case kArm64Orn: 1001 __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0), 1002 i.InputOperand2_64(1)); 1003 break; 1004 case kArm64Orn32: 1005 __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1006 i.InputOperand2_32(1)); 1007 break; 1008 case kArm64Eor: 1009 __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0), 1010 i.InputOperand2_64(1)); 1011 break; 1012 case kArm64Eor32: 1013 __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1014 i.InputOperand2_32(1)); 1015 break; 1016 case kArm64Eon: 1017 __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0), 1018 i.InputOperand2_64(1)); 1019 break; 1020 case kArm64Eon32: 1021 __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1022 i.InputOperand2_32(1)); 1023 break; 1024 case kArm64Sub: 1025 if (FlagsModeField::decode(opcode) != kFlags_none) { 1026 __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0), 1027 i.InputOperand2_64(1)); 1028 } else { 1029 __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0), 1030 i.InputOperand2_64(1)); 1031 } 1032 break; 1033 case kArm64Sub32: 1034 if (FlagsModeField::decode(opcode) != kFlags_none) { 1035 __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1036 i.InputOperand2_32(1)); 1037 } else { 1038 __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0), 1039 i.InputOperand2_32(1)); 1040 } 1041 break; 1042 case kArm64Lsl: 1043 ASSEMBLE_SHIFT(Lsl, 64); 1044 break; 1045 case kArm64Lsl32: 1046 ASSEMBLE_SHIFT(Lsl, 32); 1047 break; 1048 case kArm64Lsr: 1049 ASSEMBLE_SHIFT(Lsr, 64); 1050 break; 1051 case kArm64Lsr32: 1052 ASSEMBLE_SHIFT(Lsr, 32); 1053 break; 1054 case kArm64Asr: 1055 ASSEMBLE_SHIFT(Asr, 64); 1056 break; 1057 case kArm64Asr32: 1058 ASSEMBLE_SHIFT(Asr, 32); 1059 break; 1060 case kArm64Ror: 1061 ASSEMBLE_SHIFT(Ror, 64); 1062 break; 1063 case kArm64Ror32: 1064 ASSEMBLE_SHIFT(Ror, 32); 1065 break; 1066 case kArm64Mov32: 1067 __ Mov(i.OutputRegister32(), i.InputRegister32(0)); 1068 break; 1069 case kArm64Sxtb32: 1070 __ Sxtb(i.OutputRegister32(), i.InputRegister32(0)); 1071 break; 1072 case kArm64Sxth32: 1073 __ Sxth(i.OutputRegister32(), i.InputRegister32(0)); 1074 break; 1075 case kArm64Sxtw: 1076 __ Sxtw(i.OutputRegister(), i.InputRegister32(0)); 1077 break; 1078 case kArm64Sbfx32: 1079 __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1), 1080 i.InputInt5(2)); 1081 break; 1082 case kArm64Ubfx: 1083 __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1), 1084 i.InputInt6(2)); 1085 break; 1086 case kArm64Ubfx32: 1087 __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1), 1088 i.InputInt5(2)); 1089 break; 1090 case kArm64Ubfiz32: 1091 __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1), 1092 i.InputInt5(2)); 1093 break; 1094 case kArm64Bfi: 1095 __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2), 1096 i.InputInt6(3)); 1097 break; 1098 case kArm64TestAndBranch32: 1099 case kArm64TestAndBranch: 1100 // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch. 1101 break; 1102 case kArm64CompareAndBranch32: 1103 case kArm64CompareAndBranch: 1104 // Pseudo instruction turned into cbz/cbnz in AssembleArchBranch. 1105 break; 1106 case kArm64ClaimCSP: { 1107 int count = RoundUp(i.InputInt32(0), 2); 1108 Register prev = __ StackPointer(); 1109 if (prev.Is(jssp)) { 1110 // TODO(titzer): make this a macro-assembler method. 1111 // Align the CSP and store the previous JSSP on the stack. 1112 UseScratchRegisterScope scope(masm()); 1113 Register tmp = scope.AcquireX(); 1114 1115 int sp_alignment = __ ActivationFrameAlignment(); 1116 __ Sub(tmp, jssp, kPointerSize); 1117 __ And(tmp, tmp, Operand(~static_cast<uint64_t>(sp_alignment - 1))); 1118 __ Mov(csp, tmp); 1119 __ Str(jssp, MemOperand(csp)); 1120 if (count > 0) { 1121 __ SetStackPointer(csp); 1122 __ Claim(count); 1123 __ SetStackPointer(prev); 1124 } 1125 } else { 1126 __ AssertCspAligned(); 1127 if (count > 0) { 1128 __ Claim(count); 1129 frame_access_state()->IncreaseSPDelta(count); 1130 } 1131 } 1132 break; 1133 } 1134 case kArm64ClaimJSSP: { 1135 int count = i.InputInt32(0); 1136 if (csp.Is(__ StackPointer())) { 1137 // No JSSP is set up. Compute it from the CSP. 1138 __ AssertCspAligned(); 1139 if (count > 0) { 1140 int even = RoundUp(count, 2); 1141 __ Sub(jssp, csp, count * kPointerSize); 1142 __ Sub(csp, csp, even * kPointerSize); // Must always be aligned. 1143 frame_access_state()->IncreaseSPDelta(even); 1144 } else { 1145 __ Mov(jssp, csp); 1146 } 1147 } else { 1148 // JSSP is the current stack pointer, just use regular Claim(). 1149 __ Claim(count); 1150 frame_access_state()->IncreaseSPDelta(count); 1151 } 1152 break; 1153 } 1154 case kArm64PokeCSP: // fall through 1155 case kArm64PokeJSSP: { 1156 Register prev = __ StackPointer(); 1157 __ SetStackPointer(arch_opcode == kArm64PokeCSP ? csp : jssp); 1158 Operand operand(i.InputInt32(1) * kPointerSize); 1159 if (instr->InputAt(0)->IsFPRegister()) { 1160 __ Poke(i.InputFloat64Register(0), operand); 1161 } else { 1162 __ Poke(i.InputRegister(0), operand); 1163 } 1164 __ SetStackPointer(prev); 1165 break; 1166 } 1167 case kArm64PokePair: { 1168 int slot = i.InputInt32(2) - 1; 1169 if (instr->InputAt(0)->IsFPRegister()) { 1170 __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0), 1171 slot * kPointerSize); 1172 } else { 1173 __ PokePair(i.InputRegister(1), i.InputRegister(0), 1174 slot * kPointerSize); 1175 } 1176 break; 1177 } 1178 case kArm64Clz: 1179 __ Clz(i.OutputRegister64(), i.InputRegister64(0)); 1180 break; 1181 case kArm64Clz32: 1182 __ Clz(i.OutputRegister32(), i.InputRegister32(0)); 1183 break; 1184 case kArm64Rbit: 1185 __ Rbit(i.OutputRegister64(), i.InputRegister64(0)); 1186 break; 1187 case kArm64Rbit32: 1188 __ Rbit(i.OutputRegister32(), i.InputRegister32(0)); 1189 break; 1190 case kArm64Cmp: 1191 __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand(1)); 1192 break; 1193 case kArm64Cmp32: 1194 __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1)); 1195 break; 1196 case kArm64Cmn: 1197 __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand(1)); 1198 break; 1199 case kArm64Cmn32: 1200 __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1)); 1201 break; 1202 case kArm64Tst: 1203 __ Tst(i.InputRegister(0), i.InputOperand(1)); 1204 break; 1205 case kArm64Tst32: 1206 __ Tst(i.InputRegister32(0), i.InputOperand32(1)); 1207 break; 1208 case kArm64Float32Cmp: 1209 if (instr->InputAt(1)->IsFPRegister()) { 1210 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1)); 1211 } else { 1212 DCHECK(instr->InputAt(1)->IsImmediate()); 1213 // 0.0 is the only immediate supported by fcmp instructions. 1214 DCHECK(i.InputFloat32(1) == 0.0f); 1215 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1)); 1216 } 1217 break; 1218 case kArm64Float32Add: 1219 __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0), 1220 i.InputFloat32Register(1)); 1221 break; 1222 case kArm64Float32Sub: 1223 __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0), 1224 i.InputFloat32Register(1)); 1225 break; 1226 case kArm64Float32Mul: 1227 __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0), 1228 i.InputFloat32Register(1)); 1229 break; 1230 case kArm64Float32Div: 1231 __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0), 1232 i.InputFloat32Register(1)); 1233 break; 1234 case kArm64Float32Max: 1235 // (b < a) ? a : b 1236 __ Fcmp(i.InputFloat32Register(1), i.InputFloat32Register(0)); 1237 __ Fcsel(i.OutputFloat32Register(), i.InputFloat32Register(0), 1238 i.InputFloat32Register(1), lo); 1239 break; 1240 case kArm64Float32Min: 1241 // (a < b) ? a : b 1242 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1)); 1243 __ Fcsel(i.OutputFloat32Register(), i.InputFloat32Register(0), 1244 i.InputFloat32Register(1), lo); 1245 break; 1246 case kArm64Float32Abs: 1247 __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0)); 1248 break; 1249 case kArm64Float32Neg: 1250 __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0)); 1251 break; 1252 case kArm64Float32Sqrt: 1253 __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0)); 1254 break; 1255 case kArm64Float64Cmp: 1256 if (instr->InputAt(1)->IsFPRegister()) { 1257 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1258 } else { 1259 DCHECK(instr->InputAt(1)->IsImmediate()); 1260 // 0.0 is the only immediate supported by fcmp instructions. 1261 DCHECK(i.InputDouble(1) == 0.0); 1262 __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1)); 1263 } 1264 break; 1265 case kArm64Float64Add: 1266 __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1267 i.InputDoubleRegister(1)); 1268 break; 1269 case kArm64Float64Sub: 1270 __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1271 i.InputDoubleRegister(1)); 1272 break; 1273 case kArm64Float64Mul: 1274 __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1275 i.InputDoubleRegister(1)); 1276 break; 1277 case kArm64Float64Div: 1278 __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1279 i.InputDoubleRegister(1)); 1280 break; 1281 case kArm64Float64Mod: { 1282 // TODO(dcarney): implement directly. See note in lithium-codegen-arm64.cc 1283 FrameScope scope(masm(), StackFrame::MANUAL); 1284 DCHECK(d0.is(i.InputDoubleRegister(0))); 1285 DCHECK(d1.is(i.InputDoubleRegister(1))); 1286 DCHECK(d0.is(i.OutputDoubleRegister())); 1287 // TODO(dcarney): make sure this saves all relevant registers. 1288 __ CallCFunction(ExternalReference::mod_two_doubles_operation(isolate()), 1289 0, 2); 1290 break; 1291 } 1292 case kArm64Float64Max: 1293 // (b < a) ? a : b 1294 __ Fcmp(i.InputDoubleRegister(1), i.InputDoubleRegister(0)); 1295 __ Fcsel(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1296 i.InputDoubleRegister(1), lo); 1297 break; 1298 case kArm64Float64Min: 1299 // (a < b) ? a : b 1300 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1301 __ Fcsel(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1302 i.InputDoubleRegister(1), lo); 1303 break; 1304 case kArm64Float64Abs: 1305 __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1306 break; 1307 case kArm64Float64Neg: 1308 __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1309 break; 1310 case kArm64Float64Sqrt: 1311 __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1312 break; 1313 case kArm64Float32ToFloat64: 1314 __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S()); 1315 break; 1316 case kArm64Float64ToFloat32: 1317 __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0)); 1318 break; 1319 case kArm64Float32ToInt32: 1320 __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0)); 1321 break; 1322 case kArm64Float64ToInt32: 1323 __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0)); 1324 break; 1325 case kArm64Float32ToUint32: 1326 __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0)); 1327 break; 1328 case kArm64Float64ToUint32: 1329 __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0)); 1330 break; 1331 case kArm64Float32ToInt64: 1332 __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0)); 1333 if (i.OutputCount() > 1) { 1334 __ Mov(i.OutputRegister(1), 1); 1335 Label done; 1336 __ Cmp(i.OutputRegister(0), 1); 1337 __ Ccmp(i.OutputRegister(0), -1, VFlag, vc); 1338 __ Fccmp(i.InputFloat32Register(0), i.InputFloat32Register(0), VFlag, 1339 vc); 1340 __ B(vc, &done); 1341 __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN)); 1342 __ Cset(i.OutputRegister(1), eq); 1343 __ Bind(&done); 1344 } 1345 break; 1346 case kArm64Float64ToInt64: 1347 __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0)); 1348 if (i.OutputCount() > 1) { 1349 __ Mov(i.OutputRegister(1), 1); 1350 Label done; 1351 __ Cmp(i.OutputRegister(0), 1); 1352 __ Ccmp(i.OutputRegister(0), -1, VFlag, vc); 1353 __ Fccmp(i.InputDoubleRegister(0), i.InputDoubleRegister(0), VFlag, vc); 1354 __ B(vc, &done); 1355 __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN)); 1356 __ Cset(i.OutputRegister(1), eq); 1357 __ Bind(&done); 1358 } 1359 break; 1360 case kArm64Float32ToUint64: 1361 __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0)); 1362 if (i.OutputCount() > 1) { 1363 __ Fcmp(i.InputFloat32Register(0), -1.0); 1364 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt); 1365 __ Cset(i.OutputRegister(1), ne); 1366 } 1367 break; 1368 case kArm64Float64ToUint64: 1369 __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0)); 1370 if (i.OutputCount() > 1) { 1371 __ Fcmp(i.InputDoubleRegister(0), -1.0); 1372 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt); 1373 __ Cset(i.OutputRegister(1), ne); 1374 } 1375 break; 1376 case kArm64Int32ToFloat32: 1377 __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0)); 1378 break; 1379 case kArm64Int32ToFloat64: 1380 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0)); 1381 break; 1382 case kArm64Int64ToFloat32: 1383 __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0)); 1384 break; 1385 case kArm64Int64ToFloat64: 1386 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0)); 1387 break; 1388 case kArm64Uint32ToFloat32: 1389 __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0)); 1390 break; 1391 case kArm64Uint32ToFloat64: 1392 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0)); 1393 break; 1394 case kArm64Uint64ToFloat32: 1395 __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0)); 1396 break; 1397 case kArm64Uint64ToFloat64: 1398 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0)); 1399 break; 1400 case kArm64Float64ExtractLowWord32: 1401 __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0)); 1402 break; 1403 case kArm64Float64ExtractHighWord32: 1404 // TODO(arm64): This should use MOV (to general) when NEON is supported. 1405 __ Fmov(i.OutputRegister(), i.InputFloat64Register(0)); 1406 __ Lsr(i.OutputRegister(), i.OutputRegister(), 32); 1407 break; 1408 case kArm64Float64InsertLowWord32: { 1409 // TODO(arm64): This should use MOV (from general) when NEON is supported. 1410 UseScratchRegisterScope scope(masm()); 1411 Register tmp = scope.AcquireX(); 1412 __ Fmov(tmp, i.InputFloat64Register(0)); 1413 __ Bfi(tmp, i.InputRegister(1), 0, 32); 1414 __ Fmov(i.OutputFloat64Register(), tmp); 1415 break; 1416 } 1417 case kArm64Float64InsertHighWord32: { 1418 // TODO(arm64): This should use MOV (from general) when NEON is supported. 1419 UseScratchRegisterScope scope(masm()); 1420 Register tmp = scope.AcquireX(); 1421 __ Fmov(tmp.W(), i.InputFloat32Register(0)); 1422 __ Bfi(tmp, i.InputRegister(1), 32, 32); 1423 __ Fmov(i.OutputFloat64Register(), tmp); 1424 break; 1425 } 1426 case kArm64Float64MoveU64: 1427 __ Fmov(i.OutputFloat64Register(), i.InputRegister(0)); 1428 break; 1429 case kArm64Float64SilenceNaN: 1430 __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); 1431 break; 1432 case kArm64U64MoveFloat64: 1433 __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0)); 1434 break; 1435 case kArm64Ldrb: 1436 __ Ldrb(i.OutputRegister(), i.MemoryOperand()); 1437 break; 1438 case kArm64Ldrsb: 1439 __ Ldrsb(i.OutputRegister(), i.MemoryOperand()); 1440 break; 1441 case kArm64Strb: 1442 __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1)); 1443 break; 1444 case kArm64Ldrh: 1445 __ Ldrh(i.OutputRegister(), i.MemoryOperand()); 1446 break; 1447 case kArm64Ldrsh: 1448 __ Ldrsh(i.OutputRegister(), i.MemoryOperand()); 1449 break; 1450 case kArm64Strh: 1451 __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1)); 1452 break; 1453 case kArm64LdrW: 1454 __ Ldr(i.OutputRegister32(), i.MemoryOperand()); 1455 break; 1456 case kArm64StrW: 1457 __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1)); 1458 break; 1459 case kArm64Ldr: 1460 __ Ldr(i.OutputRegister(), i.MemoryOperand()); 1461 break; 1462 case kArm64Str: 1463 __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1)); 1464 break; 1465 case kArm64LdrS: 1466 __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand()); 1467 break; 1468 case kArm64StrS: 1469 __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1)); 1470 break; 1471 case kArm64LdrD: 1472 __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand()); 1473 break; 1474 case kArm64StrD: 1475 __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1)); 1476 break; 1477 case kCheckedLoadInt8: 1478 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsb); 1479 break; 1480 case kCheckedLoadUint8: 1481 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrb); 1482 break; 1483 case kCheckedLoadInt16: 1484 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsh); 1485 break; 1486 case kCheckedLoadUint16: 1487 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrh); 1488 break; 1489 case kCheckedLoadWord32: 1490 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldr); 1491 break; 1492 case kCheckedLoadWord64: 1493 ASSEMBLE_CHECKED_LOAD_INTEGER_64(Ldr); 1494 break; 1495 case kCheckedLoadFloat32: 1496 ASSEMBLE_CHECKED_LOAD_FLOAT(32); 1497 break; 1498 case kCheckedLoadFloat64: 1499 ASSEMBLE_CHECKED_LOAD_FLOAT(64); 1500 break; 1501 case kCheckedStoreWord8: 1502 ASSEMBLE_CHECKED_STORE_INTEGER(Strb); 1503 break; 1504 case kCheckedStoreWord16: 1505 ASSEMBLE_CHECKED_STORE_INTEGER(Strh); 1506 break; 1507 case kCheckedStoreWord32: 1508 ASSEMBLE_CHECKED_STORE_INTEGER(Str); 1509 break; 1510 case kCheckedStoreWord64: 1511 ASSEMBLE_CHECKED_STORE_INTEGER_64(Str); 1512 break; 1513 case kCheckedStoreFloat32: 1514 ASSEMBLE_CHECKED_STORE_FLOAT(32); 1515 break; 1516 case kCheckedStoreFloat64: 1517 ASSEMBLE_CHECKED_STORE_FLOAT(64); 1518 break; 1519 case kAtomicLoadInt8: 1520 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb); 1521 break; 1522 case kAtomicLoadUint8: 1523 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb); 1524 break; 1525 case kAtomicLoadInt16: 1526 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh); 1527 break; 1528 case kAtomicLoadUint16: 1529 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh); 1530 break; 1531 case kAtomicLoadWord32: 1532 __ Ldr(i.OutputRegister32(), 1533 MemOperand(i.InputRegister(0), i.InputRegister(1))); 1534 __ Dmb(InnerShareable, BarrierAll); 1535 break; 1536 case kAtomicStoreWord8: 1537 ASSEMBLE_ATOMIC_STORE_INTEGER(Strb); 1538 break; 1539 case kAtomicStoreWord16: 1540 ASSEMBLE_ATOMIC_STORE_INTEGER(Strh); 1541 break; 1542 case kAtomicStoreWord32: 1543 __ Dmb(InnerShareable, BarrierAll); 1544 __ Str(i.InputRegister32(2), 1545 MemOperand(i.InputRegister(0), i.InputRegister(1))); 1546 __ Dmb(InnerShareable, BarrierAll); 1547 break; 1548 } 1549 return kSuccess; 1550 } // NOLINT(readability/fn_size) 1551 1552 1553 // Assemble branches after this instruction. 1554 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { 1555 Arm64OperandConverter i(this, instr); 1556 Label* tlabel = branch->true_label; 1557 Label* flabel = branch->false_label; 1558 FlagsCondition condition = branch->condition; 1559 ArchOpcode opcode = instr->arch_opcode(); 1560 1561 if (opcode == kArm64CompareAndBranch32) { 1562 switch (condition) { 1563 case kEqual: 1564 __ Cbz(i.InputRegister32(0), tlabel); 1565 break; 1566 case kNotEqual: 1567 __ Cbnz(i.InputRegister32(0), tlabel); 1568 break; 1569 default: 1570 UNREACHABLE(); 1571 } 1572 } else if (opcode == kArm64CompareAndBranch) { 1573 switch (condition) { 1574 case kEqual: 1575 __ Cbz(i.InputRegister64(0), tlabel); 1576 break; 1577 case kNotEqual: 1578 __ Cbnz(i.InputRegister64(0), tlabel); 1579 break; 1580 default: 1581 UNREACHABLE(); 1582 } 1583 } else if (opcode == kArm64TestAndBranch32) { 1584 switch (condition) { 1585 case kEqual: 1586 __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel); 1587 break; 1588 case kNotEqual: 1589 __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel); 1590 break; 1591 default: 1592 UNREACHABLE(); 1593 } 1594 } else if (opcode == kArm64TestAndBranch) { 1595 switch (condition) { 1596 case kEqual: 1597 __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel); 1598 break; 1599 case kNotEqual: 1600 __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel); 1601 break; 1602 default: 1603 UNREACHABLE(); 1604 } 1605 } else { 1606 Condition cc = FlagsConditionToCondition(condition); 1607 __ B(cc, tlabel); 1608 } 1609 if (!branch->fallthru) __ B(flabel); // no fallthru to flabel. 1610 } 1611 1612 1613 void CodeGenerator::AssembleArchJump(RpoNumber target) { 1614 if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target)); 1615 } 1616 1617 1618 // Assemble boolean materializations after this instruction. 1619 void CodeGenerator::AssembleArchBoolean(Instruction* instr, 1620 FlagsCondition condition) { 1621 Arm64OperandConverter i(this, instr); 1622 1623 // Materialize a full 64-bit 1 or 0 value. The result register is always the 1624 // last output of the instruction. 1625 DCHECK_NE(0u, instr->OutputCount()); 1626 Register reg = i.OutputRegister(instr->OutputCount() - 1); 1627 Condition cc = FlagsConditionToCondition(condition); 1628 __ Cset(reg, cc); 1629 } 1630 1631 1632 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) { 1633 Arm64OperandConverter i(this, instr); 1634 Register input = i.InputRegister32(0); 1635 for (size_t index = 2; index < instr->InputCount(); index += 2) { 1636 __ Cmp(input, i.InputInt32(index + 0)); 1637 __ B(eq, GetLabel(i.InputRpo(index + 1))); 1638 } 1639 AssembleArchJump(i.InputRpo(1)); 1640 } 1641 1642 1643 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) { 1644 Arm64OperandConverter i(this, instr); 1645 UseScratchRegisterScope scope(masm()); 1646 Register input = i.InputRegister32(0); 1647 Register temp = scope.AcquireX(); 1648 size_t const case_count = instr->InputCount() - 2; 1649 Label table; 1650 __ Cmp(input, case_count); 1651 __ B(hs, GetLabel(i.InputRpo(1))); 1652 __ Adr(temp, &table); 1653 __ Add(temp, temp, Operand(input, UXTW, 2)); 1654 __ Br(temp); 1655 __ StartBlockPools(); 1656 __ Bind(&table); 1657 for (size_t index = 0; index < case_count; ++index) { 1658 __ B(GetLabel(i.InputRpo(index + 2))); 1659 } 1660 __ EndBlockPools(); 1661 } 1662 1663 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall( 1664 int deoptimization_id, Deoptimizer::BailoutType bailout_type) { 1665 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry( 1666 isolate(), deoptimization_id, bailout_type); 1667 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts; 1668 __ Call(deopt_entry, RelocInfo::RUNTIME_ENTRY); 1669 return kSuccess; 1670 } 1671 1672 void CodeGenerator::FinishFrame(Frame* frame) { 1673 frame->AlignFrame(16); 1674 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1675 1676 if (descriptor->UseNativeStack() || descriptor->IsCFunctionCall()) { 1677 __ SetStackPointer(csp); 1678 } else { 1679 __ SetStackPointer(jssp); 1680 } 1681 1682 // Save FP registers. 1683 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 1684 descriptor->CalleeSavedFPRegisters()); 1685 int saved_count = saves_fp.Count(); 1686 if (saved_count != 0) { 1687 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list()); 1688 frame->AllocateSavedCalleeRegisterSlots(saved_count * 1689 (kDoubleSize / kPointerSize)); 1690 } 1691 1692 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1693 descriptor->CalleeSavedRegisters()); 1694 saved_count = saves.Count(); 1695 if (saved_count != 0) { 1696 frame->AllocateSavedCalleeRegisterSlots(saved_count); 1697 } 1698 } 1699 1700 void CodeGenerator::AssembleConstructFrame() { 1701 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1702 if (descriptor->UseNativeStack()) { 1703 __ AssertCspAligned(); 1704 } 1705 1706 if (frame_access_state()->has_frame()) { 1707 if (descriptor->IsJSFunctionCall()) { 1708 DCHECK(!descriptor->UseNativeStack()); 1709 __ Prologue(this->info()->GeneratePreagedPrologue()); 1710 } else { 1711 if (descriptor->IsCFunctionCall()) { 1712 __ Push(lr, fp); 1713 __ Mov(fp, masm_.StackPointer()); 1714 __ Claim(frame()->GetSpillSlotCount()); 1715 } else { 1716 __ StubPrologue(info()->GetOutputStackFrameType(), 1717 frame()->GetTotalFrameSlotCount()); 1718 } 1719 } 1720 } 1721 1722 int shrink_slots = frame()->GetSpillSlotCount(); 1723 1724 if (info()->is_osr()) { 1725 // TurboFan OSR-compiled functions cannot be entered directly. 1726 __ Abort(kShouldNotDirectlyEnterOsrFunction); 1727 1728 // Unoptimized code jumps directly to this entrypoint while the unoptimized 1729 // frame is still on the stack. Optimized code uses OSR values directly from 1730 // the unoptimized frame. Thus, all that needs to be done is to allocate the 1731 // remaining stack slots. 1732 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --"); 1733 osr_pc_offset_ = __ pc_offset(); 1734 shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots(); 1735 } 1736 1737 if (descriptor->IsJSFunctionCall()) { 1738 __ Claim(shrink_slots); 1739 } 1740 1741 // Save FP registers. 1742 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 1743 descriptor->CalleeSavedFPRegisters()); 1744 int saved_count = saves_fp.Count(); 1745 if (saved_count != 0) { 1746 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list()); 1747 __ PushCPURegList(saves_fp); 1748 } 1749 // Save registers. 1750 // TODO(palfia): TF save list is not in sync with 1751 // CPURegList::GetCalleeSaved(): x30 is missing. 1752 // DCHECK(saves.list() == CPURegList::GetCalleeSaved().list()); 1753 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1754 descriptor->CalleeSavedRegisters()); 1755 saved_count = saves.Count(); 1756 if (saved_count != 0) { 1757 __ PushCPURegList(saves); 1758 } 1759 } 1760 1761 1762 void CodeGenerator::AssembleReturn() { 1763 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1764 1765 // Restore registers. 1766 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1767 descriptor->CalleeSavedRegisters()); 1768 if (saves.Count() != 0) { 1769 __ PopCPURegList(saves); 1770 } 1771 1772 // Restore fp registers. 1773 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 1774 descriptor->CalleeSavedFPRegisters()); 1775 if (saves_fp.Count() != 0) { 1776 __ PopCPURegList(saves_fp); 1777 } 1778 1779 int pop_count = static_cast<int>(descriptor->StackParameterCount()); 1780 if (descriptor->IsCFunctionCall()) { 1781 AssembleDeconstructFrame(); 1782 } else if (frame_access_state()->has_frame()) { 1783 // Canonicalize JSFunction return sites for now. 1784 if (return_label_.is_bound()) { 1785 __ B(&return_label_); 1786 return; 1787 } else { 1788 __ Bind(&return_label_); 1789 AssembleDeconstructFrame(); 1790 if (descriptor->UseNativeStack()) { 1791 pop_count += (pop_count & 1); // align 1792 } 1793 } 1794 } else if (descriptor->UseNativeStack()) { 1795 pop_count += (pop_count & 1); // align 1796 } 1797 __ Drop(pop_count); 1798 1799 if (descriptor->UseNativeStack()) { 1800 __ AssertCspAligned(); 1801 } 1802 __ Ret(); 1803 } 1804 1805 1806 void CodeGenerator::AssembleMove(InstructionOperand* source, 1807 InstructionOperand* destination) { 1808 Arm64OperandConverter g(this, nullptr); 1809 // Dispatch on the source and destination operand kinds. Not all 1810 // combinations are possible. 1811 if (source->IsRegister()) { 1812 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 1813 Register src = g.ToRegister(source); 1814 if (destination->IsRegister()) { 1815 __ Mov(g.ToRegister(destination), src); 1816 } else { 1817 __ Str(src, g.ToMemOperand(destination, masm())); 1818 } 1819 } else if (source->IsStackSlot()) { 1820 MemOperand src = g.ToMemOperand(source, masm()); 1821 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 1822 if (destination->IsRegister()) { 1823 __ Ldr(g.ToRegister(destination), src); 1824 } else { 1825 UseScratchRegisterScope scope(masm()); 1826 Register temp = scope.AcquireX(); 1827 __ Ldr(temp, src); 1828 __ Str(temp, g.ToMemOperand(destination, masm())); 1829 } 1830 } else if (source->IsConstant()) { 1831 Constant src = g.ToConstant(ConstantOperand::cast(source)); 1832 if (destination->IsRegister() || destination->IsStackSlot()) { 1833 UseScratchRegisterScope scope(masm()); 1834 Register dst = destination->IsRegister() ? g.ToRegister(destination) 1835 : scope.AcquireX(); 1836 if (src.type() == Constant::kHeapObject) { 1837 Handle<HeapObject> src_object = src.ToHeapObject(); 1838 Heap::RootListIndex index; 1839 int slot; 1840 if (IsMaterializableFromFrame(src_object, &slot)) { 1841 __ Ldr(dst, g.SlotToMemOperand(slot, masm())); 1842 } else if (IsMaterializableFromRoot(src_object, &index)) { 1843 __ LoadRoot(dst, index); 1844 } else { 1845 __ LoadObject(dst, src_object); 1846 } 1847 } else { 1848 __ Mov(dst, g.ToImmediate(source)); 1849 } 1850 if (destination->IsStackSlot()) { 1851 __ Str(dst, g.ToMemOperand(destination, masm())); 1852 } 1853 } else if (src.type() == Constant::kFloat32) { 1854 if (destination->IsFPRegister()) { 1855 FPRegister dst = g.ToDoubleRegister(destination).S(); 1856 __ Fmov(dst, src.ToFloat32()); 1857 } else { 1858 DCHECK(destination->IsFPStackSlot()); 1859 UseScratchRegisterScope scope(masm()); 1860 FPRegister temp = scope.AcquireS(); 1861 __ Fmov(temp, src.ToFloat32()); 1862 __ Str(temp, g.ToMemOperand(destination, masm())); 1863 } 1864 } else { 1865 DCHECK_EQ(Constant::kFloat64, src.type()); 1866 if (destination->IsFPRegister()) { 1867 FPRegister dst = g.ToDoubleRegister(destination); 1868 __ Fmov(dst, src.ToFloat64()); 1869 } else { 1870 DCHECK(destination->IsFPStackSlot()); 1871 UseScratchRegisterScope scope(masm()); 1872 FPRegister temp = scope.AcquireD(); 1873 __ Fmov(temp, src.ToFloat64()); 1874 __ Str(temp, g.ToMemOperand(destination, masm())); 1875 } 1876 } 1877 } else if (source->IsFPRegister()) { 1878 FPRegister src = g.ToDoubleRegister(source); 1879 if (destination->IsFPRegister()) { 1880 FPRegister dst = g.ToDoubleRegister(destination); 1881 __ Fmov(dst, src); 1882 } else { 1883 DCHECK(destination->IsFPStackSlot()); 1884 __ Str(src, g.ToMemOperand(destination, masm())); 1885 } 1886 } else if (source->IsFPStackSlot()) { 1887 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot()); 1888 MemOperand src = g.ToMemOperand(source, masm()); 1889 if (destination->IsFPRegister()) { 1890 __ Ldr(g.ToDoubleRegister(destination), src); 1891 } else { 1892 UseScratchRegisterScope scope(masm()); 1893 FPRegister temp = scope.AcquireD(); 1894 __ Ldr(temp, src); 1895 __ Str(temp, g.ToMemOperand(destination, masm())); 1896 } 1897 } else { 1898 UNREACHABLE(); 1899 } 1900 } 1901 1902 1903 void CodeGenerator::AssembleSwap(InstructionOperand* source, 1904 InstructionOperand* destination) { 1905 Arm64OperandConverter g(this, nullptr); 1906 // Dispatch on the source and destination operand kinds. Not all 1907 // combinations are possible. 1908 if (source->IsRegister()) { 1909 // Register-register. 1910 UseScratchRegisterScope scope(masm()); 1911 Register temp = scope.AcquireX(); 1912 Register src = g.ToRegister(source); 1913 if (destination->IsRegister()) { 1914 Register dst = g.ToRegister(destination); 1915 __ Mov(temp, src); 1916 __ Mov(src, dst); 1917 __ Mov(dst, temp); 1918 } else { 1919 DCHECK(destination->IsStackSlot()); 1920 MemOperand dst = g.ToMemOperand(destination, masm()); 1921 __ Mov(temp, src); 1922 __ Ldr(src, dst); 1923 __ Str(temp, dst); 1924 } 1925 } else if (source->IsStackSlot() || source->IsFPStackSlot()) { 1926 UseScratchRegisterScope scope(masm()); 1927 DoubleRegister temp_0 = scope.AcquireD(); 1928 DoubleRegister temp_1 = scope.AcquireD(); 1929 MemOperand src = g.ToMemOperand(source, masm()); 1930 MemOperand dst = g.ToMemOperand(destination, masm()); 1931 __ Ldr(temp_0, src); 1932 __ Ldr(temp_1, dst); 1933 __ Str(temp_0, dst); 1934 __ Str(temp_1, src); 1935 } else if (source->IsFPRegister()) { 1936 UseScratchRegisterScope scope(masm()); 1937 FPRegister temp = scope.AcquireD(); 1938 FPRegister src = g.ToDoubleRegister(source); 1939 if (destination->IsFPRegister()) { 1940 FPRegister dst = g.ToDoubleRegister(destination); 1941 __ Fmov(temp, src); 1942 __ Fmov(src, dst); 1943 __ Fmov(dst, temp); 1944 } else { 1945 DCHECK(destination->IsFPStackSlot()); 1946 MemOperand dst = g.ToMemOperand(destination, masm()); 1947 __ Fmov(temp, src); 1948 __ Ldr(src, dst); 1949 __ Str(temp, dst); 1950 } 1951 } else { 1952 // No other combinations are possible. 1953 UNREACHABLE(); 1954 } 1955 } 1956 1957 1958 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) { 1959 // On 64-bit ARM we emit the jump tables inline. 1960 UNREACHABLE(); 1961 } 1962 1963 1964 void CodeGenerator::EnsureSpaceForLazyDeopt() { 1965 if (!info()->ShouldEnsureSpaceForLazyDeopt()) { 1966 return; 1967 } 1968 1969 int space_needed = Deoptimizer::patch_size(); 1970 // Ensure that we have enough space after the previous lazy-bailout 1971 // instruction for patching the code here. 1972 intptr_t current_pc = masm()->pc_offset(); 1973 1974 if (current_pc < (last_lazy_deopt_pc_ + space_needed)) { 1975 intptr_t padding_size = last_lazy_deopt_pc_ + space_needed - current_pc; 1976 DCHECK((padding_size % kInstructionSize) == 0); 1977 InstructionAccurateScope instruction_accurate( 1978 masm(), padding_size / kInstructionSize); 1979 1980 while (padding_size > 0) { 1981 __ nop(); 1982 padding_size -= kInstructionSize; 1983 } 1984 } 1985 } 1986 1987 #undef __ 1988 1989 } // namespace compiler 1990 } // namespace internal 1991 } // namespace v8 1992