1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/code-generator.h" 6 7 #include "src/ast/scopes.h" 8 #include "src/compiler/code-generator-impl.h" 9 #include "src/compiler/gap-resolver.h" 10 #include "src/compiler/node-matchers.h" 11 #include "src/compiler/osr.h" 12 #include "src/x64/assembler-x64.h" 13 #include "src/x64/macro-assembler-x64.h" 14 15 namespace v8 { 16 namespace internal { 17 namespace compiler { 18 19 #define __ masm()-> 20 21 22 #define kScratchDoubleReg xmm0 23 24 25 // Adds X64 specific methods for decoding operands. 26 class X64OperandConverter : public InstructionOperandConverter { 27 public: 28 X64OperandConverter(CodeGenerator* gen, Instruction* instr) 29 : InstructionOperandConverter(gen, instr) {} 30 31 Immediate InputImmediate(size_t index) { 32 return ToImmediate(instr_->InputAt(index)); 33 } 34 35 Operand InputOperand(size_t index, int extra = 0) { 36 return ToOperand(instr_->InputAt(index), extra); 37 } 38 39 Operand OutputOperand() { return ToOperand(instr_->Output()); } 40 41 Immediate ToImmediate(InstructionOperand* operand) { 42 Constant constant = ToConstant(operand); 43 if (constant.type() == Constant::kFloat64) { 44 DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64())); 45 return Immediate(0); 46 } 47 return Immediate(constant.ToInt32()); 48 } 49 50 Operand ToOperand(InstructionOperand* op, int extra = 0) { 51 DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot()); 52 FrameOffset offset = frame_access_state()->GetFrameOffset( 53 AllocatedOperand::cast(op)->index()); 54 return Operand(offset.from_stack_pointer() ? rsp : rbp, 55 offset.offset() + extra); 56 } 57 58 static size_t NextOffset(size_t* offset) { 59 size_t i = *offset; 60 (*offset)++; 61 return i; 62 } 63 64 static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) { 65 STATIC_ASSERT(0 == static_cast<int>(times_1)); 66 STATIC_ASSERT(1 == static_cast<int>(times_2)); 67 STATIC_ASSERT(2 == static_cast<int>(times_4)); 68 STATIC_ASSERT(3 == static_cast<int>(times_8)); 69 int scale = static_cast<int>(mode - one); 70 DCHECK(scale >= 0 && scale < 4); 71 return static_cast<ScaleFactor>(scale); 72 } 73 74 Operand MemoryOperand(size_t* offset) { 75 AddressingMode mode = AddressingModeField::decode(instr_->opcode()); 76 switch (mode) { 77 case kMode_MR: { 78 Register base = InputRegister(NextOffset(offset)); 79 int32_t disp = 0; 80 return Operand(base, disp); 81 } 82 case kMode_MRI: { 83 Register base = InputRegister(NextOffset(offset)); 84 int32_t disp = InputInt32(NextOffset(offset)); 85 return Operand(base, disp); 86 } 87 case kMode_MR1: 88 case kMode_MR2: 89 case kMode_MR4: 90 case kMode_MR8: { 91 Register base = InputRegister(NextOffset(offset)); 92 Register index = InputRegister(NextOffset(offset)); 93 ScaleFactor scale = ScaleFor(kMode_MR1, mode); 94 int32_t disp = 0; 95 return Operand(base, index, scale, disp); 96 } 97 case kMode_MR1I: 98 case kMode_MR2I: 99 case kMode_MR4I: 100 case kMode_MR8I: { 101 Register base = InputRegister(NextOffset(offset)); 102 Register index = InputRegister(NextOffset(offset)); 103 ScaleFactor scale = ScaleFor(kMode_MR1I, mode); 104 int32_t disp = InputInt32(NextOffset(offset)); 105 return Operand(base, index, scale, disp); 106 } 107 case kMode_M1: { 108 Register base = InputRegister(NextOffset(offset)); 109 int32_t disp = 0; 110 return Operand(base, disp); 111 } 112 case kMode_M2: 113 UNREACHABLE(); // Should use kModeMR with more compact encoding instead 114 return Operand(no_reg, 0); 115 case kMode_M4: 116 case kMode_M8: { 117 Register index = InputRegister(NextOffset(offset)); 118 ScaleFactor scale = ScaleFor(kMode_M1, mode); 119 int32_t disp = 0; 120 return Operand(index, scale, disp); 121 } 122 case kMode_M1I: 123 case kMode_M2I: 124 case kMode_M4I: 125 case kMode_M8I: { 126 Register index = InputRegister(NextOffset(offset)); 127 ScaleFactor scale = ScaleFor(kMode_M1I, mode); 128 int32_t disp = InputInt32(NextOffset(offset)); 129 return Operand(index, scale, disp); 130 } 131 case kMode_None: 132 UNREACHABLE(); 133 return Operand(no_reg, 0); 134 } 135 UNREACHABLE(); 136 return Operand(no_reg, 0); 137 } 138 139 Operand MemoryOperand(size_t first_input = 0) { 140 return MemoryOperand(&first_input); 141 } 142 }; 143 144 145 namespace { 146 147 bool HasImmediateInput(Instruction* instr, size_t index) { 148 return instr->InputAt(index)->IsImmediate(); 149 } 150 151 152 class OutOfLineLoadZero final : public OutOfLineCode { 153 public: 154 OutOfLineLoadZero(CodeGenerator* gen, Register result) 155 : OutOfLineCode(gen), result_(result) {} 156 157 void Generate() final { __ xorl(result_, result_); } 158 159 private: 160 Register const result_; 161 }; 162 163 164 class OutOfLineLoadNaN final : public OutOfLineCode { 165 public: 166 OutOfLineLoadNaN(CodeGenerator* gen, XMMRegister result) 167 : OutOfLineCode(gen), result_(result) {} 168 169 void Generate() final { __ Pcmpeqd(result_, result_); } 170 171 private: 172 XMMRegister const result_; 173 }; 174 175 176 class OutOfLineTruncateDoubleToI final : public OutOfLineCode { 177 public: 178 OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result, 179 XMMRegister input) 180 : OutOfLineCode(gen), result_(result), input_(input) {} 181 182 void Generate() final { 183 __ subp(rsp, Immediate(kDoubleSize)); 184 __ Movsd(MemOperand(rsp, 0), input_); 185 __ SlowTruncateToI(result_, rsp, 0); 186 __ addp(rsp, Immediate(kDoubleSize)); 187 } 188 189 private: 190 Register const result_; 191 XMMRegister const input_; 192 }; 193 194 195 class OutOfLineRecordWrite final : public OutOfLineCode { 196 public: 197 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand, 198 Register value, Register scratch0, Register scratch1, 199 RecordWriteMode mode) 200 : OutOfLineCode(gen), 201 object_(object), 202 operand_(operand), 203 value_(value), 204 scratch0_(scratch0), 205 scratch1_(scratch1), 206 mode_(mode) {} 207 208 void Generate() final { 209 if (mode_ > RecordWriteMode::kValueIsPointer) { 210 __ JumpIfSmi(value_, exit()); 211 } 212 if (mode_ > RecordWriteMode::kValueIsMap) { 213 __ CheckPageFlag(value_, scratch0_, 214 MemoryChunk::kPointersToHereAreInterestingMask, zero, 215 exit()); 216 } 217 SaveFPRegsMode const save_fp_mode = 218 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; 219 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_, 220 EMIT_REMEMBERED_SET, save_fp_mode); 221 __ leap(scratch1_, operand_); 222 __ CallStub(&stub); 223 } 224 225 private: 226 Register const object_; 227 Operand const operand_; 228 Register const value_; 229 Register const scratch0_; 230 Register const scratch1_; 231 RecordWriteMode const mode_; 232 }; 233 234 } // namespace 235 236 237 #define ASSEMBLE_UNOP(asm_instr) \ 238 do { \ 239 if (instr->Output()->IsRegister()) { \ 240 __ asm_instr(i.OutputRegister()); \ 241 } else { \ 242 __ asm_instr(i.OutputOperand()); \ 243 } \ 244 } while (0) 245 246 247 #define ASSEMBLE_BINOP(asm_instr) \ 248 do { \ 249 if (HasImmediateInput(instr, 1)) { \ 250 if (instr->InputAt(0)->IsRegister()) { \ 251 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ 252 } else { \ 253 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ 254 } \ 255 } else { \ 256 if (instr->InputAt(1)->IsRegister()) { \ 257 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ 258 } else { \ 259 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ 260 } \ 261 } \ 262 } while (0) 263 264 265 #define ASSEMBLE_MULT(asm_instr) \ 266 do { \ 267 if (HasImmediateInput(instr, 1)) { \ 268 if (instr->InputAt(0)->IsRegister()) { \ 269 __ asm_instr(i.OutputRegister(), i.InputRegister(0), \ 270 i.InputImmediate(1)); \ 271 } else { \ 272 __ asm_instr(i.OutputRegister(), i.InputOperand(0), \ 273 i.InputImmediate(1)); \ 274 } \ 275 } else { \ 276 if (instr->InputAt(1)->IsRegister()) { \ 277 __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \ 278 } else { \ 279 __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \ 280 } \ 281 } \ 282 } while (0) 283 284 285 #define ASSEMBLE_SHIFT(asm_instr, width) \ 286 do { \ 287 if (HasImmediateInput(instr, 1)) { \ 288 if (instr->Output()->IsRegister()) { \ 289 __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \ 290 } else { \ 291 __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \ 292 } \ 293 } else { \ 294 if (instr->Output()->IsRegister()) { \ 295 __ asm_instr##_cl(i.OutputRegister()); \ 296 } else { \ 297 __ asm_instr##_cl(i.OutputOperand()); \ 298 } \ 299 } \ 300 } while (0) 301 302 303 #define ASSEMBLE_MOVX(asm_instr) \ 304 do { \ 305 if (instr->addressing_mode() != kMode_None) { \ 306 __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \ 307 } else if (instr->InputAt(0)->IsRegister()) { \ 308 __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \ 309 } else { \ 310 __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \ 311 } \ 312 } while (0) 313 314 315 #define ASSEMBLE_SSE_BINOP(asm_instr) \ 316 do { \ 317 if (instr->InputAt(1)->IsDoubleRegister()) { \ 318 __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \ 319 } else { \ 320 __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \ 321 } \ 322 } while (0) 323 324 325 #define ASSEMBLE_SSE_UNOP(asm_instr) \ 326 do { \ 327 if (instr->InputAt(0)->IsDoubleRegister()) { \ 328 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \ 329 } else { \ 330 __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \ 331 } \ 332 } while (0) 333 334 335 #define ASSEMBLE_AVX_BINOP(asm_instr) \ 336 do { \ 337 CpuFeatureScope avx_scope(masm(), AVX); \ 338 if (instr->InputAt(1)->IsDoubleRegister()) { \ 339 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ 340 i.InputDoubleRegister(1)); \ 341 } else { \ 342 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ 343 i.InputOperand(1)); \ 344 } \ 345 } while (0) 346 347 348 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr) \ 349 do { \ 350 auto result = i.OutputDoubleRegister(); \ 351 auto buffer = i.InputRegister(0); \ 352 auto index1 = i.InputRegister(1); \ 353 auto index2 = i.InputInt32(2); \ 354 OutOfLineCode* ool; \ 355 if (instr->InputAt(3)->IsRegister()) { \ 356 auto length = i.InputRegister(3); \ 357 DCHECK_EQ(0, index2); \ 358 __ cmpl(index1, length); \ 359 ool = new (zone()) OutOfLineLoadNaN(this, result); \ 360 } else { \ 361 auto length = i.InputInt32(3); \ 362 DCHECK_LE(index2, length); \ 363 __ cmpq(index1, Immediate(length - index2)); \ 364 class OutOfLineLoadFloat final : public OutOfLineCode { \ 365 public: \ 366 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \ 367 Register buffer, Register index1, int32_t index2, \ 368 int32_t length) \ 369 : OutOfLineCode(gen), \ 370 result_(result), \ 371 buffer_(buffer), \ 372 index1_(index1), \ 373 index2_(index2), \ 374 length_(length) {} \ 375 \ 376 void Generate() final { \ 377 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 378 __ Pcmpeqd(result_, result_); \ 379 __ cmpl(kScratchRegister, Immediate(length_)); \ 380 __ j(above_equal, exit()); \ 381 __ asm_instr(result_, \ 382 Operand(buffer_, kScratchRegister, times_1, 0)); \ 383 } \ 384 \ 385 private: \ 386 XMMRegister const result_; \ 387 Register const buffer_; \ 388 Register const index1_; \ 389 int32_t const index2_; \ 390 int32_t const length_; \ 391 }; \ 392 ool = new (zone()) \ 393 OutOfLineLoadFloat(this, result, buffer, index1, index2, length); \ 394 } \ 395 __ j(above_equal, ool->entry()); \ 396 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \ 397 __ bind(ool->exit()); \ 398 } while (false) 399 400 401 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \ 402 do { \ 403 auto result = i.OutputRegister(); \ 404 auto buffer = i.InputRegister(0); \ 405 auto index1 = i.InputRegister(1); \ 406 auto index2 = i.InputInt32(2); \ 407 OutOfLineCode* ool; \ 408 if (instr->InputAt(3)->IsRegister()) { \ 409 auto length = i.InputRegister(3); \ 410 DCHECK_EQ(0, index2); \ 411 __ cmpl(index1, length); \ 412 ool = new (zone()) OutOfLineLoadZero(this, result); \ 413 } else { \ 414 auto length = i.InputInt32(3); \ 415 DCHECK_LE(index2, length); \ 416 __ cmpq(index1, Immediate(length - index2)); \ 417 class OutOfLineLoadInteger final : public OutOfLineCode { \ 418 public: \ 419 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \ 420 Register buffer, Register index1, int32_t index2, \ 421 int32_t length) \ 422 : OutOfLineCode(gen), \ 423 result_(result), \ 424 buffer_(buffer), \ 425 index1_(index1), \ 426 index2_(index2), \ 427 length_(length) {} \ 428 \ 429 void Generate() final { \ 430 Label oob; \ 431 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 432 __ cmpl(kScratchRegister, Immediate(length_)); \ 433 __ j(above_equal, &oob, Label::kNear); \ 434 __ asm_instr(result_, \ 435 Operand(buffer_, kScratchRegister, times_1, 0)); \ 436 __ jmp(exit()); \ 437 __ bind(&oob); \ 438 __ xorl(result_, result_); \ 439 } \ 440 \ 441 private: \ 442 Register const result_; \ 443 Register const buffer_; \ 444 Register const index1_; \ 445 int32_t const index2_; \ 446 int32_t const length_; \ 447 }; \ 448 ool = new (zone()) \ 449 OutOfLineLoadInteger(this, result, buffer, index1, index2, length); \ 450 } \ 451 __ j(above_equal, ool->entry()); \ 452 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \ 453 __ bind(ool->exit()); \ 454 } while (false) 455 456 457 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \ 458 do { \ 459 auto buffer = i.InputRegister(0); \ 460 auto index1 = i.InputRegister(1); \ 461 auto index2 = i.InputInt32(2); \ 462 auto value = i.InputDoubleRegister(4); \ 463 if (instr->InputAt(3)->IsRegister()) { \ 464 auto length = i.InputRegister(3); \ 465 DCHECK_EQ(0, index2); \ 466 Label done; \ 467 __ cmpl(index1, length); \ 468 __ j(above_equal, &done, Label::kNear); \ 469 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 470 __ bind(&done); \ 471 } else { \ 472 auto length = i.InputInt32(3); \ 473 DCHECK_LE(index2, length); \ 474 __ cmpq(index1, Immediate(length - index2)); \ 475 class OutOfLineStoreFloat final : public OutOfLineCode { \ 476 public: \ 477 OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \ 478 Register index1, int32_t index2, int32_t length, \ 479 XMMRegister value) \ 480 : OutOfLineCode(gen), \ 481 buffer_(buffer), \ 482 index1_(index1), \ 483 index2_(index2), \ 484 length_(length), \ 485 value_(value) {} \ 486 \ 487 void Generate() final { \ 488 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 489 __ cmpl(kScratchRegister, Immediate(length_)); \ 490 __ j(above_equal, exit()); \ 491 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \ 492 value_); \ 493 } \ 494 \ 495 private: \ 496 Register const buffer_; \ 497 Register const index1_; \ 498 int32_t const index2_; \ 499 int32_t const length_; \ 500 XMMRegister const value_; \ 501 }; \ 502 auto ool = new (zone()) \ 503 OutOfLineStoreFloat(this, buffer, index1, index2, length, value); \ 504 __ j(above_equal, ool->entry()); \ 505 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 506 __ bind(ool->exit()); \ 507 } \ 508 } while (false) 509 510 511 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \ 512 do { \ 513 auto buffer = i.InputRegister(0); \ 514 auto index1 = i.InputRegister(1); \ 515 auto index2 = i.InputInt32(2); \ 516 if (instr->InputAt(3)->IsRegister()) { \ 517 auto length = i.InputRegister(3); \ 518 DCHECK_EQ(0, index2); \ 519 Label done; \ 520 __ cmpl(index1, length); \ 521 __ j(above_equal, &done, Label::kNear); \ 522 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 523 __ bind(&done); \ 524 } else { \ 525 auto length = i.InputInt32(3); \ 526 DCHECK_LE(index2, length); \ 527 __ cmpq(index1, Immediate(length - index2)); \ 528 class OutOfLineStoreInteger final : public OutOfLineCode { \ 529 public: \ 530 OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \ 531 Register index1, int32_t index2, int32_t length, \ 532 Value value) \ 533 : OutOfLineCode(gen), \ 534 buffer_(buffer), \ 535 index1_(index1), \ 536 index2_(index2), \ 537 length_(length), \ 538 value_(value) {} \ 539 \ 540 void Generate() final { \ 541 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 542 __ cmpl(kScratchRegister, Immediate(length_)); \ 543 __ j(above_equal, exit()); \ 544 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \ 545 value_); \ 546 } \ 547 \ 548 private: \ 549 Register const buffer_; \ 550 Register const index1_; \ 551 int32_t const index2_; \ 552 int32_t const length_; \ 553 Value const value_; \ 554 }; \ 555 auto ool = new (zone()) \ 556 OutOfLineStoreInteger(this, buffer, index1, index2, length, value); \ 557 __ j(above_equal, ool->entry()); \ 558 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 559 __ bind(ool->exit()); \ 560 } \ 561 } while (false) 562 563 564 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \ 565 do { \ 566 if (instr->InputAt(4)->IsRegister()) { \ 567 Register value = i.InputRegister(4); \ 568 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \ 569 } else { \ 570 Immediate value = i.InputImmediate(4); \ 571 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \ 572 } \ 573 } while (false) 574 575 576 void CodeGenerator::AssembleDeconstructActivationRecord(int stack_param_delta) { 577 int sp_slot_delta = TailCallFrameStackSlotDelta(stack_param_delta); 578 if (sp_slot_delta > 0) { 579 __ addq(rsp, Immediate(sp_slot_delta * kPointerSize)); 580 } 581 frame_access_state()->SetFrameAccessToDefault(); 582 } 583 584 585 void CodeGenerator::AssemblePrepareTailCall(int stack_param_delta) { 586 int sp_slot_delta = TailCallFrameStackSlotDelta(stack_param_delta); 587 if (sp_slot_delta < 0) { 588 __ subq(rsp, Immediate(-sp_slot_delta * kPointerSize)); 589 frame_access_state()->IncreaseSPDelta(-sp_slot_delta); 590 } 591 if (frame()->needs_frame()) { 592 __ movq(rbp, MemOperand(rbp, 0)); 593 } 594 frame_access_state()->SetFrameAccessToSP(); 595 } 596 597 598 // Assembles an instruction after register allocation, producing machine code. 599 void CodeGenerator::AssembleArchInstruction(Instruction* instr) { 600 X64OperandConverter i(this, instr); 601 602 switch (ArchOpcodeField::decode(instr->opcode())) { 603 case kArchCallCodeObject: { 604 EnsureSpaceForLazyDeopt(); 605 if (HasImmediateInput(instr, 0)) { 606 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0)); 607 __ Call(code, RelocInfo::CODE_TARGET); 608 } else { 609 Register reg = i.InputRegister(0); 610 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); 611 __ call(reg); 612 } 613 RecordCallPosition(instr); 614 frame_access_state()->ClearSPDelta(); 615 break; 616 } 617 case kArchTailCallCodeObject: { 618 int stack_param_delta = i.InputInt32(instr->InputCount() - 1); 619 AssembleDeconstructActivationRecord(stack_param_delta); 620 if (HasImmediateInput(instr, 0)) { 621 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0)); 622 __ jmp(code, RelocInfo::CODE_TARGET); 623 } else { 624 Register reg = i.InputRegister(0); 625 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); 626 __ jmp(reg); 627 } 628 frame_access_state()->ClearSPDelta(); 629 break; 630 } 631 case kArchCallJSFunction: { 632 EnsureSpaceForLazyDeopt(); 633 Register func = i.InputRegister(0); 634 if (FLAG_debug_code) { 635 // Check the function's context matches the context argument. 636 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset)); 637 __ Assert(equal, kWrongFunctionContext); 638 } 639 __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset)); 640 frame_access_state()->ClearSPDelta(); 641 RecordCallPosition(instr); 642 break; 643 } 644 case kArchTailCallJSFunction: { 645 Register func = i.InputRegister(0); 646 if (FLAG_debug_code) { 647 // Check the function's context matches the context argument. 648 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset)); 649 __ Assert(equal, kWrongFunctionContext); 650 } 651 int stack_param_delta = i.InputInt32(instr->InputCount() - 1); 652 AssembleDeconstructActivationRecord(stack_param_delta); 653 __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset)); 654 frame_access_state()->ClearSPDelta(); 655 break; 656 } 657 case kArchLazyBailout: { 658 EnsureSpaceForLazyDeopt(); 659 RecordCallPosition(instr); 660 break; 661 } 662 case kArchPrepareCallCFunction: { 663 // Frame alignment requires using FP-relative frame addressing. 664 frame_access_state()->SetFrameAccessToFP(); 665 int const num_parameters = MiscField::decode(instr->opcode()); 666 __ PrepareCallCFunction(num_parameters); 667 break; 668 } 669 case kArchPrepareTailCall: 670 AssemblePrepareTailCall(i.InputInt32(instr->InputCount() - 1)); 671 break; 672 case kArchCallCFunction: { 673 int const num_parameters = MiscField::decode(instr->opcode()); 674 if (HasImmediateInput(instr, 0)) { 675 ExternalReference ref = i.InputExternalReference(0); 676 __ CallCFunction(ref, num_parameters); 677 } else { 678 Register func = i.InputRegister(0); 679 __ CallCFunction(func, num_parameters); 680 } 681 frame_access_state()->SetFrameAccessToDefault(); 682 frame_access_state()->ClearSPDelta(); 683 break; 684 } 685 case kArchJmp: 686 AssembleArchJump(i.InputRpo(0)); 687 break; 688 case kArchLookupSwitch: 689 AssembleArchLookupSwitch(instr); 690 break; 691 case kArchTableSwitch: 692 AssembleArchTableSwitch(instr); 693 break; 694 case kArchNop: 695 case kArchThrowTerminator: 696 // don't emit code for nops. 697 break; 698 case kArchDeoptimize: { 699 int deopt_state_id = 700 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); 701 Deoptimizer::BailoutType bailout_type = 702 Deoptimizer::BailoutType(MiscField::decode(instr->opcode())); 703 AssembleDeoptimizerCall(deopt_state_id, bailout_type); 704 break; 705 } 706 case kArchRet: 707 AssembleReturn(); 708 break; 709 case kArchStackPointer: 710 __ movq(i.OutputRegister(), rsp); 711 break; 712 case kArchFramePointer: 713 __ movq(i.OutputRegister(), rbp); 714 break; 715 case kArchTruncateDoubleToI: { 716 auto result = i.OutputRegister(); 717 auto input = i.InputDoubleRegister(0); 718 auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input); 719 __ Cvttsd2siq(result, input); 720 __ cmpq(result, Immediate(1)); 721 __ j(overflow, ool->entry()); 722 __ bind(ool->exit()); 723 break; 724 } 725 case kArchStoreWithWriteBarrier: { 726 RecordWriteMode mode = 727 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode())); 728 Register object = i.InputRegister(0); 729 size_t index = 0; 730 Operand operand = i.MemoryOperand(&index); 731 Register value = i.InputRegister(index); 732 Register scratch0 = i.TempRegister(0); 733 Register scratch1 = i.TempRegister(1); 734 auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value, 735 scratch0, scratch1, mode); 736 __ movp(operand, value); 737 __ CheckPageFlag(object, scratch0, 738 MemoryChunk::kPointersFromHereAreInterestingMask, 739 not_zero, ool->entry()); 740 __ bind(ool->exit()); 741 break; 742 } 743 case kX64Add32: 744 ASSEMBLE_BINOP(addl); 745 break; 746 case kX64Add: 747 ASSEMBLE_BINOP(addq); 748 break; 749 case kX64Sub32: 750 ASSEMBLE_BINOP(subl); 751 break; 752 case kX64Sub: 753 ASSEMBLE_BINOP(subq); 754 break; 755 case kX64And32: 756 ASSEMBLE_BINOP(andl); 757 break; 758 case kX64And: 759 ASSEMBLE_BINOP(andq); 760 break; 761 case kX64Cmp32: 762 ASSEMBLE_BINOP(cmpl); 763 break; 764 case kX64Cmp: 765 ASSEMBLE_BINOP(cmpq); 766 break; 767 case kX64Test32: 768 ASSEMBLE_BINOP(testl); 769 break; 770 case kX64Test: 771 ASSEMBLE_BINOP(testq); 772 break; 773 case kX64Imul32: 774 ASSEMBLE_MULT(imull); 775 break; 776 case kX64Imul: 777 ASSEMBLE_MULT(imulq); 778 break; 779 case kX64ImulHigh32: 780 if (instr->InputAt(1)->IsRegister()) { 781 __ imull(i.InputRegister(1)); 782 } else { 783 __ imull(i.InputOperand(1)); 784 } 785 break; 786 case kX64UmulHigh32: 787 if (instr->InputAt(1)->IsRegister()) { 788 __ mull(i.InputRegister(1)); 789 } else { 790 __ mull(i.InputOperand(1)); 791 } 792 break; 793 case kX64Idiv32: 794 __ cdq(); 795 __ idivl(i.InputRegister(1)); 796 break; 797 case kX64Idiv: 798 __ cqo(); 799 __ idivq(i.InputRegister(1)); 800 break; 801 case kX64Udiv32: 802 __ xorl(rdx, rdx); 803 __ divl(i.InputRegister(1)); 804 break; 805 case kX64Udiv: 806 __ xorq(rdx, rdx); 807 __ divq(i.InputRegister(1)); 808 break; 809 case kX64Not: 810 ASSEMBLE_UNOP(notq); 811 break; 812 case kX64Not32: 813 ASSEMBLE_UNOP(notl); 814 break; 815 case kX64Neg: 816 ASSEMBLE_UNOP(negq); 817 break; 818 case kX64Neg32: 819 ASSEMBLE_UNOP(negl); 820 break; 821 case kX64Or32: 822 ASSEMBLE_BINOP(orl); 823 break; 824 case kX64Or: 825 ASSEMBLE_BINOP(orq); 826 break; 827 case kX64Xor32: 828 ASSEMBLE_BINOP(xorl); 829 break; 830 case kX64Xor: 831 ASSEMBLE_BINOP(xorq); 832 break; 833 case kX64Shl32: 834 ASSEMBLE_SHIFT(shll, 5); 835 break; 836 case kX64Shl: 837 ASSEMBLE_SHIFT(shlq, 6); 838 break; 839 case kX64Shr32: 840 ASSEMBLE_SHIFT(shrl, 5); 841 break; 842 case kX64Shr: 843 ASSEMBLE_SHIFT(shrq, 6); 844 break; 845 case kX64Sar32: 846 ASSEMBLE_SHIFT(sarl, 5); 847 break; 848 case kX64Sar: 849 ASSEMBLE_SHIFT(sarq, 6); 850 break; 851 case kX64Ror32: 852 ASSEMBLE_SHIFT(rorl, 5); 853 break; 854 case kX64Ror: 855 ASSEMBLE_SHIFT(rorq, 6); 856 break; 857 case kX64Lzcnt: 858 if (instr->InputAt(0)->IsRegister()) { 859 __ Lzcntq(i.OutputRegister(), i.InputRegister(0)); 860 } else { 861 __ Lzcntq(i.OutputRegister(), i.InputOperand(0)); 862 } 863 break; 864 case kX64Lzcnt32: 865 if (instr->InputAt(0)->IsRegister()) { 866 __ Lzcntl(i.OutputRegister(), i.InputRegister(0)); 867 } else { 868 __ Lzcntl(i.OutputRegister(), i.InputOperand(0)); 869 } 870 break; 871 case kX64Tzcnt: 872 if (instr->InputAt(0)->IsRegister()) { 873 __ Tzcntq(i.OutputRegister(), i.InputRegister(0)); 874 } else { 875 __ Tzcntq(i.OutputRegister(), i.InputOperand(0)); 876 } 877 break; 878 case kX64Tzcnt32: 879 if (instr->InputAt(0)->IsRegister()) { 880 __ Tzcntl(i.OutputRegister(), i.InputRegister(0)); 881 } else { 882 __ Tzcntl(i.OutputRegister(), i.InputOperand(0)); 883 } 884 break; 885 case kX64Popcnt: 886 if (instr->InputAt(0)->IsRegister()) { 887 __ Popcntq(i.OutputRegister(), i.InputRegister(0)); 888 } else { 889 __ Popcntq(i.OutputRegister(), i.InputOperand(0)); 890 } 891 break; 892 case kX64Popcnt32: 893 if (instr->InputAt(0)->IsRegister()) { 894 __ Popcntl(i.OutputRegister(), i.InputRegister(0)); 895 } else { 896 __ Popcntl(i.OutputRegister(), i.InputOperand(0)); 897 } 898 break; 899 case kSSEFloat32Cmp: 900 ASSEMBLE_SSE_BINOP(Ucomiss); 901 break; 902 case kSSEFloat32Add: 903 ASSEMBLE_SSE_BINOP(addss); 904 break; 905 case kSSEFloat32Sub: 906 ASSEMBLE_SSE_BINOP(subss); 907 break; 908 case kSSEFloat32Mul: 909 ASSEMBLE_SSE_BINOP(mulss); 910 break; 911 case kSSEFloat32Div: 912 ASSEMBLE_SSE_BINOP(divss); 913 // Don't delete this mov. It may improve performance on some CPUs, 914 // when there is a (v)mulss depending on the result. 915 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 916 break; 917 case kSSEFloat32Abs: { 918 // TODO(bmeurer): Use RIP relative 128-bit constants. 919 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 920 __ psrlq(kScratchDoubleReg, 33); 921 __ andps(i.OutputDoubleRegister(), kScratchDoubleReg); 922 break; 923 } 924 case kSSEFloat32Neg: { 925 // TODO(bmeurer): Use RIP relative 128-bit constants. 926 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 927 __ psllq(kScratchDoubleReg, 31); 928 __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg); 929 break; 930 } 931 case kSSEFloat32Sqrt: 932 ASSEMBLE_SSE_UNOP(sqrtss); 933 break; 934 case kSSEFloat32Max: 935 ASSEMBLE_SSE_BINOP(maxss); 936 break; 937 case kSSEFloat32Min: 938 ASSEMBLE_SSE_BINOP(minss); 939 break; 940 case kSSEFloat32ToFloat64: 941 ASSEMBLE_SSE_UNOP(Cvtss2sd); 942 break; 943 case kSSEFloat32Round: { 944 CpuFeatureScope sse_scope(masm(), SSE4_1); 945 RoundingMode const mode = 946 static_cast<RoundingMode>(MiscField::decode(instr->opcode())); 947 __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); 948 break; 949 } 950 case kSSEFloat64Cmp: 951 ASSEMBLE_SSE_BINOP(Ucomisd); 952 break; 953 case kSSEFloat64Add: 954 ASSEMBLE_SSE_BINOP(addsd); 955 break; 956 case kSSEFloat64Sub: 957 ASSEMBLE_SSE_BINOP(subsd); 958 break; 959 case kSSEFloat64Mul: 960 ASSEMBLE_SSE_BINOP(mulsd); 961 break; 962 case kSSEFloat64Div: 963 ASSEMBLE_SSE_BINOP(divsd); 964 // Don't delete this mov. It may improve performance on some CPUs, 965 // when there is a (v)mulsd depending on the result. 966 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 967 break; 968 case kSSEFloat64Mod: { 969 __ subq(rsp, Immediate(kDoubleSize)); 970 // Move values to st(0) and st(1). 971 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1)); 972 __ fld_d(Operand(rsp, 0)); 973 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); 974 __ fld_d(Operand(rsp, 0)); 975 // Loop while fprem isn't done. 976 Label mod_loop; 977 __ bind(&mod_loop); 978 // This instructions traps on all kinds inputs, but we are assuming the 979 // floating point control word is set to ignore them all. 980 __ fprem(); 981 // The following 2 instruction implicitly use rax. 982 __ fnstsw_ax(); 983 if (CpuFeatures::IsSupported(SAHF)) { 984 CpuFeatureScope sahf_scope(masm(), SAHF); 985 __ sahf(); 986 } else { 987 __ shrl(rax, Immediate(8)); 988 __ andl(rax, Immediate(0xFF)); 989 __ pushq(rax); 990 __ popfq(); 991 } 992 __ j(parity_even, &mod_loop); 993 // Move output to stack and clean up. 994 __ fstp(1); 995 __ fstp_d(Operand(rsp, 0)); 996 __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0)); 997 __ addq(rsp, Immediate(kDoubleSize)); 998 break; 999 } 1000 case kSSEFloat64Max: 1001 ASSEMBLE_SSE_BINOP(maxsd); 1002 break; 1003 case kSSEFloat64Min: 1004 ASSEMBLE_SSE_BINOP(minsd); 1005 break; 1006 case kSSEFloat64Abs: { 1007 // TODO(bmeurer): Use RIP relative 128-bit constants. 1008 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1009 __ psrlq(kScratchDoubleReg, 1); 1010 __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg); 1011 break; 1012 } 1013 case kSSEFloat64Neg: { 1014 // TODO(bmeurer): Use RIP relative 128-bit constants. 1015 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1016 __ psllq(kScratchDoubleReg, 63); 1017 __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg); 1018 break; 1019 } 1020 case kSSEFloat64Sqrt: 1021 ASSEMBLE_SSE_UNOP(sqrtsd); 1022 break; 1023 case kSSEFloat64Round: { 1024 CpuFeatureScope sse_scope(masm(), SSE4_1); 1025 RoundingMode const mode = 1026 static_cast<RoundingMode>(MiscField::decode(instr->opcode())); 1027 __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); 1028 break; 1029 } 1030 case kSSEFloat64ToFloat32: 1031 ASSEMBLE_SSE_UNOP(Cvtsd2ss); 1032 break; 1033 case kSSEFloat64ToInt32: 1034 if (instr->InputAt(0)->IsDoubleRegister()) { 1035 __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0)); 1036 } else { 1037 __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0)); 1038 } 1039 break; 1040 case kSSEFloat64ToUint32: { 1041 if (instr->InputAt(0)->IsDoubleRegister()) { 1042 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1043 } else { 1044 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); 1045 } 1046 __ AssertZeroExtended(i.OutputRegister()); 1047 break; 1048 } 1049 case kSSEFloat32ToInt64: 1050 if (instr->InputAt(0)->IsDoubleRegister()) { 1051 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1052 } else { 1053 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); 1054 } 1055 if (instr->OutputCount() > 1) { 1056 __ Set(i.OutputRegister(1), 1); 1057 Label done; 1058 Label fail; 1059 __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN)); 1060 if (instr->InputAt(0)->IsDoubleRegister()) { 1061 __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0)); 1062 } else { 1063 __ Ucomiss(kScratchDoubleReg, i.InputOperand(0)); 1064 } 1065 // If the input is NaN, then the conversion fails. 1066 __ j(parity_even, &fail); 1067 // If the input is INT64_MIN, then the conversion succeeds. 1068 __ j(equal, &done); 1069 __ cmpq(i.OutputRegister(0), Immediate(1)); 1070 // If the conversion results in INT64_MIN, but the input was not 1071 // INT64_MIN, then the conversion fails. 1072 __ j(no_overflow, &done); 1073 __ bind(&fail); 1074 __ Set(i.OutputRegister(1), 0); 1075 __ bind(&done); 1076 } 1077 break; 1078 case kSSEFloat64ToInt64: 1079 if (instr->InputAt(0)->IsDoubleRegister()) { 1080 __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0)); 1081 } else { 1082 __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0)); 1083 } 1084 if (instr->OutputCount() > 1) { 1085 __ Set(i.OutputRegister(1), 1); 1086 Label done; 1087 Label fail; 1088 __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN)); 1089 if (instr->InputAt(0)->IsDoubleRegister()) { 1090 __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0)); 1091 } else { 1092 __ Ucomisd(kScratchDoubleReg, i.InputOperand(0)); 1093 } 1094 // If the input is NaN, then the conversion fails. 1095 __ j(parity_even, &fail); 1096 // If the input is INT64_MIN, then the conversion succeeds. 1097 __ j(equal, &done); 1098 __ cmpq(i.OutputRegister(0), Immediate(1)); 1099 // If the conversion results in INT64_MIN, but the input was not 1100 // INT64_MIN, then the conversion fails. 1101 __ j(no_overflow, &done); 1102 __ bind(&fail); 1103 __ Set(i.OutputRegister(1), 0); 1104 __ bind(&done); 1105 } 1106 break; 1107 case kSSEFloat32ToUint64: { 1108 Label done; 1109 Label success; 1110 if (instr->OutputCount() > 1) { 1111 __ Set(i.OutputRegister(1), 0); 1112 } 1113 // There does not exist a Float32ToUint64 instruction, so we have to use 1114 // the Float32ToInt64 instruction. 1115 if (instr->InputAt(0)->IsDoubleRegister()) { 1116 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1117 } else { 1118 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); 1119 } 1120 // Check if the result of the Float32ToInt64 conversion is positive, we 1121 // are already done. 1122 __ testq(i.OutputRegister(), i.OutputRegister()); 1123 __ j(positive, &success); 1124 // The result of the first conversion was negative, which means that the 1125 // input value was not within the positive int64 range. We subtract 2^64 1126 // and convert it again to see if it is within the uint64 range. 1127 __ Move(kScratchDoubleReg, -9223372036854775808.0f); 1128 if (instr->InputAt(0)->IsDoubleRegister()) { 1129 __ addss(kScratchDoubleReg, i.InputDoubleRegister(0)); 1130 } else { 1131 __ addss(kScratchDoubleReg, i.InputOperand(0)); 1132 } 1133 __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg); 1134 __ testq(i.OutputRegister(), i.OutputRegister()); 1135 // The only possible negative value here is 0x80000000000000000, which is 1136 // used on x64 to indicate an integer overflow. 1137 __ j(negative, &done); 1138 // The input value is within uint64 range and the second conversion worked 1139 // successfully, but we still have to undo the subtraction we did 1140 // earlier. 1141 __ Set(kScratchRegister, 0x8000000000000000); 1142 __ orq(i.OutputRegister(), kScratchRegister); 1143 __ bind(&success); 1144 if (instr->OutputCount() > 1) { 1145 __ Set(i.OutputRegister(1), 1); 1146 } 1147 __ bind(&done); 1148 break; 1149 } 1150 case kSSEFloat64ToUint64: { 1151 Label done; 1152 Label success; 1153 if (instr->OutputCount() > 1) { 1154 __ Set(i.OutputRegister(1), 0); 1155 } 1156 // There does not exist a Float64ToUint64 instruction, so we have to use 1157 // the Float64ToInt64 instruction. 1158 if (instr->InputAt(0)->IsDoubleRegister()) { 1159 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1160 } else { 1161 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); 1162 } 1163 // Check if the result of the Float64ToInt64 conversion is positive, we 1164 // are already done. 1165 __ testq(i.OutputRegister(), i.OutputRegister()); 1166 __ j(positive, &success); 1167 // The result of the first conversion was negative, which means that the 1168 // input value was not within the positive int64 range. We subtract 2^64 1169 // and convert it again to see if it is within the uint64 range. 1170 __ Move(kScratchDoubleReg, -9223372036854775808.0); 1171 if (instr->InputAt(0)->IsDoubleRegister()) { 1172 __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0)); 1173 } else { 1174 __ addsd(kScratchDoubleReg, i.InputOperand(0)); 1175 } 1176 __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg); 1177 __ testq(i.OutputRegister(), i.OutputRegister()); 1178 // The only possible negative value here is 0x80000000000000000, which is 1179 // used on x64 to indicate an integer overflow. 1180 __ j(negative, &done); 1181 // The input value is within uint64 range and the second conversion worked 1182 // successfully, but we still have to undo the subtraction we did 1183 // earlier. 1184 __ Set(kScratchRegister, 0x8000000000000000); 1185 __ orq(i.OutputRegister(), kScratchRegister); 1186 __ bind(&success); 1187 if (instr->OutputCount() > 1) { 1188 __ Set(i.OutputRegister(1), 1); 1189 } 1190 __ bind(&done); 1191 break; 1192 } 1193 case kSSEInt32ToFloat64: 1194 if (instr->InputAt(0)->IsRegister()) { 1195 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); 1196 } else { 1197 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); 1198 } 1199 break; 1200 case kSSEInt64ToFloat32: 1201 if (instr->InputAt(0)->IsRegister()) { 1202 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); 1203 } else { 1204 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); 1205 } 1206 break; 1207 case kSSEInt64ToFloat64: 1208 if (instr->InputAt(0)->IsRegister()) { 1209 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); 1210 } else { 1211 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); 1212 } 1213 break; 1214 case kSSEUint64ToFloat32: 1215 if (instr->InputAt(0)->IsRegister()) { 1216 __ movq(kScratchRegister, i.InputRegister(0)); 1217 } else { 1218 __ movq(kScratchRegister, i.InputOperand(0)); 1219 } 1220 __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister, 1221 i.TempRegister(0)); 1222 break; 1223 case kSSEUint64ToFloat64: 1224 if (instr->InputAt(0)->IsRegister()) { 1225 __ movq(kScratchRegister, i.InputRegister(0)); 1226 } else { 1227 __ movq(kScratchRegister, i.InputOperand(0)); 1228 } 1229 __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister, 1230 i.TempRegister(0)); 1231 break; 1232 case kSSEUint32ToFloat64: 1233 if (instr->InputAt(0)->IsRegister()) { 1234 __ movl(kScratchRegister, i.InputRegister(0)); 1235 } else { 1236 __ movl(kScratchRegister, i.InputOperand(0)); 1237 } 1238 __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister); 1239 break; 1240 case kSSEFloat64ExtractLowWord32: 1241 if (instr->InputAt(0)->IsDoubleStackSlot()) { 1242 __ movl(i.OutputRegister(), i.InputOperand(0)); 1243 } else { 1244 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); 1245 } 1246 break; 1247 case kSSEFloat64ExtractHighWord32: 1248 if (instr->InputAt(0)->IsDoubleStackSlot()) { 1249 __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2)); 1250 } else { 1251 __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1); 1252 } 1253 break; 1254 case kSSEFloat64InsertLowWord32: 1255 if (instr->InputAt(1)->IsRegister()) { 1256 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0); 1257 } else { 1258 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0); 1259 } 1260 break; 1261 case kSSEFloat64InsertHighWord32: 1262 if (instr->InputAt(1)->IsRegister()) { 1263 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1); 1264 } else { 1265 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1); 1266 } 1267 break; 1268 case kSSEFloat64LoadLowWord32: 1269 if (instr->InputAt(0)->IsRegister()) { 1270 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); 1271 } else { 1272 __ Movd(i.OutputDoubleRegister(), i.InputOperand(0)); 1273 } 1274 break; 1275 case kAVXFloat32Cmp: { 1276 CpuFeatureScope avx_scope(masm(), AVX); 1277 if (instr->InputAt(1)->IsDoubleRegister()) { 1278 __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1279 } else { 1280 __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1281 } 1282 break; 1283 } 1284 case kAVXFloat32Add: 1285 ASSEMBLE_AVX_BINOP(vaddss); 1286 break; 1287 case kAVXFloat32Sub: 1288 ASSEMBLE_AVX_BINOP(vsubss); 1289 break; 1290 case kAVXFloat32Mul: 1291 ASSEMBLE_AVX_BINOP(vmulss); 1292 break; 1293 case kAVXFloat32Div: 1294 ASSEMBLE_AVX_BINOP(vdivss); 1295 // Don't delete this mov. It may improve performance on some CPUs, 1296 // when there is a (v)mulss depending on the result. 1297 __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1298 break; 1299 case kAVXFloat32Max: 1300 ASSEMBLE_AVX_BINOP(vmaxss); 1301 break; 1302 case kAVXFloat32Min: 1303 ASSEMBLE_AVX_BINOP(vminss); 1304 break; 1305 case kAVXFloat64Cmp: { 1306 CpuFeatureScope avx_scope(masm(), AVX); 1307 if (instr->InputAt(1)->IsDoubleRegister()) { 1308 __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1309 } else { 1310 __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1311 } 1312 break; 1313 } 1314 case kAVXFloat64Add: 1315 ASSEMBLE_AVX_BINOP(vaddsd); 1316 break; 1317 case kAVXFloat64Sub: 1318 ASSEMBLE_AVX_BINOP(vsubsd); 1319 break; 1320 case kAVXFloat64Mul: 1321 ASSEMBLE_AVX_BINOP(vmulsd); 1322 break; 1323 case kAVXFloat64Div: 1324 ASSEMBLE_AVX_BINOP(vdivsd); 1325 // Don't delete this mov. It may improve performance on some CPUs, 1326 // when there is a (v)mulsd depending on the result. 1327 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1328 break; 1329 case kAVXFloat64Max: 1330 ASSEMBLE_AVX_BINOP(vmaxsd); 1331 break; 1332 case kAVXFloat64Min: 1333 ASSEMBLE_AVX_BINOP(vminsd); 1334 break; 1335 case kAVXFloat32Abs: { 1336 // TODO(bmeurer): Use RIP relative 128-bit constants. 1337 CpuFeatureScope avx_scope(masm(), AVX); 1338 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1339 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33); 1340 if (instr->InputAt(0)->IsDoubleRegister()) { 1341 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, 1342 i.InputDoubleRegister(0)); 1343 } else { 1344 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, 1345 i.InputOperand(0)); 1346 } 1347 break; 1348 } 1349 case kAVXFloat32Neg: { 1350 // TODO(bmeurer): Use RIP relative 128-bit constants. 1351 CpuFeatureScope avx_scope(masm(), AVX); 1352 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1353 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31); 1354 if (instr->InputAt(0)->IsDoubleRegister()) { 1355 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, 1356 i.InputDoubleRegister(0)); 1357 } else { 1358 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, 1359 i.InputOperand(0)); 1360 } 1361 break; 1362 } 1363 case kAVXFloat64Abs: { 1364 // TODO(bmeurer): Use RIP relative 128-bit constants. 1365 CpuFeatureScope avx_scope(masm(), AVX); 1366 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1367 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1); 1368 if (instr->InputAt(0)->IsDoubleRegister()) { 1369 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1370 i.InputDoubleRegister(0)); 1371 } else { 1372 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1373 i.InputOperand(0)); 1374 } 1375 break; 1376 } 1377 case kAVXFloat64Neg: { 1378 // TODO(bmeurer): Use RIP relative 128-bit constants. 1379 CpuFeatureScope avx_scope(masm(), AVX); 1380 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1381 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63); 1382 if (instr->InputAt(0)->IsDoubleRegister()) { 1383 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1384 i.InputDoubleRegister(0)); 1385 } else { 1386 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1387 i.InputOperand(0)); 1388 } 1389 break; 1390 } 1391 case kX64Movsxbl: 1392 ASSEMBLE_MOVX(movsxbl); 1393 __ AssertZeroExtended(i.OutputRegister()); 1394 break; 1395 case kX64Movzxbl: 1396 ASSEMBLE_MOVX(movzxbl); 1397 __ AssertZeroExtended(i.OutputRegister()); 1398 break; 1399 case kX64Movb: { 1400 size_t index = 0; 1401 Operand operand = i.MemoryOperand(&index); 1402 if (HasImmediateInput(instr, index)) { 1403 __ movb(operand, Immediate(i.InputInt8(index))); 1404 } else { 1405 __ movb(operand, i.InputRegister(index)); 1406 } 1407 break; 1408 } 1409 case kX64Movsxwl: 1410 ASSEMBLE_MOVX(movsxwl); 1411 __ AssertZeroExtended(i.OutputRegister()); 1412 break; 1413 case kX64Movzxwl: 1414 ASSEMBLE_MOVX(movzxwl); 1415 __ AssertZeroExtended(i.OutputRegister()); 1416 break; 1417 case kX64Movw: { 1418 size_t index = 0; 1419 Operand operand = i.MemoryOperand(&index); 1420 if (HasImmediateInput(instr, index)) { 1421 __ movw(operand, Immediate(i.InputInt16(index))); 1422 } else { 1423 __ movw(operand, i.InputRegister(index)); 1424 } 1425 break; 1426 } 1427 case kX64Movl: 1428 if (instr->HasOutput()) { 1429 if (instr->addressing_mode() == kMode_None) { 1430 if (instr->InputAt(0)->IsRegister()) { 1431 __ movl(i.OutputRegister(), i.InputRegister(0)); 1432 } else { 1433 __ movl(i.OutputRegister(), i.InputOperand(0)); 1434 } 1435 } else { 1436 __ movl(i.OutputRegister(), i.MemoryOperand()); 1437 } 1438 __ AssertZeroExtended(i.OutputRegister()); 1439 } else { 1440 size_t index = 0; 1441 Operand operand = i.MemoryOperand(&index); 1442 if (HasImmediateInput(instr, index)) { 1443 __ movl(operand, i.InputImmediate(index)); 1444 } else { 1445 __ movl(operand, i.InputRegister(index)); 1446 } 1447 } 1448 break; 1449 case kX64Movsxlq: 1450 ASSEMBLE_MOVX(movsxlq); 1451 break; 1452 case kX64Movq: 1453 if (instr->HasOutput()) { 1454 __ movq(i.OutputRegister(), i.MemoryOperand()); 1455 } else { 1456 size_t index = 0; 1457 Operand operand = i.MemoryOperand(&index); 1458 if (HasImmediateInput(instr, index)) { 1459 __ movq(operand, i.InputImmediate(index)); 1460 } else { 1461 __ movq(operand, i.InputRegister(index)); 1462 } 1463 } 1464 break; 1465 case kX64Movss: 1466 if (instr->HasOutput()) { 1467 __ movss(i.OutputDoubleRegister(), i.MemoryOperand()); 1468 } else { 1469 size_t index = 0; 1470 Operand operand = i.MemoryOperand(&index); 1471 __ movss(operand, i.InputDoubleRegister(index)); 1472 } 1473 break; 1474 case kX64Movsd: 1475 if (instr->HasOutput()) { 1476 __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand()); 1477 } else { 1478 size_t index = 0; 1479 Operand operand = i.MemoryOperand(&index); 1480 __ Movsd(operand, i.InputDoubleRegister(index)); 1481 } 1482 break; 1483 case kX64BitcastFI: 1484 if (instr->InputAt(0)->IsDoubleStackSlot()) { 1485 __ movl(i.OutputRegister(), i.InputOperand(0)); 1486 } else { 1487 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); 1488 } 1489 break; 1490 case kX64BitcastDL: 1491 if (instr->InputAt(0)->IsDoubleStackSlot()) { 1492 __ movq(i.OutputRegister(), i.InputOperand(0)); 1493 } else { 1494 __ Movq(i.OutputRegister(), i.InputDoubleRegister(0)); 1495 } 1496 break; 1497 case kX64BitcastIF: 1498 if (instr->InputAt(0)->IsRegister()) { 1499 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); 1500 } else { 1501 __ movss(i.OutputDoubleRegister(), i.InputOperand(0)); 1502 } 1503 break; 1504 case kX64BitcastLD: 1505 if (instr->InputAt(0)->IsRegister()) { 1506 __ Movq(i.OutputDoubleRegister(), i.InputRegister(0)); 1507 } else { 1508 __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0)); 1509 } 1510 break; 1511 case kX64Lea32: { 1512 AddressingMode mode = AddressingModeField::decode(instr->opcode()); 1513 // Shorten "leal" to "addl", "subl" or "shll" if the register allocation 1514 // and addressing mode just happens to work out. The "addl"/"subl" forms 1515 // in these cases are faster based on measurements. 1516 if (i.InputRegister(0).is(i.OutputRegister())) { 1517 if (mode == kMode_MRI) { 1518 int32_t constant_summand = i.InputInt32(1); 1519 if (constant_summand > 0) { 1520 __ addl(i.OutputRegister(), Immediate(constant_summand)); 1521 } else if (constant_summand < 0) { 1522 __ subl(i.OutputRegister(), Immediate(-constant_summand)); 1523 } 1524 } else if (mode == kMode_MR1) { 1525 if (i.InputRegister(1).is(i.OutputRegister())) { 1526 __ shll(i.OutputRegister(), Immediate(1)); 1527 } else { 1528 __ leal(i.OutputRegister(), i.MemoryOperand()); 1529 } 1530 } else if (mode == kMode_M2) { 1531 __ shll(i.OutputRegister(), Immediate(1)); 1532 } else if (mode == kMode_M4) { 1533 __ shll(i.OutputRegister(), Immediate(2)); 1534 } else if (mode == kMode_M8) { 1535 __ shll(i.OutputRegister(), Immediate(3)); 1536 } else { 1537 __ leal(i.OutputRegister(), i.MemoryOperand()); 1538 } 1539 } else { 1540 __ leal(i.OutputRegister(), i.MemoryOperand()); 1541 } 1542 __ AssertZeroExtended(i.OutputRegister()); 1543 break; 1544 } 1545 case kX64Lea: 1546 __ leaq(i.OutputRegister(), i.MemoryOperand()); 1547 break; 1548 case kX64Dec32: 1549 __ decl(i.OutputRegister()); 1550 break; 1551 case kX64Inc32: 1552 __ incl(i.OutputRegister()); 1553 break; 1554 case kX64Push: 1555 if (HasImmediateInput(instr, 0)) { 1556 __ pushq(i.InputImmediate(0)); 1557 frame_access_state()->IncreaseSPDelta(1); 1558 } else { 1559 if (instr->InputAt(0)->IsRegister()) { 1560 __ pushq(i.InputRegister(0)); 1561 frame_access_state()->IncreaseSPDelta(1); 1562 } else if (instr->InputAt(0)->IsDoubleRegister()) { 1563 // TODO(titzer): use another machine instruction? 1564 __ subq(rsp, Immediate(kDoubleSize)); 1565 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 1566 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); 1567 } else { 1568 __ pushq(i.InputOperand(0)); 1569 frame_access_state()->IncreaseSPDelta(1); 1570 } 1571 } 1572 break; 1573 case kX64Poke: { 1574 int const slot = MiscField::decode(instr->opcode()); 1575 if (HasImmediateInput(instr, 0)) { 1576 __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0)); 1577 } else { 1578 __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0)); 1579 } 1580 break; 1581 } 1582 case kCheckedLoadInt8: 1583 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl); 1584 break; 1585 case kCheckedLoadUint8: 1586 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl); 1587 break; 1588 case kCheckedLoadInt16: 1589 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl); 1590 break; 1591 case kCheckedLoadUint16: 1592 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl); 1593 break; 1594 case kCheckedLoadWord32: 1595 ASSEMBLE_CHECKED_LOAD_INTEGER(movl); 1596 break; 1597 case kCheckedLoadWord64: 1598 ASSEMBLE_CHECKED_LOAD_INTEGER(movq); 1599 break; 1600 case kCheckedLoadFloat32: 1601 ASSEMBLE_CHECKED_LOAD_FLOAT(Movss); 1602 break; 1603 case kCheckedLoadFloat64: 1604 ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd); 1605 break; 1606 case kCheckedStoreWord8: 1607 ASSEMBLE_CHECKED_STORE_INTEGER(movb); 1608 break; 1609 case kCheckedStoreWord16: 1610 ASSEMBLE_CHECKED_STORE_INTEGER(movw); 1611 break; 1612 case kCheckedStoreWord32: 1613 ASSEMBLE_CHECKED_STORE_INTEGER(movl); 1614 break; 1615 case kCheckedStoreWord64: 1616 ASSEMBLE_CHECKED_STORE_INTEGER(movq); 1617 break; 1618 case kCheckedStoreFloat32: 1619 ASSEMBLE_CHECKED_STORE_FLOAT(Movss); 1620 break; 1621 case kCheckedStoreFloat64: 1622 ASSEMBLE_CHECKED_STORE_FLOAT(Movsd); 1623 break; 1624 case kX64StackCheck: 1625 __ CompareRoot(rsp, Heap::kStackLimitRootIndex); 1626 break; 1627 } 1628 } // NOLINT(readability/fn_size) 1629 1630 1631 // Assembles branches after this instruction. 1632 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { 1633 X64OperandConverter i(this, instr); 1634 Label::Distance flabel_distance = 1635 branch->fallthru ? Label::kNear : Label::kFar; 1636 Label* tlabel = branch->true_label; 1637 Label* flabel = branch->false_label; 1638 switch (branch->condition) { 1639 case kUnorderedEqual: 1640 __ j(parity_even, flabel, flabel_distance); 1641 // Fall through. 1642 case kEqual: 1643 __ j(equal, tlabel); 1644 break; 1645 case kUnorderedNotEqual: 1646 __ j(parity_even, tlabel); 1647 // Fall through. 1648 case kNotEqual: 1649 __ j(not_equal, tlabel); 1650 break; 1651 case kSignedLessThan: 1652 __ j(less, tlabel); 1653 break; 1654 case kSignedGreaterThanOrEqual: 1655 __ j(greater_equal, tlabel); 1656 break; 1657 case kSignedLessThanOrEqual: 1658 __ j(less_equal, tlabel); 1659 break; 1660 case kSignedGreaterThan: 1661 __ j(greater, tlabel); 1662 break; 1663 case kUnsignedLessThan: 1664 __ j(below, tlabel); 1665 break; 1666 case kUnsignedGreaterThanOrEqual: 1667 __ j(above_equal, tlabel); 1668 break; 1669 case kUnsignedLessThanOrEqual: 1670 __ j(below_equal, tlabel); 1671 break; 1672 case kUnsignedGreaterThan: 1673 __ j(above, tlabel); 1674 break; 1675 case kOverflow: 1676 __ j(overflow, tlabel); 1677 break; 1678 case kNotOverflow: 1679 __ j(no_overflow, tlabel); 1680 break; 1681 default: 1682 UNREACHABLE(); 1683 break; 1684 } 1685 if (!branch->fallthru) __ jmp(flabel, flabel_distance); 1686 } 1687 1688 1689 void CodeGenerator::AssembleArchJump(RpoNumber target) { 1690 if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target)); 1691 } 1692 1693 1694 // Assembles boolean materializations after this instruction. 1695 void CodeGenerator::AssembleArchBoolean(Instruction* instr, 1696 FlagsCondition condition) { 1697 X64OperandConverter i(this, instr); 1698 Label done; 1699 1700 // Materialize a full 64-bit 1 or 0 value. The result register is always the 1701 // last output of the instruction. 1702 Label check; 1703 DCHECK_NE(0u, instr->OutputCount()); 1704 Register reg = i.OutputRegister(instr->OutputCount() - 1); 1705 Condition cc = no_condition; 1706 switch (condition) { 1707 case kUnorderedEqual: 1708 __ j(parity_odd, &check, Label::kNear); 1709 __ movl(reg, Immediate(0)); 1710 __ jmp(&done, Label::kNear); 1711 // Fall through. 1712 case kEqual: 1713 cc = equal; 1714 break; 1715 case kUnorderedNotEqual: 1716 __ j(parity_odd, &check, Label::kNear); 1717 __ movl(reg, Immediate(1)); 1718 __ jmp(&done, Label::kNear); 1719 // Fall through. 1720 case kNotEqual: 1721 cc = not_equal; 1722 break; 1723 case kSignedLessThan: 1724 cc = less; 1725 break; 1726 case kSignedGreaterThanOrEqual: 1727 cc = greater_equal; 1728 break; 1729 case kSignedLessThanOrEqual: 1730 cc = less_equal; 1731 break; 1732 case kSignedGreaterThan: 1733 cc = greater; 1734 break; 1735 case kUnsignedLessThan: 1736 cc = below; 1737 break; 1738 case kUnsignedGreaterThanOrEqual: 1739 cc = above_equal; 1740 break; 1741 case kUnsignedLessThanOrEqual: 1742 cc = below_equal; 1743 break; 1744 case kUnsignedGreaterThan: 1745 cc = above; 1746 break; 1747 case kOverflow: 1748 cc = overflow; 1749 break; 1750 case kNotOverflow: 1751 cc = no_overflow; 1752 break; 1753 default: 1754 UNREACHABLE(); 1755 break; 1756 } 1757 __ bind(&check); 1758 __ setcc(cc, reg); 1759 __ movzxbl(reg, reg); 1760 __ bind(&done); 1761 } 1762 1763 1764 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) { 1765 X64OperandConverter i(this, instr); 1766 Register input = i.InputRegister(0); 1767 for (size_t index = 2; index < instr->InputCount(); index += 2) { 1768 __ cmpl(input, Immediate(i.InputInt32(index + 0))); 1769 __ j(equal, GetLabel(i.InputRpo(index + 1))); 1770 } 1771 AssembleArchJump(i.InputRpo(1)); 1772 } 1773 1774 1775 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) { 1776 X64OperandConverter i(this, instr); 1777 Register input = i.InputRegister(0); 1778 int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2); 1779 Label** cases = zone()->NewArray<Label*>(case_count); 1780 for (int32_t index = 0; index < case_count; ++index) { 1781 cases[index] = GetLabel(i.InputRpo(index + 2)); 1782 } 1783 Label* const table = AddJumpTable(cases, case_count); 1784 __ cmpl(input, Immediate(case_count)); 1785 __ j(above_equal, GetLabel(i.InputRpo(1))); 1786 __ leaq(kScratchRegister, Operand(table)); 1787 __ jmp(Operand(kScratchRegister, input, times_8, 0)); 1788 } 1789 1790 1791 void CodeGenerator::AssembleDeoptimizerCall( 1792 int deoptimization_id, Deoptimizer::BailoutType bailout_type) { 1793 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry( 1794 isolate(), deoptimization_id, bailout_type); 1795 __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY); 1796 } 1797 1798 1799 namespace { 1800 1801 static const int kQuadWordSize = 16; 1802 1803 } // namespace 1804 1805 1806 void CodeGenerator::AssemblePrologue() { 1807 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1808 if (descriptor->IsCFunctionCall()) { 1809 __ pushq(rbp); 1810 __ movq(rbp, rsp); 1811 } else if (descriptor->IsJSFunctionCall()) { 1812 __ Prologue(this->info()->GeneratePreagedPrologue()); 1813 } else if (frame()->needs_frame()) { 1814 __ StubPrologue(); 1815 } else { 1816 frame()->SetElidedFrameSizeInSlots(kPCOnStackSize / kPointerSize); 1817 } 1818 frame_access_state()->SetFrameAccessToDefault(); 1819 1820 int stack_shrink_slots = frame()->GetSpillSlotCount(); 1821 if (info()->is_osr()) { 1822 // TurboFan OSR-compiled functions cannot be entered directly. 1823 __ Abort(kShouldNotDirectlyEnterOsrFunction); 1824 1825 // Unoptimized code jumps directly to this entrypoint while the unoptimized 1826 // frame is still on the stack. Optimized code uses OSR values directly from 1827 // the unoptimized frame. Thus, all that needs to be done is to allocate the 1828 // remaining stack slots. 1829 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --"); 1830 osr_pc_offset_ = __ pc_offset(); 1831 // TODO(titzer): cannot address target function == local #-1 1832 __ movq(rdi, Operand(rbp, JavaScriptFrameConstants::kFunctionOffset)); 1833 stack_shrink_slots -= 1834 static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots()); 1835 } 1836 1837 const RegList saves_fp = descriptor->CalleeSavedFPRegisters(); 1838 if (saves_fp != 0) { 1839 stack_shrink_slots += frame()->AlignSavedCalleeRegisterSlots(); 1840 } 1841 if (stack_shrink_slots > 0) { 1842 __ subq(rsp, Immediate(stack_shrink_slots * kPointerSize)); 1843 } 1844 1845 if (saves_fp != 0) { // Save callee-saved XMM registers. 1846 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp); 1847 const int stack_size = saves_fp_count * kQuadWordSize; 1848 // Adjust the stack pointer. 1849 __ subp(rsp, Immediate(stack_size)); 1850 // Store the registers on the stack. 1851 int slot_idx = 0; 1852 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) { 1853 if (!((1 << i) & saves_fp)) continue; 1854 __ movdqu(Operand(rsp, kQuadWordSize * slot_idx), 1855 XMMRegister::from_code(i)); 1856 slot_idx++; 1857 } 1858 frame()->AllocateSavedCalleeRegisterSlots(saves_fp_count * 1859 (kQuadWordSize / kPointerSize)); 1860 } 1861 1862 const RegList saves = descriptor->CalleeSavedRegisters(); 1863 if (saves != 0) { // Save callee-saved registers. 1864 for (int i = Register::kNumRegisters - 1; i >= 0; i--) { 1865 if (!((1 << i) & saves)) continue; 1866 __ pushq(Register::from_code(i)); 1867 frame()->AllocateSavedCalleeRegisterSlots(1); 1868 } 1869 } 1870 } 1871 1872 1873 void CodeGenerator::AssembleReturn() { 1874 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 1875 1876 // Restore registers. 1877 const RegList saves = descriptor->CalleeSavedRegisters(); 1878 if (saves != 0) { 1879 for (int i = 0; i < Register::kNumRegisters; i++) { 1880 if (!((1 << i) & saves)) continue; 1881 __ popq(Register::from_code(i)); 1882 } 1883 } 1884 const RegList saves_fp = descriptor->CalleeSavedFPRegisters(); 1885 if (saves_fp != 0) { 1886 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp); 1887 const int stack_size = saves_fp_count * kQuadWordSize; 1888 // Load the registers from the stack. 1889 int slot_idx = 0; 1890 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) { 1891 if (!((1 << i) & saves_fp)) continue; 1892 __ movdqu(XMMRegister::from_code(i), 1893 Operand(rsp, kQuadWordSize * slot_idx)); 1894 slot_idx++; 1895 } 1896 // Adjust the stack pointer. 1897 __ addp(rsp, Immediate(stack_size)); 1898 } 1899 1900 if (descriptor->IsCFunctionCall()) { 1901 __ movq(rsp, rbp); // Move stack pointer back to frame pointer. 1902 __ popq(rbp); // Pop caller's frame pointer. 1903 } else if (frame()->needs_frame()) { 1904 // Canonicalize JSFunction return sites for now. 1905 if (return_label_.is_bound()) { 1906 __ jmp(&return_label_); 1907 return; 1908 } else { 1909 __ bind(&return_label_); 1910 __ movq(rsp, rbp); // Move stack pointer back to frame pointer. 1911 __ popq(rbp); // Pop caller's frame pointer. 1912 } 1913 } 1914 size_t pop_size = descriptor->StackParameterCount() * kPointerSize; 1915 // Might need rcx for scratch if pop_size is too big. 1916 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit()); 1917 __ Ret(static_cast<int>(pop_size), rcx); 1918 } 1919 1920 1921 void CodeGenerator::AssembleMove(InstructionOperand* source, 1922 InstructionOperand* destination) { 1923 X64OperandConverter g(this, nullptr); 1924 // Dispatch on the source and destination operand kinds. Not all 1925 // combinations are possible. 1926 if (source->IsRegister()) { 1927 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 1928 Register src = g.ToRegister(source); 1929 if (destination->IsRegister()) { 1930 __ movq(g.ToRegister(destination), src); 1931 } else { 1932 __ movq(g.ToOperand(destination), src); 1933 } 1934 } else if (source->IsStackSlot()) { 1935 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 1936 Operand src = g.ToOperand(source); 1937 if (destination->IsRegister()) { 1938 Register dst = g.ToRegister(destination); 1939 __ movq(dst, src); 1940 } else { 1941 // Spill on demand to use a temporary register for memory-to-memory 1942 // moves. 1943 Register tmp = kScratchRegister; 1944 Operand dst = g.ToOperand(destination); 1945 __ movq(tmp, src); 1946 __ movq(dst, tmp); 1947 } 1948 } else if (source->IsConstant()) { 1949 ConstantOperand* constant_source = ConstantOperand::cast(source); 1950 Constant src = g.ToConstant(constant_source); 1951 if (destination->IsRegister() || destination->IsStackSlot()) { 1952 Register dst = destination->IsRegister() ? g.ToRegister(destination) 1953 : kScratchRegister; 1954 switch (src.type()) { 1955 case Constant::kInt32: 1956 // TODO(dcarney): don't need scratch in this case. 1957 __ Set(dst, src.ToInt32()); 1958 break; 1959 case Constant::kInt64: 1960 __ Set(dst, src.ToInt64()); 1961 break; 1962 case Constant::kFloat32: 1963 __ Move(dst, 1964 isolate()->factory()->NewNumber(src.ToFloat32(), TENURED)); 1965 break; 1966 case Constant::kFloat64: 1967 __ Move(dst, 1968 isolate()->factory()->NewNumber(src.ToFloat64(), TENURED)); 1969 break; 1970 case Constant::kExternalReference: 1971 __ Move(dst, src.ToExternalReference()); 1972 break; 1973 case Constant::kHeapObject: { 1974 Handle<HeapObject> src_object = src.ToHeapObject(); 1975 Heap::RootListIndex index; 1976 int offset; 1977 if (IsMaterializableFromFrame(src_object, &offset)) { 1978 __ movp(dst, Operand(rbp, offset)); 1979 } else if (IsMaterializableFromRoot(src_object, &index)) { 1980 __ LoadRoot(dst, index); 1981 } else { 1982 __ Move(dst, src_object); 1983 } 1984 break; 1985 } 1986 case Constant::kRpoNumber: 1987 UNREACHABLE(); // TODO(dcarney): load of labels on x64. 1988 break; 1989 } 1990 if (destination->IsStackSlot()) { 1991 __ movq(g.ToOperand(destination), kScratchRegister); 1992 } 1993 } else if (src.type() == Constant::kFloat32) { 1994 // TODO(turbofan): Can we do better here? 1995 uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32()); 1996 if (destination->IsDoubleRegister()) { 1997 __ Move(g.ToDoubleRegister(destination), src_const); 1998 } else { 1999 DCHECK(destination->IsDoubleStackSlot()); 2000 Operand dst = g.ToOperand(destination); 2001 __ movl(dst, Immediate(src_const)); 2002 } 2003 } else { 2004 DCHECK_EQ(Constant::kFloat64, src.type()); 2005 uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64()); 2006 if (destination->IsDoubleRegister()) { 2007 __ Move(g.ToDoubleRegister(destination), src_const); 2008 } else { 2009 DCHECK(destination->IsDoubleStackSlot()); 2010 __ movq(kScratchRegister, src_const); 2011 __ movq(g.ToOperand(destination), kScratchRegister); 2012 } 2013 } 2014 } else if (source->IsDoubleRegister()) { 2015 XMMRegister src = g.ToDoubleRegister(source); 2016 if (destination->IsDoubleRegister()) { 2017 XMMRegister dst = g.ToDoubleRegister(destination); 2018 __ Movapd(dst, src); 2019 } else { 2020 DCHECK(destination->IsDoubleStackSlot()); 2021 Operand dst = g.ToOperand(destination); 2022 __ Movsd(dst, src); 2023 } 2024 } else if (source->IsDoubleStackSlot()) { 2025 DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot()); 2026 Operand src = g.ToOperand(source); 2027 if (destination->IsDoubleRegister()) { 2028 XMMRegister dst = g.ToDoubleRegister(destination); 2029 __ Movsd(dst, src); 2030 } else { 2031 // We rely on having xmm0 available as a fixed scratch register. 2032 Operand dst = g.ToOperand(destination); 2033 __ Movsd(xmm0, src); 2034 __ Movsd(dst, xmm0); 2035 } 2036 } else { 2037 UNREACHABLE(); 2038 } 2039 } 2040 2041 2042 void CodeGenerator::AssembleSwap(InstructionOperand* source, 2043 InstructionOperand* destination) { 2044 X64OperandConverter g(this, nullptr); 2045 // Dispatch on the source and destination operand kinds. Not all 2046 // combinations are possible. 2047 if (source->IsRegister() && destination->IsRegister()) { 2048 // Register-register. 2049 Register src = g.ToRegister(source); 2050 Register dst = g.ToRegister(destination); 2051 __ movq(kScratchRegister, src); 2052 __ movq(src, dst); 2053 __ movq(dst, kScratchRegister); 2054 } else if (source->IsRegister() && destination->IsStackSlot()) { 2055 Register src = g.ToRegister(source); 2056 __ pushq(src); 2057 frame_access_state()->IncreaseSPDelta(1); 2058 Operand dst = g.ToOperand(destination); 2059 __ movq(src, dst); 2060 frame_access_state()->IncreaseSPDelta(-1); 2061 dst = g.ToOperand(destination); 2062 __ popq(dst); 2063 } else if ((source->IsStackSlot() && destination->IsStackSlot()) || 2064 (source->IsDoubleStackSlot() && 2065 destination->IsDoubleStackSlot())) { 2066 // Memory-memory. 2067 Register tmp = kScratchRegister; 2068 Operand src = g.ToOperand(source); 2069 Operand dst = g.ToOperand(destination); 2070 __ movq(tmp, dst); 2071 __ pushq(src); 2072 frame_access_state()->IncreaseSPDelta(1); 2073 src = g.ToOperand(source); 2074 __ movq(src, tmp); 2075 frame_access_state()->IncreaseSPDelta(-1); 2076 dst = g.ToOperand(destination); 2077 __ popq(dst); 2078 } else if (source->IsDoubleRegister() && destination->IsDoubleRegister()) { 2079 // XMM register-register swap. We rely on having xmm0 2080 // available as a fixed scratch register. 2081 XMMRegister src = g.ToDoubleRegister(source); 2082 XMMRegister dst = g.ToDoubleRegister(destination); 2083 __ Movapd(xmm0, src); 2084 __ Movapd(src, dst); 2085 __ Movapd(dst, xmm0); 2086 } else if (source->IsDoubleRegister() && destination->IsDoubleStackSlot()) { 2087 // XMM register-memory swap. We rely on having xmm0 2088 // available as a fixed scratch register. 2089 XMMRegister src = g.ToDoubleRegister(source); 2090 Operand dst = g.ToOperand(destination); 2091 __ Movsd(xmm0, src); 2092 __ Movsd(src, dst); 2093 __ Movsd(dst, xmm0); 2094 } else { 2095 // No other combinations are possible. 2096 UNREACHABLE(); 2097 } 2098 } 2099 2100 2101 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) { 2102 for (size_t index = 0; index < target_count; ++index) { 2103 __ dq(targets[index]); 2104 } 2105 } 2106 2107 2108 void CodeGenerator::AddNopForSmiCodeInlining() { __ nop(); } 2109 2110 2111 void CodeGenerator::EnsureSpaceForLazyDeopt() { 2112 if (!info()->ShouldEnsureSpaceForLazyDeopt()) { 2113 return; 2114 } 2115 2116 int space_needed = Deoptimizer::patch_size(); 2117 // Ensure that we have enough space after the previous lazy-bailout 2118 // instruction for patching the code here. 2119 int current_pc = masm()->pc_offset(); 2120 if (current_pc < last_lazy_deopt_pc_ + space_needed) { 2121 int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc; 2122 __ Nop(padding_size); 2123 } 2124 } 2125 2126 #undef __ 2127 2128 } // namespace compiler 2129 } // namespace internal 2130 } // namespace v8 2131
