1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/code-generator.h" 6 7 #include "src/compilation-info.h" 8 #include "src/compiler/code-generator-impl.h" 9 #include "src/compiler/gap-resolver.h" 10 #include "src/compiler/node-matchers.h" 11 #include "src/compiler/osr.h" 12 #include "src/frames.h" 13 #include "src/ia32/assembler-ia32.h" 14 #include "src/ia32/frames-ia32.h" 15 #include "src/ia32/macro-assembler-ia32.h" 16 17 namespace v8 { 18 namespace internal { 19 namespace compiler { 20 21 #define __ masm()-> 22 23 24 #define kScratchDoubleReg xmm0 25 26 27 // Adds IA-32 specific methods for decoding operands. 28 class IA32OperandConverter : public InstructionOperandConverter { 29 public: 30 IA32OperandConverter(CodeGenerator* gen, Instruction* instr) 31 : InstructionOperandConverter(gen, instr) {} 32 33 Operand InputOperand(size_t index, int extra = 0) { 34 return ToOperand(instr_->InputAt(index), extra); 35 } 36 37 Immediate InputImmediate(size_t index) { 38 return ToImmediate(instr_->InputAt(index)); 39 } 40 41 Operand OutputOperand() { return ToOperand(instr_->Output()); } 42 43 Operand ToOperand(InstructionOperand* op, int extra = 0) { 44 if (op->IsRegister()) { 45 DCHECK(extra == 0); 46 return Operand(ToRegister(op)); 47 } else if (op->IsFPRegister()) { 48 DCHECK(extra == 0); 49 return Operand(ToDoubleRegister(op)); 50 } 51 DCHECK(op->IsStackSlot() || op->IsFPStackSlot()); 52 return SlotToOperand(AllocatedOperand::cast(op)->index(), extra); 53 } 54 55 Operand SlotToOperand(int slot, int extra = 0) { 56 FrameOffset offset = frame_access_state()->GetFrameOffset(slot); 57 return Operand(offset.from_stack_pointer() ? esp : ebp, 58 offset.offset() + extra); 59 } 60 61 Operand HighOperand(InstructionOperand* op) { 62 DCHECK(op->IsFPStackSlot()); 63 return ToOperand(op, kPointerSize); 64 } 65 66 Immediate ToImmediate(InstructionOperand* operand) { 67 Constant constant = ToConstant(operand); 68 if (constant.type() == Constant::kInt32 && 69 RelocInfo::IsWasmReference(constant.rmode())) { 70 return Immediate(reinterpret_cast<Address>(constant.ToInt32()), 71 constant.rmode()); 72 } 73 switch (constant.type()) { 74 case Constant::kInt32: 75 return Immediate(constant.ToInt32()); 76 case Constant::kFloat32: 77 return Immediate( 78 isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED)); 79 case Constant::kFloat64: 80 return Immediate( 81 isolate()->factory()->NewNumber(constant.ToFloat64(), TENURED)); 82 case Constant::kExternalReference: 83 return Immediate(constant.ToExternalReference()); 84 case Constant::kHeapObject: 85 return Immediate(constant.ToHeapObject()); 86 case Constant::kInt64: 87 break; 88 case Constant::kRpoNumber: 89 return Immediate::CodeRelativeOffset(ToLabel(operand)); 90 } 91 UNREACHABLE(); 92 return Immediate(-1); 93 } 94 95 static size_t NextOffset(size_t* offset) { 96 size_t i = *offset; 97 (*offset)++; 98 return i; 99 } 100 101 static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) { 102 STATIC_ASSERT(0 == static_cast<int>(times_1)); 103 STATIC_ASSERT(1 == static_cast<int>(times_2)); 104 STATIC_ASSERT(2 == static_cast<int>(times_4)); 105 STATIC_ASSERT(3 == static_cast<int>(times_8)); 106 int scale = static_cast<int>(mode - one); 107 DCHECK(scale >= 0 && scale < 4); 108 return static_cast<ScaleFactor>(scale); 109 } 110 111 Operand MemoryOperand(size_t* offset) { 112 AddressingMode mode = AddressingModeField::decode(instr_->opcode()); 113 switch (mode) { 114 case kMode_MR: { 115 Register base = InputRegister(NextOffset(offset)); 116 int32_t disp = 0; 117 return Operand(base, disp); 118 } 119 case kMode_MRI: { 120 Register base = InputRegister(NextOffset(offset)); 121 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset))); 122 return Operand(base, ctant.ToInt32(), ctant.rmode()); 123 } 124 case kMode_MR1: 125 case kMode_MR2: 126 case kMode_MR4: 127 case kMode_MR8: { 128 Register base = InputRegister(NextOffset(offset)); 129 Register index = InputRegister(NextOffset(offset)); 130 ScaleFactor scale = ScaleFor(kMode_MR1, mode); 131 int32_t disp = 0; 132 return Operand(base, index, scale, disp); 133 } 134 case kMode_MR1I: 135 case kMode_MR2I: 136 case kMode_MR4I: 137 case kMode_MR8I: { 138 Register base = InputRegister(NextOffset(offset)); 139 Register index = InputRegister(NextOffset(offset)); 140 ScaleFactor scale = ScaleFor(kMode_MR1I, mode); 141 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset))); 142 return Operand(base, index, scale, ctant.ToInt32(), ctant.rmode()); 143 } 144 case kMode_M1: 145 case kMode_M2: 146 case kMode_M4: 147 case kMode_M8: { 148 Register index = InputRegister(NextOffset(offset)); 149 ScaleFactor scale = ScaleFor(kMode_M1, mode); 150 int32_t disp = 0; 151 return Operand(index, scale, disp); 152 } 153 case kMode_M1I: 154 case kMode_M2I: 155 case kMode_M4I: 156 case kMode_M8I: { 157 Register index = InputRegister(NextOffset(offset)); 158 ScaleFactor scale = ScaleFor(kMode_M1I, mode); 159 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset))); 160 return Operand(index, scale, ctant.ToInt32(), ctant.rmode()); 161 } 162 case kMode_MI: { 163 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset))); 164 return Operand(ctant.ToInt32(), ctant.rmode()); 165 } 166 case kMode_None: 167 UNREACHABLE(); 168 return Operand(no_reg, 0); 169 } 170 UNREACHABLE(); 171 return Operand(no_reg, 0); 172 } 173 174 Operand MemoryOperand(size_t first_input = 0) { 175 return MemoryOperand(&first_input); 176 } 177 }; 178 179 180 namespace { 181 182 bool HasImmediateInput(Instruction* instr, size_t index) { 183 return instr->InputAt(index)->IsImmediate(); 184 } 185 186 class OutOfLineLoadZero final : public OutOfLineCode { 187 public: 188 OutOfLineLoadZero(CodeGenerator* gen, Register result) 189 : OutOfLineCode(gen), result_(result) {} 190 191 void Generate() final { __ xor_(result_, result_); } 192 193 private: 194 Register const result_; 195 }; 196 197 class OutOfLineLoadFloat32NaN final : public OutOfLineCode { 198 public: 199 OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result) 200 : OutOfLineCode(gen), result_(result) {} 201 202 void Generate() final { 203 __ xorps(result_, result_); 204 __ divss(result_, result_); 205 } 206 207 private: 208 XMMRegister const result_; 209 }; 210 211 class OutOfLineLoadFloat64NaN final : public OutOfLineCode { 212 public: 213 OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result) 214 : OutOfLineCode(gen), result_(result) {} 215 216 void Generate() final { 217 __ xorpd(result_, result_); 218 __ divsd(result_, result_); 219 } 220 221 private: 222 XMMRegister const result_; 223 }; 224 225 class OutOfLineTruncateDoubleToI final : public OutOfLineCode { 226 public: 227 OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result, 228 XMMRegister input) 229 : OutOfLineCode(gen), result_(result), input_(input) {} 230 231 void Generate() final { 232 __ sub(esp, Immediate(kDoubleSize)); 233 __ movsd(MemOperand(esp, 0), input_); 234 __ SlowTruncateToI(result_, esp, 0); 235 __ add(esp, Immediate(kDoubleSize)); 236 } 237 238 private: 239 Register const result_; 240 XMMRegister const input_; 241 }; 242 243 244 class OutOfLineRecordWrite final : public OutOfLineCode { 245 public: 246 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand, 247 Register value, Register scratch0, Register scratch1, 248 RecordWriteMode mode) 249 : OutOfLineCode(gen), 250 object_(object), 251 operand_(operand), 252 value_(value), 253 scratch0_(scratch0), 254 scratch1_(scratch1), 255 mode_(mode) {} 256 257 void Generate() final { 258 if (mode_ > RecordWriteMode::kValueIsPointer) { 259 __ JumpIfSmi(value_, exit()); 260 } 261 __ CheckPageFlag(value_, scratch0_, 262 MemoryChunk::kPointersToHereAreInterestingMask, zero, 263 exit()); 264 RememberedSetAction const remembered_set_action = 265 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET 266 : OMIT_REMEMBERED_SET; 267 SaveFPRegsMode const save_fp_mode = 268 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; 269 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_, 270 remembered_set_action, save_fp_mode); 271 __ lea(scratch1_, operand_); 272 __ CallStub(&stub); 273 } 274 275 private: 276 Register const object_; 277 Operand const operand_; 278 Register const value_; 279 Register const scratch0_; 280 Register const scratch1_; 281 RecordWriteMode const mode_; 282 }; 283 284 } // namespace 285 286 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN, \ 287 SingleOrDouble) \ 288 do { \ 289 auto result = i.OutputDoubleRegister(); \ 290 if (instr->InputAt(0)->IsRegister()) { \ 291 auto offset = i.InputRegister(0); \ 292 if (instr->InputAt(1)->IsRegister()) { \ 293 __ cmp(offset, i.InputRegister(1)); \ 294 } else { \ 295 __ cmp(offset, i.InputImmediate(1)); \ 296 } \ 297 OutOfLineCode* ool = new (zone()) OutOfLineLoadNaN(this, result); \ 298 __ j(above_equal, ool->entry()); \ 299 __ asm_instr(result, i.MemoryOperand(2)); \ 300 __ bind(ool->exit()); \ 301 } else { \ 302 auto index2 = i.InputInt32(0); \ 303 auto length = i.InputInt32(1); \ 304 auto index1 = i.InputRegister(2); \ 305 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \ 306 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(3)).rmode(); \ 307 DCHECK_LE(index2, length); \ 308 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \ 309 rmode_length)); \ 310 class OutOfLineLoadFloat final : public OutOfLineCode { \ 311 public: \ 312 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \ 313 Register buffer, Register index1, int32_t index2, \ 314 int32_t length, RelocInfo::Mode rmode_length, \ 315 RelocInfo::Mode rmode_buffer) \ 316 : OutOfLineCode(gen), \ 317 result_(result), \ 318 buffer_reg_(buffer), \ 319 buffer_int_(0), \ 320 index1_(index1), \ 321 index2_(index2), \ 322 length_(length), \ 323 rmode_length_(rmode_length), \ 324 rmode_buffer_(rmode_buffer) {} \ 325 \ 326 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \ 327 int32_t buffer, Register index1, int32_t index2, \ 328 int32_t length, RelocInfo::Mode rmode_length, \ 329 RelocInfo::Mode rmode_buffer) \ 330 : OutOfLineCode(gen), \ 331 result_(result), \ 332 buffer_reg_({-1}), \ 333 buffer_int_(buffer), \ 334 index1_(index1), \ 335 index2_(index2), \ 336 length_(length), \ 337 rmode_length_(rmode_length), \ 338 rmode_buffer_(rmode_buffer) {} \ 339 \ 340 void Generate() final { \ 341 Label oob; \ 342 __ push(index1_); \ 343 __ lea(index1_, Operand(index1_, index2_)); \ 344 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \ 345 rmode_length_)); \ 346 __ j(above_equal, &oob, Label::kNear); \ 347 if (buffer_reg_.is_valid()) { \ 348 __ asm_instr(result_, Operand(buffer_reg_, index1_, times_1, 0)); \ 349 } else { \ 350 __ asm_instr(result_, \ 351 Operand(index1_, buffer_int_, rmode_buffer_)); \ 352 } \ 353 __ pop(index1_); \ 354 __ jmp(exit()); \ 355 __ bind(&oob); \ 356 __ pop(index1_); \ 357 __ xorp##SingleOrDouble(result_, result_); \ 358 __ divs##SingleOrDouble(result_, result_); \ 359 } \ 360 \ 361 private: \ 362 XMMRegister const result_; \ 363 Register const buffer_reg_; \ 364 int32_t const buffer_int_; \ 365 Register const index1_; \ 366 int32_t const index2_; \ 367 int32_t const length_; \ 368 RelocInfo::Mode rmode_length_; \ 369 RelocInfo::Mode rmode_buffer_; \ 370 }; \ 371 if (instr->InputAt(3)->IsRegister()) { \ 372 auto buffer = i.InputRegister(3); \ 373 OutOfLineCode* ool = new (zone()) \ 374 OutOfLineLoadFloat(this, result, buffer, index1, index2, length, \ 375 rmode_length, rmode_buffer); \ 376 __ j(above_equal, ool->entry()); \ 377 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \ 378 __ bind(ool->exit()); \ 379 } else { \ 380 auto buffer = i.InputInt32(3); \ 381 OutOfLineCode* ool = new (zone()) \ 382 OutOfLineLoadFloat(this, result, buffer, index1, index2, length, \ 383 rmode_length, rmode_buffer); \ 384 __ j(above_equal, ool->entry()); \ 385 __ asm_instr(result, Operand(index1, buffer + index2, rmode_buffer)); \ 386 __ bind(ool->exit()); \ 387 } \ 388 } \ 389 } while (false) 390 391 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \ 392 do { \ 393 auto result = i.OutputRegister(); \ 394 if (instr->InputAt(0)->IsRegister()) { \ 395 auto offset = i.InputRegister(0); \ 396 if (instr->InputAt(1)->IsRegister()) { \ 397 __ cmp(offset, i.InputRegister(1)); \ 398 } else { \ 399 __ cmp(offset, i.InputImmediate(1)); \ 400 } \ 401 OutOfLineCode* ool = new (zone()) OutOfLineLoadZero(this, result); \ 402 __ j(above_equal, ool->entry()); \ 403 __ asm_instr(result, i.MemoryOperand(2)); \ 404 __ bind(ool->exit()); \ 405 } else { \ 406 auto index2 = i.InputInt32(0); \ 407 auto length = i.InputInt32(1); \ 408 auto index1 = i.InputRegister(2); \ 409 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \ 410 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(3)).rmode(); \ 411 DCHECK_LE(index2, length); \ 412 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \ 413 rmode_length)); \ 414 class OutOfLineLoadInteger final : public OutOfLineCode { \ 415 public: \ 416 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \ 417 Register buffer, Register index1, int32_t index2, \ 418 int32_t length, RelocInfo::Mode rmode_length, \ 419 RelocInfo::Mode rmode_buffer) \ 420 : OutOfLineCode(gen), \ 421 result_(result), \ 422 buffer_reg_(buffer), \ 423 buffer_int_(0), \ 424 index1_(index1), \ 425 index2_(index2), \ 426 length_(length), \ 427 rmode_length_(rmode_length), \ 428 rmode_buffer_(rmode_buffer) {} \ 429 \ 430 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \ 431 int32_t buffer, Register index1, int32_t index2, \ 432 int32_t length, RelocInfo::Mode rmode_length, \ 433 RelocInfo::Mode rmode_buffer) \ 434 : OutOfLineCode(gen), \ 435 result_(result), \ 436 buffer_reg_({-1}), \ 437 buffer_int_(buffer), \ 438 index1_(index1), \ 439 index2_(index2), \ 440 length_(length), \ 441 rmode_length_(rmode_length), \ 442 rmode_buffer_(rmode_buffer) {} \ 443 \ 444 void Generate() final { \ 445 Label oob; \ 446 bool need_cache = !result_.is(index1_); \ 447 if (need_cache) __ push(index1_); \ 448 __ lea(index1_, Operand(index1_, index2_)); \ 449 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \ 450 rmode_length_)); \ 451 __ j(above_equal, &oob, Label::kNear); \ 452 if (buffer_reg_.is_valid()) { \ 453 __ asm_instr(result_, Operand(buffer_reg_, index1_, times_1, 0)); \ 454 } else { \ 455 __ asm_instr(result_, \ 456 Operand(index1_, buffer_int_, rmode_buffer_)); \ 457 } \ 458 if (need_cache) __ pop(index1_); \ 459 __ jmp(exit()); \ 460 __ bind(&oob); \ 461 if (need_cache) __ pop(index1_); \ 462 __ xor_(result_, result_); \ 463 } \ 464 \ 465 private: \ 466 Register const result_; \ 467 Register const buffer_reg_; \ 468 int32_t const buffer_int_; \ 469 Register const index1_; \ 470 int32_t const index2_; \ 471 int32_t const length_; \ 472 RelocInfo::Mode rmode_length_; \ 473 RelocInfo::Mode rmode_buffer_; \ 474 }; \ 475 if (instr->InputAt(3)->IsRegister()) { \ 476 auto buffer = i.InputRegister(3); \ 477 OutOfLineCode* ool = new (zone()) \ 478 OutOfLineLoadInteger(this, result, buffer, index1, index2, length, \ 479 rmode_length, rmode_buffer); \ 480 __ j(above_equal, ool->entry()); \ 481 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \ 482 __ bind(ool->exit()); \ 483 } else { \ 484 auto buffer = i.InputInt32(3); \ 485 OutOfLineCode* ool = new (zone()) \ 486 OutOfLineLoadInteger(this, result, buffer, index1, index2, length, \ 487 rmode_length, rmode_buffer); \ 488 __ j(above_equal, ool->entry()); \ 489 __ asm_instr(result, Operand(index1, buffer + index2, rmode_buffer)); \ 490 __ bind(ool->exit()); \ 491 } \ 492 } \ 493 } while (false) 494 495 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \ 496 do { \ 497 auto value = i.InputDoubleRegister(2); \ 498 if (instr->InputAt(0)->IsRegister()) { \ 499 auto offset = i.InputRegister(0); \ 500 if (instr->InputAt(1)->IsRegister()) { \ 501 __ cmp(offset, i.InputRegister(1)); \ 502 } else { \ 503 __ cmp(offset, i.InputImmediate(1)); \ 504 } \ 505 Label done; \ 506 __ j(above_equal, &done, Label::kNear); \ 507 __ asm_instr(i.MemoryOperand(3), value); \ 508 __ bind(&done); \ 509 } else { \ 510 auto index2 = i.InputInt32(0); \ 511 auto length = i.InputInt32(1); \ 512 auto index1 = i.InputRegister(3); \ 513 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \ 514 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(4)).rmode(); \ 515 DCHECK_LE(index2, length); \ 516 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \ 517 rmode_length)); \ 518 class OutOfLineStoreFloat final : public OutOfLineCode { \ 519 public: \ 520 OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \ 521 Register index1, int32_t index2, int32_t length, \ 522 XMMRegister value, RelocInfo::Mode rmode_length, \ 523 RelocInfo::Mode rmode_buffer) \ 524 : OutOfLineCode(gen), \ 525 buffer_reg_(buffer), \ 526 buffer_int_(0), \ 527 index1_(index1), \ 528 index2_(index2), \ 529 length_(length), \ 530 value_(value), \ 531 rmode_length_(rmode_length), \ 532 rmode_buffer_(rmode_buffer) {} \ 533 \ 534 OutOfLineStoreFloat(CodeGenerator* gen, int32_t buffer, \ 535 Register index1, int32_t index2, int32_t length, \ 536 XMMRegister value, RelocInfo::Mode rmode_length, \ 537 RelocInfo::Mode rmode_buffer) \ 538 : OutOfLineCode(gen), \ 539 buffer_reg_({-1}), \ 540 buffer_int_(buffer), \ 541 index1_(index1), \ 542 index2_(index2), \ 543 length_(length), \ 544 value_(value), \ 545 rmode_length_(rmode_length), \ 546 rmode_buffer_(rmode_buffer) {} \ 547 \ 548 void Generate() final { \ 549 Label oob; \ 550 __ push(index1_); \ 551 __ lea(index1_, Operand(index1_, index2_)); \ 552 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \ 553 rmode_length_)); \ 554 __ j(above_equal, &oob, Label::kNear); \ 555 if (buffer_reg_.is_valid()) { \ 556 __ asm_instr(Operand(buffer_reg_, index1_, times_1, 0), value_); \ 557 } else { \ 558 __ asm_instr(Operand(index1_, buffer_int_, rmode_buffer_), \ 559 value_); \ 560 } \ 561 __ bind(&oob); \ 562 __ pop(index1_); \ 563 } \ 564 \ 565 private: \ 566 Register const buffer_reg_; \ 567 int32_t const buffer_int_; \ 568 Register const index1_; \ 569 int32_t const index2_; \ 570 int32_t const length_; \ 571 XMMRegister const value_; \ 572 RelocInfo::Mode rmode_length_; \ 573 RelocInfo::Mode rmode_buffer_; \ 574 }; \ 575 if (instr->InputAt(4)->IsRegister()) { \ 576 auto buffer = i.InputRegister(4); \ 577 OutOfLineCode* ool = new (zone()) \ 578 OutOfLineStoreFloat(this, buffer, index1, index2, length, value, \ 579 rmode_length, rmode_buffer); \ 580 __ j(above_equal, ool->entry()); \ 581 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 582 __ bind(ool->exit()); \ 583 } else { \ 584 auto buffer = i.InputInt32(4); \ 585 OutOfLineCode* ool = new (zone()) \ 586 OutOfLineStoreFloat(this, buffer, index1, index2, length, value, \ 587 rmode_length, rmode_buffer); \ 588 __ j(above_equal, ool->entry()); \ 589 __ asm_instr(Operand(index1, buffer + index2, rmode_buffer), value); \ 590 __ bind(ool->exit()); \ 591 } \ 592 } \ 593 } while (false) 594 595 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \ 596 do { \ 597 if (instr->InputAt(0)->IsRegister()) { \ 598 auto offset = i.InputRegister(0); \ 599 if (instr->InputAt(1)->IsRegister()) { \ 600 __ cmp(offset, i.InputRegister(1)); \ 601 } else { \ 602 __ cmp(offset, i.InputImmediate(1)); \ 603 } \ 604 Label done; \ 605 __ j(above_equal, &done, Label::kNear); \ 606 __ asm_instr(i.MemoryOperand(3), value); \ 607 __ bind(&done); \ 608 } else { \ 609 auto index2 = i.InputInt32(0); \ 610 auto length = i.InputInt32(1); \ 611 auto index1 = i.InputRegister(3); \ 612 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \ 613 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(4)).rmode(); \ 614 DCHECK_LE(index2, length); \ 615 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \ 616 rmode_length)); \ 617 class OutOfLineStoreInteger final : public OutOfLineCode { \ 618 public: \ 619 OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \ 620 Register index1, int32_t index2, int32_t length, \ 621 Value value, RelocInfo::Mode rmode_length, \ 622 RelocInfo::Mode rmode_buffer) \ 623 : OutOfLineCode(gen), \ 624 buffer_reg_(buffer), \ 625 buffer_int_(0), \ 626 index1_(index1), \ 627 index2_(index2), \ 628 length_(length), \ 629 value_(value), \ 630 rmode_length_(rmode_length), \ 631 rmode_buffer_(rmode_buffer) {} \ 632 \ 633 OutOfLineStoreInteger(CodeGenerator* gen, int32_t buffer, \ 634 Register index1, int32_t index2, int32_t length, \ 635 Value value, RelocInfo::Mode rmode_length, \ 636 RelocInfo::Mode rmode_buffer) \ 637 : OutOfLineCode(gen), \ 638 buffer_reg_({-1}), \ 639 buffer_int_(buffer), \ 640 index1_(index1), \ 641 index2_(index2), \ 642 length_(length), \ 643 value_(value), \ 644 rmode_length_(rmode_length), \ 645 rmode_buffer_(rmode_buffer) {} \ 646 \ 647 void Generate() final { \ 648 Label oob; \ 649 __ push(index1_); \ 650 __ lea(index1_, Operand(index1_, index2_)); \ 651 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \ 652 rmode_length_)); \ 653 __ j(above_equal, &oob, Label::kNear); \ 654 if (buffer_reg_.is_valid()) { \ 655 __ asm_instr(Operand(buffer_reg_, index1_, times_1, 0), value_); \ 656 } else { \ 657 __ asm_instr(Operand(index1_, buffer_int_, rmode_buffer_), \ 658 value_); \ 659 } \ 660 __ bind(&oob); \ 661 __ pop(index1_); \ 662 } \ 663 \ 664 private: \ 665 Register const buffer_reg_; \ 666 int32_t const buffer_int_; \ 667 Register const index1_; \ 668 int32_t const index2_; \ 669 int32_t const length_; \ 670 Value const value_; \ 671 RelocInfo::Mode rmode_length_; \ 672 RelocInfo::Mode rmode_buffer_; \ 673 }; \ 674 if (instr->InputAt(4)->IsRegister()) { \ 675 auto buffer = i.InputRegister(4); \ 676 OutOfLineCode* ool = new (zone()) \ 677 OutOfLineStoreInteger(this, buffer, index1, index2, length, value, \ 678 rmode_length, rmode_buffer); \ 679 __ j(above_equal, ool->entry()); \ 680 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 681 __ bind(ool->exit()); \ 682 } else { \ 683 auto buffer = i.InputInt32(4); \ 684 OutOfLineCode* ool = new (zone()) \ 685 OutOfLineStoreInteger(this, buffer, index1, index2, length, value, \ 686 rmode_length, rmode_buffer); \ 687 __ j(above_equal, ool->entry()); \ 688 __ asm_instr(Operand(index1, buffer + index2, rmode_buffer), value); \ 689 __ bind(ool->exit()); \ 690 } \ 691 } \ 692 } while (false) 693 694 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \ 695 do { \ 696 if (instr->InputAt(2)->IsRegister()) { \ 697 Register value = i.InputRegister(2); \ 698 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \ 699 } else { \ 700 Immediate value = i.InputImmediate(2); \ 701 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \ 702 } \ 703 } while (false) 704 705 #define ASSEMBLE_COMPARE(asm_instr) \ 706 do { \ 707 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \ 708 size_t index = 0; \ 709 Operand left = i.MemoryOperand(&index); \ 710 if (HasImmediateInput(instr, index)) { \ 711 __ asm_instr(left, i.InputImmediate(index)); \ 712 } else { \ 713 __ asm_instr(left, i.InputRegister(index)); \ 714 } \ 715 } else { \ 716 if (HasImmediateInput(instr, 1)) { \ 717 if (instr->InputAt(0)->IsRegister()) { \ 718 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ 719 } else { \ 720 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ 721 } \ 722 } else { \ 723 if (instr->InputAt(1)->IsRegister()) { \ 724 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ 725 } else { \ 726 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ 727 } \ 728 } \ 729 } \ 730 } while (0) 731 732 #define ASSEMBLE_IEEE754_BINOP(name) \ 733 do { \ 734 /* Pass two doubles as arguments on the stack. */ \ 735 __ PrepareCallCFunction(4, eax); \ 736 __ movsd(Operand(esp, 0 * kDoubleSize), i.InputDoubleRegister(0)); \ 737 __ movsd(Operand(esp, 1 * kDoubleSize), i.InputDoubleRegister(1)); \ 738 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 739 4); \ 740 /* Return value is in st(0) on ia32. */ \ 741 /* Store it into the result register. */ \ 742 __ sub(esp, Immediate(kDoubleSize)); \ 743 __ fstp_d(Operand(esp, 0)); \ 744 __ movsd(i.OutputDoubleRegister(), Operand(esp, 0)); \ 745 __ add(esp, Immediate(kDoubleSize)); \ 746 } while (false) 747 748 #define ASSEMBLE_IEEE754_UNOP(name) \ 749 do { \ 750 /* Pass one double as argument on the stack. */ \ 751 __ PrepareCallCFunction(2, eax); \ 752 __ movsd(Operand(esp, 0 * kDoubleSize), i.InputDoubleRegister(0)); \ 753 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 754 2); \ 755 /* Return value is in st(0) on ia32. */ \ 756 /* Store it into the result register. */ \ 757 __ sub(esp, Immediate(kDoubleSize)); \ 758 __ fstp_d(Operand(esp, 0)); \ 759 __ movsd(i.OutputDoubleRegister(), Operand(esp, 0)); \ 760 __ add(esp, Immediate(kDoubleSize)); \ 761 } while (false) 762 763 void CodeGenerator::AssembleDeconstructFrame() { 764 __ mov(esp, ebp); 765 __ pop(ebp); 766 } 767 768 void CodeGenerator::AssemblePrepareTailCall() { 769 if (frame_access_state()->has_frame()) { 770 __ mov(ebp, MemOperand(ebp, 0)); 771 } 772 frame_access_state()->SetFrameAccessToSP(); 773 } 774 775 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg, 776 Register, Register, 777 Register) { 778 // There are not enough temp registers left on ia32 for a call instruction 779 // so we pick some scratch registers and save/restore them manually here. 780 int scratch_count = 3; 781 Register scratch1 = ebx; 782 Register scratch2 = ecx; 783 Register scratch3 = edx; 784 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3)); 785 Label done; 786 787 // Check if current frame is an arguments adaptor frame. 788 __ cmp(Operand(ebp, StandardFrameConstants::kContextOffset), 789 Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR))); 790 __ j(not_equal, &done, Label::kNear); 791 792 __ push(scratch1); 793 __ push(scratch2); 794 __ push(scratch3); 795 796 // Load arguments count from current arguments adaptor frame (note, it 797 // does not include receiver). 798 Register caller_args_count_reg = scratch1; 799 __ mov(caller_args_count_reg, 800 Operand(ebp, ArgumentsAdaptorFrameConstants::kLengthOffset)); 801 __ SmiUntag(caller_args_count_reg); 802 803 ParameterCount callee_args_count(args_reg); 804 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2, 805 scratch3, ReturnAddressState::kOnStack, scratch_count); 806 __ pop(scratch3); 807 __ pop(scratch2); 808 __ pop(scratch1); 809 810 __ bind(&done); 811 } 812 813 namespace { 814 815 void AdjustStackPointerForTailCall(MacroAssembler* masm, 816 FrameAccessState* state, 817 int new_slot_above_sp, 818 bool allow_shrinkage = true) { 819 int current_sp_offset = state->GetSPToFPSlotCount() + 820 StandardFrameConstants::kFixedSlotCountAboveFp; 821 int stack_slot_delta = new_slot_above_sp - current_sp_offset; 822 if (stack_slot_delta > 0) { 823 masm->sub(esp, Immediate(stack_slot_delta * kPointerSize)); 824 state->IncreaseSPDelta(stack_slot_delta); 825 } else if (allow_shrinkage && stack_slot_delta < 0) { 826 masm->add(esp, Immediate(-stack_slot_delta * kPointerSize)); 827 state->IncreaseSPDelta(stack_slot_delta); 828 } 829 } 830 831 } // namespace 832 833 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, 834 int first_unused_stack_slot) { 835 CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush); 836 ZoneVector<MoveOperands*> pushes(zone()); 837 GetPushCompatibleMoves(instr, flags, &pushes); 838 839 if (!pushes.empty() && 840 (LocationOperand::cast(pushes.back()->destination()).index() + 1 == 841 first_unused_stack_slot)) { 842 IA32OperandConverter g(this, instr); 843 for (auto move : pushes) { 844 LocationOperand destination_location( 845 LocationOperand::cast(move->destination())); 846 InstructionOperand source(move->source()); 847 AdjustStackPointerForTailCall(masm(), frame_access_state(), 848 destination_location.index()); 849 if (source.IsStackSlot()) { 850 LocationOperand source_location(LocationOperand::cast(source)); 851 __ push(g.SlotToOperand(source_location.index())); 852 } else if (source.IsRegister()) { 853 LocationOperand source_location(LocationOperand::cast(source)); 854 __ push(source_location.GetRegister()); 855 } else if (source.IsImmediate()) { 856 __ push(Immediate(ImmediateOperand::cast(source).inline_value())); 857 } else { 858 // Pushes of non-scalar data types is not supported. 859 UNIMPLEMENTED(); 860 } 861 frame_access_state()->IncreaseSPDelta(1); 862 move->Eliminate(); 863 } 864 } 865 AdjustStackPointerForTailCall(masm(), frame_access_state(), 866 first_unused_stack_slot, false); 867 } 868 869 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr, 870 int first_unused_stack_slot) { 871 AdjustStackPointerForTailCall(masm(), frame_access_state(), 872 first_unused_stack_slot); 873 } 874 875 // Assembles an instruction after register allocation, producing machine code. 876 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( 877 Instruction* instr) { 878 IA32OperandConverter i(this, instr); 879 InstructionCode opcode = instr->opcode(); 880 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode); 881 switch (arch_opcode) { 882 case kArchCallCodeObject: { 883 EnsureSpaceForLazyDeopt(); 884 if (HasImmediateInput(instr, 0)) { 885 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0)); 886 __ call(code, RelocInfo::CODE_TARGET); 887 } else { 888 Register reg = i.InputRegister(0); 889 __ add(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); 890 __ call(reg); 891 } 892 RecordCallPosition(instr); 893 frame_access_state()->ClearSPDelta(); 894 break; 895 } 896 case kArchTailCallCodeObjectFromJSFunction: 897 case kArchTailCallCodeObject: { 898 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) { 899 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 900 no_reg, no_reg, no_reg); 901 } 902 if (HasImmediateInput(instr, 0)) { 903 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0)); 904 __ jmp(code, RelocInfo::CODE_TARGET); 905 } else { 906 Register reg = i.InputRegister(0); 907 __ add(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); 908 __ jmp(reg); 909 } 910 frame_access_state()->ClearSPDelta(); 911 frame_access_state()->SetFrameAccessToDefault(); 912 break; 913 } 914 case kArchTailCallAddress: { 915 CHECK(!HasImmediateInput(instr, 0)); 916 Register reg = i.InputRegister(0); 917 __ jmp(reg); 918 frame_access_state()->ClearSPDelta(); 919 frame_access_state()->SetFrameAccessToDefault(); 920 break; 921 } 922 case kArchCallJSFunction: { 923 EnsureSpaceForLazyDeopt(); 924 Register func = i.InputRegister(0); 925 if (FLAG_debug_code) { 926 // Check the function's context matches the context argument. 927 __ cmp(esi, FieldOperand(func, JSFunction::kContextOffset)); 928 __ Assert(equal, kWrongFunctionContext); 929 } 930 __ call(FieldOperand(func, JSFunction::kCodeEntryOffset)); 931 RecordCallPosition(instr); 932 frame_access_state()->ClearSPDelta(); 933 break; 934 } 935 case kArchTailCallJSFunctionFromJSFunction: { 936 Register func = i.InputRegister(0); 937 if (FLAG_debug_code) { 938 // Check the function's context matches the context argument. 939 __ cmp(esi, FieldOperand(func, JSFunction::kContextOffset)); 940 __ Assert(equal, kWrongFunctionContext); 941 } 942 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, no_reg, 943 no_reg, no_reg); 944 __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset)); 945 frame_access_state()->ClearSPDelta(); 946 frame_access_state()->SetFrameAccessToDefault(); 947 break; 948 } 949 case kArchPrepareCallCFunction: { 950 // Frame alignment requires using FP-relative frame addressing. 951 frame_access_state()->SetFrameAccessToFP(); 952 int const num_parameters = MiscField::decode(instr->opcode()); 953 __ PrepareCallCFunction(num_parameters, i.TempRegister(0)); 954 break; 955 } 956 case kArchPrepareTailCall: 957 AssemblePrepareTailCall(); 958 break; 959 case kArchCallCFunction: { 960 int const num_parameters = MiscField::decode(instr->opcode()); 961 if (HasImmediateInput(instr, 0)) { 962 ExternalReference ref = i.InputExternalReference(0); 963 __ CallCFunction(ref, num_parameters); 964 } else { 965 Register func = i.InputRegister(0); 966 __ CallCFunction(func, num_parameters); 967 } 968 frame_access_state()->SetFrameAccessToDefault(); 969 frame_access_state()->ClearSPDelta(); 970 break; 971 } 972 case kArchJmp: 973 AssembleArchJump(i.InputRpo(0)); 974 break; 975 case kArchLookupSwitch: 976 AssembleArchLookupSwitch(instr); 977 break; 978 case kArchTableSwitch: 979 AssembleArchTableSwitch(instr); 980 break; 981 case kArchComment: { 982 Address comment_string = i.InputExternalReference(0).address(); 983 __ RecordComment(reinterpret_cast<const char*>(comment_string)); 984 break; 985 } 986 case kArchDebugBreak: 987 __ int3(); 988 break; 989 case kArchNop: 990 case kArchThrowTerminator: 991 // don't emit code for nops. 992 break; 993 case kArchDeoptimize: { 994 int deopt_state_id = 995 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); 996 CodeGenResult result = 997 AssembleDeoptimizerCall(deopt_state_id, current_source_position_); 998 if (result != kSuccess) return result; 999 break; 1000 } 1001 case kArchRet: 1002 AssembleReturn(instr->InputAt(0)); 1003 break; 1004 case kArchStackPointer: 1005 __ mov(i.OutputRegister(), esp); 1006 break; 1007 case kArchFramePointer: 1008 __ mov(i.OutputRegister(), ebp); 1009 break; 1010 case kArchParentFramePointer: 1011 if (frame_access_state()->has_frame()) { 1012 __ mov(i.OutputRegister(), Operand(ebp, 0)); 1013 } else { 1014 __ mov(i.OutputRegister(), ebp); 1015 } 1016 break; 1017 case kArchTruncateDoubleToI: { 1018 auto result = i.OutputRegister(); 1019 auto input = i.InputDoubleRegister(0); 1020 auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input); 1021 __ cvttsd2si(result, Operand(input)); 1022 __ cmp(result, 1); 1023 __ j(overflow, ool->entry()); 1024 __ bind(ool->exit()); 1025 break; 1026 } 1027 case kArchStoreWithWriteBarrier: { 1028 RecordWriteMode mode = 1029 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode())); 1030 Register object = i.InputRegister(0); 1031 size_t index = 0; 1032 Operand operand = i.MemoryOperand(&index); 1033 Register value = i.InputRegister(index); 1034 Register scratch0 = i.TempRegister(0); 1035 Register scratch1 = i.TempRegister(1); 1036 auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value, 1037 scratch0, scratch1, mode); 1038 __ mov(operand, value); 1039 __ CheckPageFlag(object, scratch0, 1040 MemoryChunk::kPointersFromHereAreInterestingMask, 1041 not_zero, ool->entry()); 1042 __ bind(ool->exit()); 1043 break; 1044 } 1045 case kArchStackSlot: { 1046 FrameOffset offset = 1047 frame_access_state()->GetFrameOffset(i.InputInt32(0)); 1048 Register base; 1049 if (offset.from_stack_pointer()) { 1050 base = esp; 1051 } else { 1052 base = ebp; 1053 } 1054 __ lea(i.OutputRegister(), Operand(base, offset.offset())); 1055 break; 1056 } 1057 case kIeee754Float64Acos: 1058 ASSEMBLE_IEEE754_UNOP(acos); 1059 break; 1060 case kIeee754Float64Acosh: 1061 ASSEMBLE_IEEE754_UNOP(acosh); 1062 break; 1063 case kIeee754Float64Asin: 1064 ASSEMBLE_IEEE754_UNOP(asin); 1065 break; 1066 case kIeee754Float64Asinh: 1067 ASSEMBLE_IEEE754_UNOP(asinh); 1068 break; 1069 case kIeee754Float64Atan: 1070 ASSEMBLE_IEEE754_UNOP(atan); 1071 break; 1072 case kIeee754Float64Atanh: 1073 ASSEMBLE_IEEE754_UNOP(atanh); 1074 break; 1075 case kIeee754Float64Atan2: 1076 ASSEMBLE_IEEE754_BINOP(atan2); 1077 break; 1078 case kIeee754Float64Cbrt: 1079 ASSEMBLE_IEEE754_UNOP(cbrt); 1080 break; 1081 case kIeee754Float64Cos: 1082 ASSEMBLE_IEEE754_UNOP(cos); 1083 break; 1084 case kIeee754Float64Cosh: 1085 ASSEMBLE_IEEE754_UNOP(cosh); 1086 break; 1087 case kIeee754Float64Expm1: 1088 ASSEMBLE_IEEE754_UNOP(expm1); 1089 break; 1090 case kIeee754Float64Exp: 1091 ASSEMBLE_IEEE754_UNOP(exp); 1092 break; 1093 case kIeee754Float64Log: 1094 ASSEMBLE_IEEE754_UNOP(log); 1095 break; 1096 case kIeee754Float64Log1p: 1097 ASSEMBLE_IEEE754_UNOP(log1p); 1098 break; 1099 case kIeee754Float64Log2: 1100 ASSEMBLE_IEEE754_UNOP(log2); 1101 break; 1102 case kIeee754Float64Log10: 1103 ASSEMBLE_IEEE754_UNOP(log10); 1104 break; 1105 case kIeee754Float64Pow: { 1106 // TODO(bmeurer): Improve integration of the stub. 1107 if (!i.InputDoubleRegister(1).is(xmm2)) { 1108 __ movaps(xmm2, i.InputDoubleRegister(0)); 1109 __ movaps(xmm1, i.InputDoubleRegister(1)); 1110 } else { 1111 __ movaps(xmm0, i.InputDoubleRegister(0)); 1112 __ movaps(xmm1, xmm2); 1113 __ movaps(xmm2, xmm0); 1114 } 1115 MathPowStub stub(isolate(), MathPowStub::DOUBLE); 1116 __ CallStub(&stub); 1117 __ movaps(i.OutputDoubleRegister(), xmm3); 1118 break; 1119 } 1120 case kIeee754Float64Sin: 1121 ASSEMBLE_IEEE754_UNOP(sin); 1122 break; 1123 case kIeee754Float64Sinh: 1124 ASSEMBLE_IEEE754_UNOP(sinh); 1125 break; 1126 case kIeee754Float64Tan: 1127 ASSEMBLE_IEEE754_UNOP(tan); 1128 break; 1129 case kIeee754Float64Tanh: 1130 ASSEMBLE_IEEE754_UNOP(tanh); 1131 break; 1132 case kIA32Add: 1133 if (HasImmediateInput(instr, 1)) { 1134 __ add(i.InputOperand(0), i.InputImmediate(1)); 1135 } else { 1136 __ add(i.InputRegister(0), i.InputOperand(1)); 1137 } 1138 break; 1139 case kIA32And: 1140 if (HasImmediateInput(instr, 1)) { 1141 __ and_(i.InputOperand(0), i.InputImmediate(1)); 1142 } else { 1143 __ and_(i.InputRegister(0), i.InputOperand(1)); 1144 } 1145 break; 1146 case kIA32Cmp: 1147 ASSEMBLE_COMPARE(cmp); 1148 break; 1149 case kIA32Cmp16: 1150 ASSEMBLE_COMPARE(cmpw); 1151 break; 1152 case kIA32Cmp8: 1153 ASSEMBLE_COMPARE(cmpb); 1154 break; 1155 case kIA32Test: 1156 ASSEMBLE_COMPARE(test); 1157 break; 1158 case kIA32Test16: 1159 ASSEMBLE_COMPARE(test_w); 1160 break; 1161 case kIA32Test8: 1162 ASSEMBLE_COMPARE(test_b); 1163 break; 1164 case kIA32Imul: 1165 if (HasImmediateInput(instr, 1)) { 1166 __ imul(i.OutputRegister(), i.InputOperand(0), i.InputInt32(1)); 1167 } else { 1168 __ imul(i.OutputRegister(), i.InputOperand(1)); 1169 } 1170 break; 1171 case kIA32ImulHigh: 1172 __ imul(i.InputRegister(1)); 1173 break; 1174 case kIA32UmulHigh: 1175 __ mul(i.InputRegister(1)); 1176 break; 1177 case kIA32Idiv: 1178 __ cdq(); 1179 __ idiv(i.InputOperand(1)); 1180 break; 1181 case kIA32Udiv: 1182 __ Move(edx, Immediate(0)); 1183 __ div(i.InputOperand(1)); 1184 break; 1185 case kIA32Not: 1186 __ not_(i.OutputOperand()); 1187 break; 1188 case kIA32Neg: 1189 __ neg(i.OutputOperand()); 1190 break; 1191 case kIA32Or: 1192 if (HasImmediateInput(instr, 1)) { 1193 __ or_(i.InputOperand(0), i.InputImmediate(1)); 1194 } else { 1195 __ or_(i.InputRegister(0), i.InputOperand(1)); 1196 } 1197 break; 1198 case kIA32Xor: 1199 if (HasImmediateInput(instr, 1)) { 1200 __ xor_(i.InputOperand(0), i.InputImmediate(1)); 1201 } else { 1202 __ xor_(i.InputRegister(0), i.InputOperand(1)); 1203 } 1204 break; 1205 case kIA32Sub: 1206 if (HasImmediateInput(instr, 1)) { 1207 __ sub(i.InputOperand(0), i.InputImmediate(1)); 1208 } else { 1209 __ sub(i.InputRegister(0), i.InputOperand(1)); 1210 } 1211 break; 1212 case kIA32Shl: 1213 if (HasImmediateInput(instr, 1)) { 1214 __ shl(i.OutputOperand(), i.InputInt5(1)); 1215 } else { 1216 __ shl_cl(i.OutputOperand()); 1217 } 1218 break; 1219 case kIA32Shr: 1220 if (HasImmediateInput(instr, 1)) { 1221 __ shr(i.OutputOperand(), i.InputInt5(1)); 1222 } else { 1223 __ shr_cl(i.OutputOperand()); 1224 } 1225 break; 1226 case kIA32Sar: 1227 if (HasImmediateInput(instr, 1)) { 1228 __ sar(i.OutputOperand(), i.InputInt5(1)); 1229 } else { 1230 __ sar_cl(i.OutputOperand()); 1231 } 1232 break; 1233 case kIA32AddPair: { 1234 // i.OutputRegister(0) == i.InputRegister(0) ... left low word. 1235 // i.InputRegister(1) ... left high word. 1236 // i.InputRegister(2) ... right low word. 1237 // i.InputRegister(3) ... right high word. 1238 bool use_temp = false; 1239 if (i.OutputRegister(0).code() == i.InputRegister(1).code() || 1240 i.OutputRegister(0).code() == i.InputRegister(3).code()) { 1241 // We cannot write to the output register directly, because it would 1242 // overwrite an input for adc. We have to use the temp register. 1243 use_temp = true; 1244 __ Move(i.TempRegister(0), i.InputRegister(0)); 1245 __ add(i.TempRegister(0), i.InputRegister(2)); 1246 } else { 1247 __ add(i.OutputRegister(0), i.InputRegister(2)); 1248 } 1249 if (i.OutputRegister(1).code() != i.InputRegister(1).code()) { 1250 __ Move(i.OutputRegister(1), i.InputRegister(1)); 1251 } 1252 __ adc(i.OutputRegister(1), Operand(i.InputRegister(3))); 1253 if (use_temp) { 1254 __ Move(i.OutputRegister(0), i.TempRegister(0)); 1255 } 1256 break; 1257 } 1258 case kIA32SubPair: { 1259 // i.OutputRegister(0) == i.InputRegister(0) ... left low word. 1260 // i.InputRegister(1) ... left high word. 1261 // i.InputRegister(2) ... right low word. 1262 // i.InputRegister(3) ... right high word. 1263 bool use_temp = false; 1264 if (i.OutputRegister(0).code() == i.InputRegister(1).code() || 1265 i.OutputRegister(0).code() == i.InputRegister(3).code()) { 1266 // We cannot write to the output register directly, because it would 1267 // overwrite an input for adc. We have to use the temp register. 1268 use_temp = true; 1269 __ Move(i.TempRegister(0), i.InputRegister(0)); 1270 __ sub(i.TempRegister(0), i.InputRegister(2)); 1271 } else { 1272 __ sub(i.OutputRegister(0), i.InputRegister(2)); 1273 } 1274 if (i.OutputRegister(1).code() != i.InputRegister(1).code()) { 1275 __ Move(i.OutputRegister(1), i.InputRegister(1)); 1276 } 1277 __ sbb(i.OutputRegister(1), Operand(i.InputRegister(3))); 1278 if (use_temp) { 1279 __ Move(i.OutputRegister(0), i.TempRegister(0)); 1280 } 1281 break; 1282 } 1283 case kIA32MulPair: { 1284 __ imul(i.OutputRegister(1), i.InputOperand(0)); 1285 __ mov(i.TempRegister(0), i.InputOperand(1)); 1286 __ imul(i.TempRegister(0), i.InputOperand(2)); 1287 __ add(i.OutputRegister(1), i.TempRegister(0)); 1288 __ mov(i.OutputRegister(0), i.InputOperand(0)); 1289 // Multiplies the low words and stores them in eax and edx. 1290 __ mul(i.InputRegister(2)); 1291 __ add(i.OutputRegister(1), i.TempRegister(0)); 1292 1293 break; 1294 } 1295 case kIA32ShlPair: 1296 if (HasImmediateInput(instr, 2)) { 1297 __ ShlPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2)); 1298 } else { 1299 // Shift has been loaded into CL by the register allocator. 1300 __ ShlPair_cl(i.InputRegister(1), i.InputRegister(0)); 1301 } 1302 break; 1303 case kIA32ShrPair: 1304 if (HasImmediateInput(instr, 2)) { 1305 __ ShrPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2)); 1306 } else { 1307 // Shift has been loaded into CL by the register allocator. 1308 __ ShrPair_cl(i.InputRegister(1), i.InputRegister(0)); 1309 } 1310 break; 1311 case kIA32SarPair: 1312 if (HasImmediateInput(instr, 2)) { 1313 __ SarPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2)); 1314 } else { 1315 // Shift has been loaded into CL by the register allocator. 1316 __ SarPair_cl(i.InputRegister(1), i.InputRegister(0)); 1317 } 1318 break; 1319 case kIA32Ror: 1320 if (HasImmediateInput(instr, 1)) { 1321 __ ror(i.OutputOperand(), i.InputInt5(1)); 1322 } else { 1323 __ ror_cl(i.OutputOperand()); 1324 } 1325 break; 1326 case kIA32Lzcnt: 1327 __ Lzcnt(i.OutputRegister(), i.InputOperand(0)); 1328 break; 1329 case kIA32Tzcnt: 1330 __ Tzcnt(i.OutputRegister(), i.InputOperand(0)); 1331 break; 1332 case kIA32Popcnt: 1333 __ Popcnt(i.OutputRegister(), i.InputOperand(0)); 1334 break; 1335 case kSSEFloat32Cmp: 1336 __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1337 break; 1338 case kSSEFloat32Add: 1339 __ addss(i.InputDoubleRegister(0), i.InputOperand(1)); 1340 break; 1341 case kSSEFloat32Sub: 1342 __ subss(i.InputDoubleRegister(0), i.InputOperand(1)); 1343 break; 1344 case kSSEFloat32Mul: 1345 __ mulss(i.InputDoubleRegister(0), i.InputOperand(1)); 1346 break; 1347 case kSSEFloat32Div: 1348 __ divss(i.InputDoubleRegister(0), i.InputOperand(1)); 1349 // Don't delete this mov. It may improve performance on some CPUs, 1350 // when there is a (v)mulss depending on the result. 1351 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1352 break; 1353 case kSSEFloat32Sqrt: 1354 __ sqrtss(i.OutputDoubleRegister(), i.InputOperand(0)); 1355 break; 1356 case kSSEFloat32Abs: { 1357 // TODO(bmeurer): Use 128-bit constants. 1358 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1359 __ psrlq(kScratchDoubleReg, 33); 1360 __ andps(i.OutputDoubleRegister(), kScratchDoubleReg); 1361 break; 1362 } 1363 case kSSEFloat32Neg: { 1364 // TODO(bmeurer): Use 128-bit constants. 1365 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1366 __ psllq(kScratchDoubleReg, 31); 1367 __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg); 1368 break; 1369 } 1370 case kSSEFloat32Round: { 1371 CpuFeatureScope sse_scope(masm(), SSE4_1); 1372 RoundingMode const mode = 1373 static_cast<RoundingMode>(MiscField::decode(instr->opcode())); 1374 __ roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); 1375 break; 1376 } 1377 case kSSEFloat64Cmp: 1378 __ ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1379 break; 1380 case kSSEFloat64Add: 1381 __ addsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1382 break; 1383 case kSSEFloat64Sub: 1384 __ subsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1385 break; 1386 case kSSEFloat64Mul: 1387 __ mulsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1388 break; 1389 case kSSEFloat64Div: 1390 __ divsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1391 // Don't delete this mov. It may improve performance on some CPUs, 1392 // when there is a (v)mulsd depending on the result. 1393 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1394 break; 1395 case kSSEFloat32Max: { 1396 Label compare_nan, compare_swap, done_compare; 1397 if (instr->InputAt(1)->IsFPRegister()) { 1398 __ ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1399 } else { 1400 __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1401 } 1402 auto ool = 1403 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); 1404 __ j(parity_even, ool->entry()); 1405 __ j(above, &done_compare, Label::kNear); 1406 __ j(below, &compare_swap, Label::kNear); 1407 __ movmskps(i.TempRegister(0), i.InputDoubleRegister(0)); 1408 __ test(i.TempRegister(0), Immediate(1)); 1409 __ j(zero, &done_compare, Label::kNear); 1410 __ bind(&compare_swap); 1411 if (instr->InputAt(1)->IsFPRegister()) { 1412 __ movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1413 } else { 1414 __ movss(i.InputDoubleRegister(0), i.InputOperand(1)); 1415 } 1416 __ bind(&done_compare); 1417 __ bind(ool->exit()); 1418 break; 1419 } 1420 1421 case kSSEFloat64Max: { 1422 Label compare_nan, compare_swap, done_compare; 1423 if (instr->InputAt(1)->IsFPRegister()) { 1424 __ ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1425 } else { 1426 __ ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1427 } 1428 auto ool = 1429 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); 1430 __ j(parity_even, ool->entry()); 1431 __ j(above, &done_compare, Label::kNear); 1432 __ j(below, &compare_swap, Label::kNear); 1433 __ movmskpd(i.TempRegister(0), i.InputDoubleRegister(0)); 1434 __ test(i.TempRegister(0), Immediate(1)); 1435 __ j(zero, &done_compare, Label::kNear); 1436 __ bind(&compare_swap); 1437 if (instr->InputAt(1)->IsFPRegister()) { 1438 __ movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1439 } else { 1440 __ movsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1441 } 1442 __ bind(&done_compare); 1443 __ bind(ool->exit()); 1444 break; 1445 } 1446 case kSSEFloat32Min: { 1447 Label compare_swap, done_compare; 1448 if (instr->InputAt(1)->IsFPRegister()) { 1449 __ ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1450 } else { 1451 __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1452 } 1453 auto ool = 1454 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); 1455 __ j(parity_even, ool->entry()); 1456 __ j(below, &done_compare, Label::kNear); 1457 __ j(above, &compare_swap, Label::kNear); 1458 if (instr->InputAt(1)->IsFPRegister()) { 1459 __ movmskps(i.TempRegister(0), i.InputDoubleRegister(1)); 1460 } else { 1461 __ movss(kScratchDoubleReg, i.InputOperand(1)); 1462 __ movmskps(i.TempRegister(0), kScratchDoubleReg); 1463 } 1464 __ test(i.TempRegister(0), Immediate(1)); 1465 __ j(zero, &done_compare, Label::kNear); 1466 __ bind(&compare_swap); 1467 if (instr->InputAt(1)->IsFPRegister()) { 1468 __ movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1469 } else { 1470 __ movss(i.InputDoubleRegister(0), i.InputOperand(1)); 1471 } 1472 __ bind(&done_compare); 1473 __ bind(ool->exit()); 1474 break; 1475 } 1476 case kSSEFloat64Min: { 1477 Label compare_swap, done_compare; 1478 if (instr->InputAt(1)->IsFPRegister()) { 1479 __ ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1480 } else { 1481 __ ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1482 } 1483 auto ool = 1484 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); 1485 __ j(parity_even, ool->entry()); 1486 __ j(below, &done_compare, Label::kNear); 1487 __ j(above, &compare_swap, Label::kNear); 1488 if (instr->InputAt(1)->IsFPRegister()) { 1489 __ movmskpd(i.TempRegister(0), i.InputDoubleRegister(1)); 1490 } else { 1491 __ movsd(kScratchDoubleReg, i.InputOperand(1)); 1492 __ movmskpd(i.TempRegister(0), kScratchDoubleReg); 1493 } 1494 __ test(i.TempRegister(0), Immediate(1)); 1495 __ j(zero, &done_compare, Label::kNear); 1496 __ bind(&compare_swap); 1497 if (instr->InputAt(1)->IsFPRegister()) { 1498 __ movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1499 } else { 1500 __ movsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1501 } 1502 __ bind(&done_compare); 1503 __ bind(ool->exit()); 1504 break; 1505 } 1506 case kSSEFloat64Mod: { 1507 // TODO(dcarney): alignment is wrong. 1508 __ sub(esp, Immediate(kDoubleSize)); 1509 // Move values to st(0) and st(1). 1510 __ movsd(Operand(esp, 0), i.InputDoubleRegister(1)); 1511 __ fld_d(Operand(esp, 0)); 1512 __ movsd(Operand(esp, 0), i.InputDoubleRegister(0)); 1513 __ fld_d(Operand(esp, 0)); 1514 // Loop while fprem isn't done. 1515 Label mod_loop; 1516 __ bind(&mod_loop); 1517 // This instructions traps on all kinds inputs, but we are assuming the 1518 // floating point control word is set to ignore them all. 1519 __ fprem(); 1520 // The following 2 instruction implicitly use eax. 1521 __ fnstsw_ax(); 1522 __ sahf(); 1523 __ j(parity_even, &mod_loop); 1524 // Move output to stack and clean up. 1525 __ fstp(1); 1526 __ fstp_d(Operand(esp, 0)); 1527 __ movsd(i.OutputDoubleRegister(), Operand(esp, 0)); 1528 __ add(esp, Immediate(kDoubleSize)); 1529 break; 1530 } 1531 case kSSEFloat64Abs: { 1532 // TODO(bmeurer): Use 128-bit constants. 1533 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1534 __ psrlq(kScratchDoubleReg, 1); 1535 __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg); 1536 break; 1537 } 1538 case kSSEFloat64Neg: { 1539 // TODO(bmeurer): Use 128-bit constants. 1540 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1541 __ psllq(kScratchDoubleReg, 63); 1542 __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg); 1543 break; 1544 } 1545 case kSSEFloat64Sqrt: 1546 __ sqrtsd(i.OutputDoubleRegister(), i.InputOperand(0)); 1547 break; 1548 case kSSEFloat64Round: { 1549 CpuFeatureScope sse_scope(masm(), SSE4_1); 1550 RoundingMode const mode = 1551 static_cast<RoundingMode>(MiscField::decode(instr->opcode())); 1552 __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); 1553 break; 1554 } 1555 case kSSEFloat32ToFloat64: 1556 __ cvtss2sd(i.OutputDoubleRegister(), i.InputOperand(0)); 1557 break; 1558 case kSSEFloat64ToFloat32: 1559 __ cvtsd2ss(i.OutputDoubleRegister(), i.InputOperand(0)); 1560 break; 1561 case kSSEFloat32ToInt32: 1562 __ cvttss2si(i.OutputRegister(), i.InputOperand(0)); 1563 break; 1564 case kSSEFloat32ToUint32: { 1565 Label success; 1566 __ cvttss2si(i.OutputRegister(), i.InputOperand(0)); 1567 __ test(i.OutputRegister(), i.OutputRegister()); 1568 __ j(positive, &success); 1569 __ Move(kScratchDoubleReg, static_cast<float>(INT32_MIN)); 1570 __ addss(kScratchDoubleReg, i.InputOperand(0)); 1571 __ cvttss2si(i.OutputRegister(), kScratchDoubleReg); 1572 __ or_(i.OutputRegister(), Immediate(0x80000000)); 1573 __ bind(&success); 1574 break; 1575 } 1576 case kSSEFloat64ToInt32: 1577 __ cvttsd2si(i.OutputRegister(), i.InputOperand(0)); 1578 break; 1579 case kSSEFloat64ToUint32: { 1580 __ Move(kScratchDoubleReg, -2147483648.0); 1581 __ addsd(kScratchDoubleReg, i.InputOperand(0)); 1582 __ cvttsd2si(i.OutputRegister(), kScratchDoubleReg); 1583 __ add(i.OutputRegister(), Immediate(0x80000000)); 1584 break; 1585 } 1586 case kSSEInt32ToFloat32: 1587 __ cvtsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); 1588 break; 1589 case kSSEUint32ToFloat32: { 1590 Register scratch0 = i.TempRegister(0); 1591 Register scratch1 = i.TempRegister(1); 1592 __ mov(scratch0, i.InputOperand(0)); 1593 __ Cvtui2ss(i.OutputDoubleRegister(), scratch0, scratch1); 1594 break; 1595 } 1596 case kSSEInt32ToFloat64: 1597 __ cvtsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); 1598 break; 1599 case kSSEUint32ToFloat64: 1600 __ LoadUint32(i.OutputDoubleRegister(), i.InputOperand(0)); 1601 break; 1602 case kSSEFloat64ExtractLowWord32: 1603 if (instr->InputAt(0)->IsFPStackSlot()) { 1604 __ mov(i.OutputRegister(), i.InputOperand(0)); 1605 } else { 1606 __ movd(i.OutputRegister(), i.InputDoubleRegister(0)); 1607 } 1608 break; 1609 case kSSEFloat64ExtractHighWord32: 1610 if (instr->InputAt(0)->IsFPStackSlot()) { 1611 __ mov(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2)); 1612 } else { 1613 __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1); 1614 } 1615 break; 1616 case kSSEFloat64InsertLowWord32: 1617 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0); 1618 break; 1619 case kSSEFloat64InsertHighWord32: 1620 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1); 1621 break; 1622 case kSSEFloat64LoadLowWord32: 1623 __ movd(i.OutputDoubleRegister(), i.InputOperand(0)); 1624 break; 1625 case kAVXFloat32Add: { 1626 CpuFeatureScope avx_scope(masm(), AVX); 1627 __ vaddss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1628 i.InputOperand(1)); 1629 break; 1630 } 1631 case kAVXFloat32Sub: { 1632 CpuFeatureScope avx_scope(masm(), AVX); 1633 __ vsubss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1634 i.InputOperand(1)); 1635 break; 1636 } 1637 case kAVXFloat32Mul: { 1638 CpuFeatureScope avx_scope(masm(), AVX); 1639 __ vmulss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1640 i.InputOperand(1)); 1641 break; 1642 } 1643 case kAVXFloat32Div: { 1644 CpuFeatureScope avx_scope(masm(), AVX); 1645 __ vdivss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1646 i.InputOperand(1)); 1647 // Don't delete this mov. It may improve performance on some CPUs, 1648 // when there is a (v)mulss depending on the result. 1649 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1650 break; 1651 } 1652 case kAVXFloat64Add: { 1653 CpuFeatureScope avx_scope(masm(), AVX); 1654 __ vaddsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1655 i.InputOperand(1)); 1656 break; 1657 } 1658 case kAVXFloat64Sub: { 1659 CpuFeatureScope avx_scope(masm(), AVX); 1660 __ vsubsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1661 i.InputOperand(1)); 1662 break; 1663 } 1664 case kAVXFloat64Mul: { 1665 CpuFeatureScope avx_scope(masm(), AVX); 1666 __ vmulsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1667 i.InputOperand(1)); 1668 break; 1669 } 1670 case kAVXFloat64Div: { 1671 CpuFeatureScope avx_scope(masm(), AVX); 1672 __ vdivsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), 1673 i.InputOperand(1)); 1674 // Don't delete this mov. It may improve performance on some CPUs, 1675 // when there is a (v)mulsd depending on the result. 1676 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1677 break; 1678 } 1679 case kAVXFloat32Abs: { 1680 // TODO(bmeurer): Use RIP relative 128-bit constants. 1681 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1682 __ psrlq(kScratchDoubleReg, 33); 1683 CpuFeatureScope avx_scope(masm(), AVX); 1684 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); 1685 break; 1686 } 1687 case kAVXFloat32Neg: { 1688 // TODO(bmeurer): Use RIP relative 128-bit constants. 1689 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1690 __ psllq(kScratchDoubleReg, 31); 1691 CpuFeatureScope avx_scope(masm(), AVX); 1692 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); 1693 break; 1694 } 1695 case kAVXFloat64Abs: { 1696 // TODO(bmeurer): Use RIP relative 128-bit constants. 1697 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1698 __ psrlq(kScratchDoubleReg, 1); 1699 CpuFeatureScope avx_scope(masm(), AVX); 1700 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); 1701 break; 1702 } 1703 case kAVXFloat64Neg: { 1704 // TODO(bmeurer): Use RIP relative 128-bit constants. 1705 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1706 __ psllq(kScratchDoubleReg, 63); 1707 CpuFeatureScope avx_scope(masm(), AVX); 1708 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); 1709 break; 1710 } 1711 case kSSEFloat64SilenceNaN: 1712 __ xorpd(kScratchDoubleReg, kScratchDoubleReg); 1713 __ subsd(i.InputDoubleRegister(0), kScratchDoubleReg); 1714 break; 1715 case kIA32Movsxbl: 1716 __ movsx_b(i.OutputRegister(), i.MemoryOperand()); 1717 break; 1718 case kIA32Movzxbl: 1719 __ movzx_b(i.OutputRegister(), i.MemoryOperand()); 1720 break; 1721 case kIA32Movb: { 1722 size_t index = 0; 1723 Operand operand = i.MemoryOperand(&index); 1724 if (HasImmediateInput(instr, index)) { 1725 __ mov_b(operand, i.InputInt8(index)); 1726 } else { 1727 __ mov_b(operand, i.InputRegister(index)); 1728 } 1729 break; 1730 } 1731 case kIA32Movsxwl: 1732 __ movsx_w(i.OutputRegister(), i.MemoryOperand()); 1733 break; 1734 case kIA32Movzxwl: 1735 __ movzx_w(i.OutputRegister(), i.MemoryOperand()); 1736 break; 1737 case kIA32Movw: { 1738 size_t index = 0; 1739 Operand operand = i.MemoryOperand(&index); 1740 if (HasImmediateInput(instr, index)) { 1741 __ mov_w(operand, i.InputInt16(index)); 1742 } else { 1743 __ mov_w(operand, i.InputRegister(index)); 1744 } 1745 break; 1746 } 1747 case kIA32Movl: 1748 if (instr->HasOutput()) { 1749 __ mov(i.OutputRegister(), i.MemoryOperand()); 1750 } else { 1751 size_t index = 0; 1752 Operand operand = i.MemoryOperand(&index); 1753 if (HasImmediateInput(instr, index)) { 1754 __ mov(operand, i.InputImmediate(index)); 1755 } else { 1756 __ mov(operand, i.InputRegister(index)); 1757 } 1758 } 1759 break; 1760 case kIA32Movsd: 1761 if (instr->HasOutput()) { 1762 __ movsd(i.OutputDoubleRegister(), i.MemoryOperand()); 1763 } else { 1764 size_t index = 0; 1765 Operand operand = i.MemoryOperand(&index); 1766 __ movsd(operand, i.InputDoubleRegister(index)); 1767 } 1768 break; 1769 case kIA32Movss: 1770 if (instr->HasOutput()) { 1771 __ movss(i.OutputDoubleRegister(), i.MemoryOperand()); 1772 } else { 1773 size_t index = 0; 1774 Operand operand = i.MemoryOperand(&index); 1775 __ movss(operand, i.InputDoubleRegister(index)); 1776 } 1777 break; 1778 case kIA32BitcastFI: 1779 if (instr->InputAt(0)->IsFPStackSlot()) { 1780 __ mov(i.OutputRegister(), i.InputOperand(0)); 1781 } else { 1782 __ movd(i.OutputRegister(), i.InputDoubleRegister(0)); 1783 } 1784 break; 1785 case kIA32BitcastIF: 1786 if (instr->InputAt(0)->IsRegister()) { 1787 __ movd(i.OutputDoubleRegister(), i.InputRegister(0)); 1788 } else { 1789 __ movss(i.OutputDoubleRegister(), i.InputOperand(0)); 1790 } 1791 break; 1792 case kIA32Lea: { 1793 AddressingMode mode = AddressingModeField::decode(instr->opcode()); 1794 // Shorten "leal" to "addl", "subl" or "shll" if the register allocation 1795 // and addressing mode just happens to work out. The "addl"/"subl" forms 1796 // in these cases are faster based on measurements. 1797 if (mode == kMode_MI) { 1798 __ Move(i.OutputRegister(), Immediate(i.InputInt32(0))); 1799 } else if (i.InputRegister(0).is(i.OutputRegister())) { 1800 if (mode == kMode_MRI) { 1801 int32_t constant_summand = i.InputInt32(1); 1802 if (constant_summand > 0) { 1803 __ add(i.OutputRegister(), Immediate(constant_summand)); 1804 } else if (constant_summand < 0) { 1805 __ sub(i.OutputRegister(), Immediate(-constant_summand)); 1806 } 1807 } else if (mode == kMode_MR1) { 1808 if (i.InputRegister(1).is(i.OutputRegister())) { 1809 __ shl(i.OutputRegister(), 1); 1810 } else { 1811 __ add(i.OutputRegister(), i.InputRegister(1)); 1812 } 1813 } else if (mode == kMode_M2) { 1814 __ shl(i.OutputRegister(), 1); 1815 } else if (mode == kMode_M4) { 1816 __ shl(i.OutputRegister(), 2); 1817 } else if (mode == kMode_M8) { 1818 __ shl(i.OutputRegister(), 3); 1819 } else { 1820 __ lea(i.OutputRegister(), i.MemoryOperand()); 1821 } 1822 } else if (mode == kMode_MR1 && 1823 i.InputRegister(1).is(i.OutputRegister())) { 1824 __ add(i.OutputRegister(), i.InputRegister(0)); 1825 } else { 1826 __ lea(i.OutputRegister(), i.MemoryOperand()); 1827 } 1828 break; 1829 } 1830 case kIA32PushFloat32: 1831 if (instr->InputAt(0)->IsFPRegister()) { 1832 __ sub(esp, Immediate(kFloatSize)); 1833 __ movss(Operand(esp, 0), i.InputDoubleRegister(0)); 1834 frame_access_state()->IncreaseSPDelta(kFloatSize / kPointerSize); 1835 } else if (HasImmediateInput(instr, 0)) { 1836 __ Move(kScratchDoubleReg, i.InputDouble(0)); 1837 __ sub(esp, Immediate(kDoubleSize)); 1838 __ movss(Operand(esp, 0), kScratchDoubleReg); 1839 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 1840 } else { 1841 __ movsd(kScratchDoubleReg, i.InputOperand(0)); 1842 __ sub(esp, Immediate(kDoubleSize)); 1843 __ movss(Operand(esp, 0), kScratchDoubleReg); 1844 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 1845 } 1846 break; 1847 case kIA32PushFloat64: 1848 if (instr->InputAt(0)->IsFPRegister()) { 1849 __ sub(esp, Immediate(kDoubleSize)); 1850 __ movsd(Operand(esp, 0), i.InputDoubleRegister(0)); 1851 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 1852 } else if (HasImmediateInput(instr, 0)) { 1853 __ Move(kScratchDoubleReg, i.InputDouble(0)); 1854 __ sub(esp, Immediate(kDoubleSize)); 1855 __ movsd(Operand(esp, 0), kScratchDoubleReg); 1856 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 1857 } else { 1858 __ movsd(kScratchDoubleReg, i.InputOperand(0)); 1859 __ sub(esp, Immediate(kDoubleSize)); 1860 __ movsd(Operand(esp, 0), kScratchDoubleReg); 1861 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 1862 } 1863 break; 1864 case kIA32Push: 1865 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { 1866 size_t index = 0; 1867 Operand operand = i.MemoryOperand(&index); 1868 __ push(operand); 1869 frame_access_state()->IncreaseSPDelta(kFloatSize / kPointerSize); 1870 } else if (instr->InputAt(0)->IsFPRegister()) { 1871 __ sub(esp, Immediate(kFloatSize)); 1872 __ movsd(Operand(esp, 0), i.InputDoubleRegister(0)); 1873 frame_access_state()->IncreaseSPDelta(kFloatSize / kPointerSize); 1874 } else if (HasImmediateInput(instr, 0)) { 1875 __ push(i.InputImmediate(0)); 1876 frame_access_state()->IncreaseSPDelta(1); 1877 } else { 1878 __ push(i.InputOperand(0)); 1879 frame_access_state()->IncreaseSPDelta(1); 1880 } 1881 break; 1882 case kIA32Poke: { 1883 int const slot = MiscField::decode(instr->opcode()); 1884 if (HasImmediateInput(instr, 0)) { 1885 __ mov(Operand(esp, slot * kPointerSize), i.InputImmediate(0)); 1886 } else { 1887 __ mov(Operand(esp, slot * kPointerSize), i.InputRegister(0)); 1888 } 1889 break; 1890 } 1891 case kIA32Xchgb: { 1892 size_t index = 0; 1893 Operand operand = i.MemoryOperand(&index); 1894 __ xchg_b(i.InputRegister(index), operand); 1895 break; 1896 } 1897 case kIA32Xchgw: { 1898 size_t index = 0; 1899 Operand operand = i.MemoryOperand(&index); 1900 __ xchg_w(i.InputRegister(index), operand); 1901 break; 1902 } 1903 case kIA32Xchgl: { 1904 size_t index = 0; 1905 Operand operand = i.MemoryOperand(&index); 1906 __ xchg(i.InputRegister(index), operand); 1907 break; 1908 } 1909 case kCheckedLoadInt8: 1910 ASSEMBLE_CHECKED_LOAD_INTEGER(movsx_b); 1911 break; 1912 case kCheckedLoadUint8: 1913 ASSEMBLE_CHECKED_LOAD_INTEGER(movzx_b); 1914 break; 1915 case kCheckedLoadInt16: 1916 ASSEMBLE_CHECKED_LOAD_INTEGER(movsx_w); 1917 break; 1918 case kCheckedLoadUint16: 1919 ASSEMBLE_CHECKED_LOAD_INTEGER(movzx_w); 1920 break; 1921 case kCheckedLoadWord32: 1922 ASSEMBLE_CHECKED_LOAD_INTEGER(mov); 1923 break; 1924 case kCheckedLoadFloat32: 1925 ASSEMBLE_CHECKED_LOAD_FLOAT(movss, OutOfLineLoadFloat32NaN, s); 1926 break; 1927 case kCheckedLoadFloat64: 1928 ASSEMBLE_CHECKED_LOAD_FLOAT(movsd, OutOfLineLoadFloat64NaN, d); 1929 break; 1930 case kCheckedStoreWord8: 1931 ASSEMBLE_CHECKED_STORE_INTEGER(mov_b); 1932 break; 1933 case kCheckedStoreWord16: 1934 ASSEMBLE_CHECKED_STORE_INTEGER(mov_w); 1935 break; 1936 case kCheckedStoreWord32: 1937 ASSEMBLE_CHECKED_STORE_INTEGER(mov); 1938 break; 1939 case kCheckedStoreFloat32: 1940 ASSEMBLE_CHECKED_STORE_FLOAT(movss); 1941 break; 1942 case kCheckedStoreFloat64: 1943 ASSEMBLE_CHECKED_STORE_FLOAT(movsd); 1944 break; 1945 case kIA32StackCheck: { 1946 ExternalReference const stack_limit = 1947 ExternalReference::address_of_stack_limit(isolate()); 1948 __ cmp(esp, Operand::StaticVariable(stack_limit)); 1949 break; 1950 } 1951 case kCheckedLoadWord64: 1952 case kCheckedStoreWord64: 1953 UNREACHABLE(); // currently unsupported checked int64 load/store. 1954 break; 1955 case kAtomicLoadInt8: 1956 case kAtomicLoadUint8: 1957 case kAtomicLoadInt16: 1958 case kAtomicLoadUint16: 1959 case kAtomicLoadWord32: 1960 case kAtomicStoreWord8: 1961 case kAtomicStoreWord16: 1962 case kAtomicStoreWord32: 1963 UNREACHABLE(); // Won't be generated by instruction selector. 1964 break; 1965 } 1966 return kSuccess; 1967 } // NOLINT(readability/fn_size) 1968 1969 static Condition FlagsConditionToCondition(FlagsCondition condition) { 1970 switch (condition) { 1971 case kUnorderedEqual: 1972 case kEqual: 1973 return equal; 1974 break; 1975 case kUnorderedNotEqual: 1976 case kNotEqual: 1977 return not_equal; 1978 break; 1979 case kSignedLessThan: 1980 return less; 1981 break; 1982 case kSignedGreaterThanOrEqual: 1983 return greater_equal; 1984 break; 1985 case kSignedLessThanOrEqual: 1986 return less_equal; 1987 break; 1988 case kSignedGreaterThan: 1989 return greater; 1990 break; 1991 case kUnsignedLessThan: 1992 return below; 1993 break; 1994 case kUnsignedGreaterThanOrEqual: 1995 return above_equal; 1996 break; 1997 case kUnsignedLessThanOrEqual: 1998 return below_equal; 1999 break; 2000 case kUnsignedGreaterThan: 2001 return above; 2002 break; 2003 case kOverflow: 2004 return overflow; 2005 break; 2006 case kNotOverflow: 2007 return no_overflow; 2008 break; 2009 default: 2010 UNREACHABLE(); 2011 return no_condition; 2012 break; 2013 } 2014 } 2015 2016 // Assembles a branch after an instruction. 2017 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { 2018 Label::Distance flabel_distance = 2019 branch->fallthru ? Label::kNear : Label::kFar; 2020 Label* tlabel = branch->true_label; 2021 Label* flabel = branch->false_label; 2022 if (branch->condition == kUnorderedEqual) { 2023 __ j(parity_even, flabel, flabel_distance); 2024 } else if (branch->condition == kUnorderedNotEqual) { 2025 __ j(parity_even, tlabel); 2026 } 2027 __ j(FlagsConditionToCondition(branch->condition), tlabel); 2028 2029 // Add a jump if not falling through to the next block. 2030 if (!branch->fallthru) __ jmp(flabel); 2031 } 2032 2033 2034 void CodeGenerator::AssembleArchJump(RpoNumber target) { 2035 if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target)); 2036 } 2037 2038 void CodeGenerator::AssembleArchTrap(Instruction* instr, 2039 FlagsCondition condition) { 2040 class OutOfLineTrap final : public OutOfLineCode { 2041 public: 2042 OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr) 2043 : OutOfLineCode(gen), 2044 frame_elided_(frame_elided), 2045 instr_(instr), 2046 gen_(gen) {} 2047 2048 void Generate() final { 2049 IA32OperandConverter i(gen_, instr_); 2050 2051 Builtins::Name trap_id = 2052 static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1)); 2053 bool old_has_frame = __ has_frame(); 2054 if (frame_elided_) { 2055 __ set_has_frame(true); 2056 __ EnterFrame(StackFrame::WASM_COMPILED); 2057 } 2058 GenerateCallToTrap(trap_id); 2059 if (frame_elided_) { 2060 __ set_has_frame(old_has_frame); 2061 } 2062 } 2063 2064 private: 2065 void GenerateCallToTrap(Builtins::Name trap_id) { 2066 if (trap_id == Builtins::builtin_count) { 2067 // We cannot test calls to the runtime in cctest/test-run-wasm. 2068 // Therefore we emit a call to C here instead of a call to the runtime. 2069 __ PrepareCallCFunction(0, esi); 2070 __ CallCFunction( 2071 ExternalReference::wasm_call_trap_callback_for_testing(isolate()), 2072 0); 2073 __ LeaveFrame(StackFrame::WASM_COMPILED); 2074 __ Ret(); 2075 } else { 2076 gen_->AssembleSourcePosition(instr_); 2077 __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()), 2078 RelocInfo::CODE_TARGET); 2079 ReferenceMap* reference_map = 2080 new (gen_->zone()) ReferenceMap(gen_->zone()); 2081 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0, 2082 Safepoint::kNoLazyDeopt); 2083 if (FLAG_debug_code) { 2084 __ ud2(); 2085 } 2086 } 2087 } 2088 2089 bool frame_elided_; 2090 Instruction* instr_; 2091 CodeGenerator* gen_; 2092 }; 2093 bool frame_elided = !frame_access_state()->has_frame(); 2094 auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr); 2095 Label* tlabel = ool->entry(); 2096 Label end; 2097 if (condition == kUnorderedEqual) { 2098 __ j(parity_even, &end); 2099 } else if (condition == kUnorderedNotEqual) { 2100 __ j(parity_even, tlabel); 2101 } 2102 __ j(FlagsConditionToCondition(condition), tlabel); 2103 __ bind(&end); 2104 } 2105 2106 // Assembles boolean materializations after an instruction. 2107 void CodeGenerator::AssembleArchBoolean(Instruction* instr, 2108 FlagsCondition condition) { 2109 IA32OperandConverter i(this, instr); 2110 Label done; 2111 2112 // Materialize a full 32-bit 1 or 0 value. The result register is always the 2113 // last output of the instruction. 2114 Label check; 2115 DCHECK_NE(0u, instr->OutputCount()); 2116 Register reg = i.OutputRegister(instr->OutputCount() - 1); 2117 if (condition == kUnorderedEqual) { 2118 __ j(parity_odd, &check, Label::kNear); 2119 __ Move(reg, Immediate(0)); 2120 __ jmp(&done, Label::kNear); 2121 } else if (condition == kUnorderedNotEqual) { 2122 __ j(parity_odd, &check, Label::kNear); 2123 __ mov(reg, Immediate(1)); 2124 __ jmp(&done, Label::kNear); 2125 } 2126 Condition cc = FlagsConditionToCondition(condition); 2127 2128 __ bind(&check); 2129 if (reg.is_byte_register()) { 2130 // setcc for byte registers (al, bl, cl, dl). 2131 __ setcc(cc, reg); 2132 __ movzx_b(reg, reg); 2133 } else { 2134 // Emit a branch to set a register to either 1 or 0. 2135 Label set; 2136 __ j(cc, &set, Label::kNear); 2137 __ Move(reg, Immediate(0)); 2138 __ jmp(&done, Label::kNear); 2139 __ bind(&set); 2140 __ mov(reg, Immediate(1)); 2141 } 2142 __ bind(&done); 2143 } 2144 2145 2146 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) { 2147 IA32OperandConverter i(this, instr); 2148 Register input = i.InputRegister(0); 2149 for (size_t index = 2; index < instr->InputCount(); index += 2) { 2150 __ cmp(input, Immediate(i.InputInt32(index + 0))); 2151 __ j(equal, GetLabel(i.InputRpo(index + 1))); 2152 } 2153 AssembleArchJump(i.InputRpo(1)); 2154 } 2155 2156 2157 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) { 2158 IA32OperandConverter i(this, instr); 2159 Register input = i.InputRegister(0); 2160 size_t const case_count = instr->InputCount() - 2; 2161 Label** cases = zone()->NewArray<Label*>(case_count); 2162 for (size_t index = 0; index < case_count; ++index) { 2163 cases[index] = GetLabel(i.InputRpo(index + 2)); 2164 } 2165 Label* const table = AddJumpTable(cases, case_count); 2166 __ cmp(input, Immediate(case_count)); 2167 __ j(above_equal, GetLabel(i.InputRpo(1))); 2168 __ jmp(Operand::JumpTable(input, times_4, table)); 2169 } 2170 2171 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall( 2172 int deoptimization_id, SourcePosition pos) { 2173 DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id); 2174 DeoptimizeReason deoptimization_reason = 2175 GetDeoptimizationReason(deoptimization_id); 2176 Deoptimizer::BailoutType bailout_type = 2177 deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT 2178 : Deoptimizer::EAGER; 2179 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry( 2180 isolate(), deoptimization_id, bailout_type); 2181 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts; 2182 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id); 2183 __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY); 2184 return kSuccess; 2185 } 2186 2187 2188 // The calling convention for JSFunctions on IA32 passes arguments on the 2189 // stack and the JSFunction and context in EDI and ESI, respectively, thus 2190 // the steps of the call look as follows: 2191 2192 // --{ before the call instruction }-------------------------------------------- 2193 // | caller frame | 2194 // ^ esp ^ ebp 2195 2196 // --{ push arguments and setup ESI, EDI }-------------------------------------- 2197 // | args + receiver | caller frame | 2198 // ^ esp ^ ebp 2199 // [edi = JSFunction, esi = context] 2200 2201 // --{ call [edi + kCodeEntryOffset] }------------------------------------------ 2202 // | RET | args + receiver | caller frame | 2203 // ^ esp ^ ebp 2204 2205 // =={ prologue of called function }============================================ 2206 // --{ push ebp }--------------------------------------------------------------- 2207 // | FP | RET | args + receiver | caller frame | 2208 // ^ esp ^ ebp 2209 2210 // --{ mov ebp, esp }----------------------------------------------------------- 2211 // | FP | RET | args + receiver | caller frame | 2212 // ^ ebp,esp 2213 2214 // --{ push esi }--------------------------------------------------------------- 2215 // | CTX | FP | RET | args + receiver | caller frame | 2216 // ^esp ^ ebp 2217 2218 // --{ push edi }--------------------------------------------------------------- 2219 // | FNC | CTX | FP | RET | args + receiver | caller frame | 2220 // ^esp ^ ebp 2221 2222 // --{ subi esp, #N }----------------------------------------------------------- 2223 // | callee frame | FNC | CTX | FP | RET | args + receiver | caller frame | 2224 // ^esp ^ ebp 2225 2226 // =={ body of called function }================================================ 2227 2228 // =={ epilogue of called function }============================================ 2229 // --{ mov esp, ebp }----------------------------------------------------------- 2230 // | FP | RET | args + receiver | caller frame | 2231 // ^ esp,ebp 2232 2233 // --{ pop ebp }----------------------------------------------------------- 2234 // | | RET | args + receiver | caller frame | 2235 // ^ esp ^ ebp 2236 2237 // --{ ret #A+1 }----------------------------------------------------------- 2238 // | | caller frame | 2239 // ^ esp ^ ebp 2240 2241 2242 // Runtime function calls are accomplished by doing a stub call to the 2243 // CEntryStub (a real code object). On IA32 passes arguments on the 2244 // stack, the number of arguments in EAX, the address of the runtime function 2245 // in EBX, and the context in ESI. 2246 2247 // --{ before the call instruction }-------------------------------------------- 2248 // | caller frame | 2249 // ^ esp ^ ebp 2250 2251 // --{ push arguments and setup EAX, EBX, and ESI }----------------------------- 2252 // | args + receiver | caller frame | 2253 // ^ esp ^ ebp 2254 // [eax = #args, ebx = runtime function, esi = context] 2255 2256 // --{ call #CEntryStub }------------------------------------------------------- 2257 // | RET | args + receiver | caller frame | 2258 // ^ esp ^ ebp 2259 2260 // =={ body of runtime function }=============================================== 2261 2262 // --{ runtime returns }-------------------------------------------------------- 2263 // | caller frame | 2264 // ^ esp ^ ebp 2265 2266 // Other custom linkages (e.g. for calling directly into and out of C++) may 2267 // need to save callee-saved registers on the stack, which is done in the 2268 // function prologue of generated code. 2269 2270 // --{ before the call instruction }-------------------------------------------- 2271 // | caller frame | 2272 // ^ esp ^ ebp 2273 2274 // --{ set up arguments in registers on stack }--------------------------------- 2275 // | args | caller frame | 2276 // ^ esp ^ ebp 2277 // [r0 = arg0, r1 = arg1, ...] 2278 2279 // --{ call code }-------------------------------------------------------------- 2280 // | RET | args | caller frame | 2281 // ^ esp ^ ebp 2282 2283 // =={ prologue of called function }============================================ 2284 // --{ push ebp }--------------------------------------------------------------- 2285 // | FP | RET | args | caller frame | 2286 // ^ esp ^ ebp 2287 2288 // --{ mov ebp, esp }----------------------------------------------------------- 2289 // | FP | RET | args | caller frame | 2290 // ^ ebp,esp 2291 2292 // --{ save registers }--------------------------------------------------------- 2293 // | regs | FP | RET | args | caller frame | 2294 // ^ esp ^ ebp 2295 2296 // --{ subi esp, #N }----------------------------------------------------------- 2297 // | callee frame | regs | FP | RET | args | caller frame | 2298 // ^esp ^ ebp 2299 2300 // =={ body of called function }================================================ 2301 2302 // =={ epilogue of called function }============================================ 2303 // --{ restore registers }------------------------------------------------------ 2304 // | regs | FP | RET | args | caller frame | 2305 // ^ esp ^ ebp 2306 2307 // --{ mov esp, ebp }----------------------------------------------------------- 2308 // | FP | RET | args | caller frame | 2309 // ^ esp,ebp 2310 2311 // --{ pop ebp }---------------------------------------------------------------- 2312 // | RET | args | caller frame | 2313 // ^ esp ^ ebp 2314 2315 void CodeGenerator::FinishFrame(Frame* frame) { 2316 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 2317 const RegList saves = descriptor->CalleeSavedRegisters(); 2318 if (saves != 0) { // Save callee-saved registers. 2319 DCHECK(!info()->is_osr()); 2320 int pushed = 0; 2321 for (int i = Register::kNumRegisters - 1; i >= 0; i--) { 2322 if (!((1 << i) & saves)) continue; 2323 ++pushed; 2324 } 2325 frame->AllocateSavedCalleeRegisterSlots(pushed); 2326 } 2327 } 2328 2329 void CodeGenerator::AssembleConstructFrame() { 2330 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 2331 if (frame_access_state()->has_frame()) { 2332 if (descriptor->IsCFunctionCall()) { 2333 __ push(ebp); 2334 __ mov(ebp, esp); 2335 } else if (descriptor->IsJSFunctionCall()) { 2336 __ Prologue(this->info()->GeneratePreagedPrologue()); 2337 if (descriptor->PushArgumentCount()) { 2338 __ push(kJavaScriptCallArgCountRegister); 2339 } 2340 } else { 2341 __ StubPrologue(info()->GetOutputStackFrameType()); 2342 } 2343 } 2344 2345 int shrink_slots = 2346 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize(); 2347 2348 if (info()->is_osr()) { 2349 // TurboFan OSR-compiled functions cannot be entered directly. 2350 __ Abort(kShouldNotDirectlyEnterOsrFunction); 2351 2352 // Unoptimized code jumps directly to this entrypoint while the unoptimized 2353 // frame is still on the stack. Optimized code uses OSR values directly from 2354 // the unoptimized frame. Thus, all that needs to be done is to allocate the 2355 // remaining stack slots. 2356 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --"); 2357 osr_pc_offset_ = __ pc_offset(); 2358 shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots(); 2359 } 2360 2361 const RegList saves = descriptor->CalleeSavedRegisters(); 2362 if (shrink_slots > 0) { 2363 __ sub(esp, Immediate(shrink_slots * kPointerSize)); 2364 } 2365 2366 if (saves != 0) { // Save callee-saved registers. 2367 DCHECK(!info()->is_osr()); 2368 int pushed = 0; 2369 for (int i = Register::kNumRegisters - 1; i >= 0; i--) { 2370 if (!((1 << i) & saves)) continue; 2371 __ push(Register::from_code(i)); 2372 ++pushed; 2373 } 2374 } 2375 } 2376 2377 void CodeGenerator::AssembleReturn(InstructionOperand* pop) { 2378 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 2379 2380 const RegList saves = descriptor->CalleeSavedRegisters(); 2381 // Restore registers. 2382 if (saves != 0) { 2383 for (int i = 0; i < Register::kNumRegisters; i++) { 2384 if (!((1 << i) & saves)) continue; 2385 __ pop(Register::from_code(i)); 2386 } 2387 } 2388 2389 // Might need ecx for scratch if pop_size is too big or if there is a variable 2390 // pop count. 2391 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & ecx.bit()); 2392 size_t pop_size = descriptor->StackParameterCount() * kPointerSize; 2393 IA32OperandConverter g(this, nullptr); 2394 if (descriptor->IsCFunctionCall()) { 2395 AssembleDeconstructFrame(); 2396 } else if (frame_access_state()->has_frame()) { 2397 // Canonicalize JSFunction return sites for now if they always have the same 2398 // number of return args. 2399 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) { 2400 if (return_label_.is_bound()) { 2401 __ jmp(&return_label_); 2402 return; 2403 } else { 2404 __ bind(&return_label_); 2405 AssembleDeconstructFrame(); 2406 } 2407 } else { 2408 AssembleDeconstructFrame(); 2409 } 2410 } 2411 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & edx.bit()); 2412 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & ecx.bit()); 2413 if (pop->IsImmediate()) { 2414 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type()); 2415 pop_size += g.ToConstant(pop).ToInt32() * kPointerSize; 2416 __ Ret(static_cast<int>(pop_size), ecx); 2417 } else { 2418 Register pop_reg = g.ToRegister(pop); 2419 Register scratch_reg = pop_reg.is(ecx) ? edx : ecx; 2420 __ pop(scratch_reg); 2421 __ lea(esp, Operand(esp, pop_reg, times_4, static_cast<int>(pop_size))); 2422 __ jmp(scratch_reg); 2423 } 2424 } 2425 2426 2427 void CodeGenerator::AssembleMove(InstructionOperand* source, 2428 InstructionOperand* destination) { 2429 IA32OperandConverter g(this, nullptr); 2430 // Dispatch on the source and destination operand kinds. Not all 2431 // combinations are possible. 2432 if (source->IsRegister()) { 2433 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 2434 Register src = g.ToRegister(source); 2435 Operand dst = g.ToOperand(destination); 2436 __ mov(dst, src); 2437 } else if (source->IsStackSlot()) { 2438 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 2439 Operand src = g.ToOperand(source); 2440 if (destination->IsRegister()) { 2441 Register dst = g.ToRegister(destination); 2442 __ mov(dst, src); 2443 } else { 2444 Operand dst = g.ToOperand(destination); 2445 __ push(src); 2446 __ pop(dst); 2447 } 2448 } else if (source->IsConstant()) { 2449 Constant src_constant = g.ToConstant(source); 2450 if (src_constant.type() == Constant::kHeapObject) { 2451 Handle<HeapObject> src = src_constant.ToHeapObject(); 2452 if (destination->IsRegister()) { 2453 Register dst = g.ToRegister(destination); 2454 __ LoadHeapObject(dst, src); 2455 } else { 2456 DCHECK(destination->IsStackSlot()); 2457 Operand dst = g.ToOperand(destination); 2458 AllowDeferredHandleDereference embedding_raw_address; 2459 if (isolate()->heap()->InNewSpace(*src)) { 2460 __ PushHeapObject(src); 2461 __ pop(dst); 2462 } else { 2463 __ mov(dst, src); 2464 } 2465 } 2466 } else if (destination->IsRegister()) { 2467 Register dst = g.ToRegister(destination); 2468 __ Move(dst, g.ToImmediate(source)); 2469 } else if (destination->IsStackSlot()) { 2470 Operand dst = g.ToOperand(destination); 2471 __ Move(dst, g.ToImmediate(source)); 2472 } else if (src_constant.type() == Constant::kFloat32) { 2473 // TODO(turbofan): Can we do better here? 2474 uint32_t src = src_constant.ToFloat32AsInt(); 2475 if (destination->IsFPRegister()) { 2476 XMMRegister dst = g.ToDoubleRegister(destination); 2477 __ Move(dst, src); 2478 } else { 2479 DCHECK(destination->IsFPStackSlot()); 2480 Operand dst = g.ToOperand(destination); 2481 __ Move(dst, Immediate(src)); 2482 } 2483 } else { 2484 DCHECK_EQ(Constant::kFloat64, src_constant.type()); 2485 uint64_t src = src_constant.ToFloat64AsInt(); 2486 uint32_t lower = static_cast<uint32_t>(src); 2487 uint32_t upper = static_cast<uint32_t>(src >> 32); 2488 if (destination->IsFPRegister()) { 2489 XMMRegister dst = g.ToDoubleRegister(destination); 2490 __ Move(dst, src); 2491 } else { 2492 DCHECK(destination->IsFPStackSlot()); 2493 Operand dst0 = g.ToOperand(destination); 2494 Operand dst1 = g.HighOperand(destination); 2495 __ Move(dst0, Immediate(lower)); 2496 __ Move(dst1, Immediate(upper)); 2497 } 2498 } 2499 } else if (source->IsFPRegister()) { 2500 XMMRegister src = g.ToDoubleRegister(source); 2501 if (destination->IsFPRegister()) { 2502 XMMRegister dst = g.ToDoubleRegister(destination); 2503 __ movaps(dst, src); 2504 } else { 2505 DCHECK(destination->IsFPStackSlot()); 2506 Operand dst = g.ToOperand(destination); 2507 MachineRepresentation rep = 2508 LocationOperand::cast(source)->representation(); 2509 if (rep == MachineRepresentation::kFloat64) { 2510 __ movsd(dst, src); 2511 } else if (rep == MachineRepresentation::kFloat32) { 2512 __ movss(dst, src); 2513 } else { 2514 DCHECK_EQ(MachineRepresentation::kSimd128, rep); 2515 __ movups(dst, src); 2516 } 2517 } 2518 } else if (source->IsFPStackSlot()) { 2519 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot()); 2520 Operand src = g.ToOperand(source); 2521 MachineRepresentation rep = LocationOperand::cast(source)->representation(); 2522 if (destination->IsFPRegister()) { 2523 XMMRegister dst = g.ToDoubleRegister(destination); 2524 if (rep == MachineRepresentation::kFloat64) { 2525 __ movsd(dst, src); 2526 } else if (rep == MachineRepresentation::kFloat32) { 2527 __ movss(dst, src); 2528 } else { 2529 DCHECK_EQ(MachineRepresentation::kSimd128, rep); 2530 __ movups(dst, src); 2531 } 2532 } else { 2533 Operand dst = g.ToOperand(destination); 2534 if (rep == MachineRepresentation::kFloat64) { 2535 __ movsd(kScratchDoubleReg, src); 2536 __ movsd(dst, kScratchDoubleReg); 2537 } else if (rep == MachineRepresentation::kFloat32) { 2538 __ movss(kScratchDoubleReg, src); 2539 __ movss(dst, kScratchDoubleReg); 2540 } else { 2541 DCHECK_EQ(MachineRepresentation::kSimd128, rep); 2542 __ movups(kScratchDoubleReg, src); 2543 __ movups(dst, kScratchDoubleReg); 2544 } 2545 } 2546 } else { 2547 UNREACHABLE(); 2548 } 2549 } 2550 2551 2552 void CodeGenerator::AssembleSwap(InstructionOperand* source, 2553 InstructionOperand* destination) { 2554 IA32OperandConverter g(this, nullptr); 2555 // Dispatch on the source and destination operand kinds. Not all 2556 // combinations are possible. 2557 if (source->IsRegister() && destination->IsRegister()) { 2558 // Register-register. 2559 Register src = g.ToRegister(source); 2560 Register dst = g.ToRegister(destination); 2561 __ push(src); 2562 __ mov(src, dst); 2563 __ pop(dst); 2564 } else if (source->IsRegister() && destination->IsStackSlot()) { 2565 // Register-memory. 2566 Register src = g.ToRegister(source); 2567 __ push(src); 2568 frame_access_state()->IncreaseSPDelta(1); 2569 Operand dst = g.ToOperand(destination); 2570 __ mov(src, dst); 2571 frame_access_state()->IncreaseSPDelta(-1); 2572 dst = g.ToOperand(destination); 2573 __ pop(dst); 2574 } else if (source->IsStackSlot() && destination->IsStackSlot()) { 2575 // Memory-memory. 2576 Operand dst1 = g.ToOperand(destination); 2577 __ push(dst1); 2578 frame_access_state()->IncreaseSPDelta(1); 2579 Operand src1 = g.ToOperand(source); 2580 __ push(src1); 2581 Operand dst2 = g.ToOperand(destination); 2582 __ pop(dst2); 2583 frame_access_state()->IncreaseSPDelta(-1); 2584 Operand src2 = g.ToOperand(source); 2585 __ pop(src2); 2586 } else if (source->IsFPRegister() && destination->IsFPRegister()) { 2587 // XMM register-register swap. 2588 XMMRegister src = g.ToDoubleRegister(source); 2589 XMMRegister dst = g.ToDoubleRegister(destination); 2590 __ movaps(kScratchDoubleReg, src); 2591 __ movaps(src, dst); 2592 __ movaps(dst, kScratchDoubleReg); 2593 } else if (source->IsFPRegister() && destination->IsFPStackSlot()) { 2594 // XMM register-memory swap. 2595 XMMRegister reg = g.ToDoubleRegister(source); 2596 Operand other = g.ToOperand(destination); 2597 MachineRepresentation rep = LocationOperand::cast(source)->representation(); 2598 if (rep == MachineRepresentation::kFloat64) { 2599 __ movsd(kScratchDoubleReg, other); 2600 __ movsd(other, reg); 2601 __ movaps(reg, kScratchDoubleReg); 2602 } else if (rep == MachineRepresentation::kFloat32) { 2603 __ movss(kScratchDoubleReg, other); 2604 __ movss(other, reg); 2605 __ movaps(reg, kScratchDoubleReg); 2606 } else { 2607 DCHECK_EQ(MachineRepresentation::kSimd128, rep); 2608 __ movups(kScratchDoubleReg, other); 2609 __ movups(other, reg); 2610 __ movups(reg, kScratchDoubleReg); 2611 } 2612 } else if (source->IsFPStackSlot() && destination->IsFPStackSlot()) { 2613 // Double-width memory-to-memory. 2614 Operand src0 = g.ToOperand(source); 2615 Operand dst0 = g.ToOperand(destination); 2616 MachineRepresentation rep = LocationOperand::cast(source)->representation(); 2617 if (rep == MachineRepresentation::kFloat64) { 2618 Operand src1 = g.HighOperand(source); 2619 Operand dst1 = g.HighOperand(destination); 2620 __ movsd(kScratchDoubleReg, dst0); // Save dst in scratch register. 2621 __ push(src0); // Then use stack to copy src to destination. 2622 __ pop(dst0); 2623 __ push(src1); 2624 __ pop(dst1); 2625 __ movsd(src0, kScratchDoubleReg); 2626 } else if (rep == MachineRepresentation::kFloat32) { 2627 __ movss(kScratchDoubleReg, dst0); // Save dst in scratch register. 2628 __ push(src0); // Then use stack to copy src to destination. 2629 __ pop(dst0); 2630 __ movss(src0, kScratchDoubleReg); 2631 } else { 2632 DCHECK_EQ(MachineRepresentation::kSimd128, rep); 2633 // Use the XOR trick to swap without a temporary. 2634 __ movups(kScratchDoubleReg, src0); 2635 __ xorps(kScratchDoubleReg, dst0); // scratch contains src ^ dst. 2636 __ movups(src0, kScratchDoubleReg); 2637 __ xorps(kScratchDoubleReg, dst0); // scratch contains src. 2638 __ movups(dst0, kScratchDoubleReg); 2639 __ xorps(kScratchDoubleReg, src0); // scratch contains dst. 2640 __ movups(src0, kScratchDoubleReg); 2641 } 2642 } else { 2643 // No other combinations are possible. 2644 UNREACHABLE(); 2645 } 2646 } 2647 2648 2649 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) { 2650 for (size_t index = 0; index < target_count; ++index) { 2651 __ dd(targets[index]); 2652 } 2653 } 2654 2655 2656 void CodeGenerator::EnsureSpaceForLazyDeopt() { 2657 if (!info()->ShouldEnsureSpaceForLazyDeopt()) { 2658 return; 2659 } 2660 2661 int space_needed = Deoptimizer::patch_size(); 2662 // Ensure that we have enough space after the previous lazy-bailout 2663 // instruction for patching the code here. 2664 int current_pc = masm()->pc_offset(); 2665 if (current_pc < last_lazy_deopt_pc_ + space_needed) { 2666 int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc; 2667 __ Nop(padding_size); 2668 } 2669 } 2670 2671 #undef __ 2672 2673 } // namespace compiler 2674 } // namespace internal 2675 } // namespace v8 2676
