1 // Copyright 2014 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/base/adapters.h" 6 #include "src/compiler/instruction-selector-impl.h" 7 #include "src/compiler/node-matchers.h" 8 #include "src/compiler/node-properties.h" 9 10 namespace v8 { 11 namespace internal { 12 namespace compiler { 13 14 // Adds IA32-specific methods for generating operands. 15 class IA32OperandGenerator final : public OperandGenerator { 16 public: 17 explicit IA32OperandGenerator(InstructionSelector* selector) 18 : OperandGenerator(selector) {} 19 20 InstructionOperand UseByteRegister(Node* node) { 21 // TODO(titzer): encode byte register use constraints. 22 return UseFixed(node, edx); 23 } 24 25 InstructionOperand DefineAsByteRegister(Node* node) { 26 // TODO(titzer): encode byte register def constraints. 27 return DefineAsRegister(node); 28 } 29 30 bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input, 31 int effect_level) { 32 if (input->opcode() != IrOpcode::kLoad || 33 !selector()->CanCover(node, input)) { 34 return false; 35 } 36 if (effect_level != selector()->GetEffectLevel(input)) { 37 return false; 38 } 39 MachineRepresentation rep = 40 LoadRepresentationOf(input->op()).representation(); 41 switch (opcode) { 42 case kIA32And: 43 case kIA32Or: 44 case kIA32Xor: 45 case kIA32Add: 46 case kIA32Sub: 47 case kIA32Cmp: 48 case kIA32Test: 49 return rep == MachineRepresentation::kWord32 || IsAnyTagged(rep); 50 case kIA32Cmp16: 51 case kIA32Test16: 52 return rep == MachineRepresentation::kWord16; 53 case kIA32Cmp8: 54 case kIA32Test8: 55 return rep == MachineRepresentation::kWord8; 56 default: 57 break; 58 } 59 return false; 60 } 61 62 bool CanBeImmediate(Node* node) { 63 switch (node->opcode()) { 64 case IrOpcode::kInt32Constant: 65 case IrOpcode::kNumberConstant: 66 case IrOpcode::kExternalConstant: 67 case IrOpcode::kRelocatableInt32Constant: 68 case IrOpcode::kRelocatableInt64Constant: 69 return true; 70 case IrOpcode::kHeapConstant: { 71 // TODO(bmeurer): We must not dereference handles concurrently. If we 72 // really have to this here, then we need to find a way to put this 73 // information on the HeapConstant node already. 74 #if 0 75 // Constants in new space cannot be used as immediates in V8 because 76 // the GC does not scan code objects when collecting the new generation. 77 Handle<HeapObject> value = HeapConstantOf(node->op()); 78 return !Heap::InNewSpace(*value); 79 #else 80 return false; 81 #endif 82 } 83 default: 84 return false; 85 } 86 } 87 88 AddressingMode GenerateMemoryOperandInputs(Node* index, int scale, Node* base, 89 Node* displacement_node, 90 DisplacementMode displacement_mode, 91 InstructionOperand inputs[], 92 size_t* input_count) { 93 AddressingMode mode = kMode_MRI; 94 int32_t displacement = (displacement_node == nullptr) 95 ? 0 96 : OpParameter<int32_t>(displacement_node->op()); 97 if (displacement_mode == kNegativeDisplacement) { 98 displacement = -displacement; 99 } 100 if (base != nullptr) { 101 if (base->opcode() == IrOpcode::kInt32Constant) { 102 displacement += OpParameter<int32_t>(base->op()); 103 base = nullptr; 104 } 105 } 106 if (base != nullptr) { 107 inputs[(*input_count)++] = UseRegister(base); 108 if (index != nullptr) { 109 DCHECK(scale >= 0 && scale <= 3); 110 inputs[(*input_count)++] = UseRegister(index); 111 if (displacement != 0) { 112 inputs[(*input_count)++] = TempImmediate(displacement); 113 static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I, 114 kMode_MR4I, kMode_MR8I}; 115 mode = kMRnI_modes[scale]; 116 } else { 117 static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2, 118 kMode_MR4, kMode_MR8}; 119 mode = kMRn_modes[scale]; 120 } 121 } else { 122 if (displacement == 0) { 123 mode = kMode_MR; 124 } else { 125 inputs[(*input_count)++] = TempImmediate(displacement); 126 mode = kMode_MRI; 127 } 128 } 129 } else { 130 DCHECK(scale >= 0 && scale <= 3); 131 if (index != nullptr) { 132 inputs[(*input_count)++] = UseRegister(index); 133 if (displacement != 0) { 134 inputs[(*input_count)++] = TempImmediate(displacement); 135 static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I, 136 kMode_M4I, kMode_M8I}; 137 mode = kMnI_modes[scale]; 138 } else { 139 static const AddressingMode kMn_modes[] = {kMode_MR, kMode_M2, 140 kMode_M4, kMode_M8}; 141 mode = kMn_modes[scale]; 142 } 143 } else { 144 inputs[(*input_count)++] = TempImmediate(displacement); 145 return kMode_MI; 146 } 147 } 148 return mode; 149 } 150 151 AddressingMode GetEffectiveAddressMemoryOperand(Node* node, 152 InstructionOperand inputs[], 153 size_t* input_count) { 154 BaseWithIndexAndDisplacement32Matcher m(node, AddressOption::kAllowAll); 155 DCHECK(m.matches()); 156 if ((m.displacement() == nullptr || CanBeImmediate(m.displacement()))) { 157 return GenerateMemoryOperandInputs( 158 m.index(), m.scale(), m.base(), m.displacement(), 159 m.displacement_mode(), inputs, input_count); 160 } else { 161 inputs[(*input_count)++] = UseRegister(node->InputAt(0)); 162 inputs[(*input_count)++] = UseRegister(node->InputAt(1)); 163 return kMode_MR1; 164 } 165 } 166 167 InstructionOperand GetEffectiveIndexOperand(Node* index, 168 AddressingMode* mode) { 169 if (CanBeImmediate(index)) { 170 *mode = kMode_MRI; 171 return UseImmediate(index); 172 } else { 173 *mode = kMode_MR1; 174 return UseUniqueRegister(index); 175 } 176 } 177 178 bool CanBeBetterLeftOperand(Node* node) const { 179 return !selector()->IsLive(node); 180 } 181 }; 182 183 184 namespace { 185 186 void VisitRO(InstructionSelector* selector, Node* node, ArchOpcode opcode) { 187 IA32OperandGenerator g(selector); 188 selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); 189 } 190 191 192 void VisitRR(InstructionSelector* selector, Node* node, 193 InstructionCode opcode) { 194 IA32OperandGenerator g(selector); 195 selector->Emit(opcode, g.DefineAsRegister(node), 196 g.UseRegister(node->InputAt(0))); 197 } 198 199 200 void VisitRROFloat(InstructionSelector* selector, Node* node, 201 ArchOpcode avx_opcode, ArchOpcode sse_opcode) { 202 IA32OperandGenerator g(selector); 203 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); 204 InstructionOperand operand1 = g.Use(node->InputAt(1)); 205 if (selector->IsSupported(AVX)) { 206 selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1); 207 } else { 208 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1); 209 } 210 } 211 212 213 void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input, 214 ArchOpcode avx_opcode, ArchOpcode sse_opcode) { 215 IA32OperandGenerator g(selector); 216 if (selector->IsSupported(AVX)) { 217 selector->Emit(avx_opcode, g.DefineAsRegister(node), g.Use(input)); 218 } else { 219 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), g.UseRegister(input)); 220 } 221 } 222 223 void VisitRRSimd(InstructionSelector* selector, Node* node, 224 ArchOpcode avx_opcode, ArchOpcode sse_opcode) { 225 IA32OperandGenerator g(selector); 226 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); 227 if (selector->IsSupported(AVX)) { 228 selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0); 229 } else { 230 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0); 231 } 232 } 233 234 void VisitRRISimd(InstructionSelector* selector, Node* node, 235 ArchOpcode opcode) { 236 IA32OperandGenerator g(selector); 237 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); 238 InstructionOperand operand1 = 239 g.UseImmediate(OpParameter<int32_t>(node->op())); 240 selector->Emit(opcode, g.DefineAsRegister(node), operand0, operand1); 241 } 242 243 void VisitRRISimd(InstructionSelector* selector, Node* node, 244 ArchOpcode avx_opcode, ArchOpcode sse_opcode) { 245 IA32OperandGenerator g(selector); 246 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); 247 InstructionOperand operand1 = 248 g.UseImmediate(OpParameter<int32_t>(node->op())); 249 if (selector->IsSupported(AVX)) { 250 selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1); 251 } else { 252 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1); 253 } 254 } 255 256 } // namespace 257 258 void InstructionSelector::VisitStackSlot(Node* node) { 259 StackSlotRepresentation rep = StackSlotRepresentationOf(node->op()); 260 int slot = frame_->AllocateSpillSlot(rep.size()); 261 OperandGenerator g(this); 262 263 Emit(kArchStackSlot, g.DefineAsRegister(node), 264 sequence()->AddImmediate(Constant(slot)), 0, nullptr); 265 } 266 267 void InstructionSelector::VisitDebugAbort(Node* node) { 268 IA32OperandGenerator g(this); 269 Emit(kArchDebugAbort, g.NoOutput(), g.UseFixed(node->InputAt(0), edx)); 270 } 271 272 void InstructionSelector::VisitSpeculationFence(Node* node) { 273 IA32OperandGenerator g(this); 274 Emit(kLFence, g.NoOutput()); 275 } 276 277 void InstructionSelector::VisitLoad(Node* node) { 278 LoadRepresentation load_rep = LoadRepresentationOf(node->op()); 279 280 ArchOpcode opcode = kArchNop; 281 switch (load_rep.representation()) { 282 case MachineRepresentation::kFloat32: 283 opcode = kIA32Movss; 284 break; 285 case MachineRepresentation::kFloat64: 286 opcode = kIA32Movsd; 287 break; 288 case MachineRepresentation::kBit: // Fall through. 289 case MachineRepresentation::kWord8: 290 opcode = load_rep.IsSigned() ? kIA32Movsxbl : kIA32Movzxbl; 291 break; 292 case MachineRepresentation::kWord16: 293 opcode = load_rep.IsSigned() ? kIA32Movsxwl : kIA32Movzxwl; 294 break; 295 case MachineRepresentation::kTaggedSigned: // Fall through. 296 case MachineRepresentation::kTaggedPointer: // Fall through. 297 case MachineRepresentation::kTagged: // Fall through. 298 case MachineRepresentation::kWord32: 299 opcode = kIA32Movl; 300 break; 301 case MachineRepresentation::kSimd128: 302 opcode = kIA32Movdqu; 303 break; 304 case MachineRepresentation::kWord64: // Fall through. 305 case MachineRepresentation::kNone: 306 UNREACHABLE(); 307 return; 308 } 309 310 IA32OperandGenerator g(this); 311 InstructionOperand outputs[1]; 312 outputs[0] = g.DefineAsRegister(node); 313 InstructionOperand inputs[3]; 314 size_t input_count = 0; 315 AddressingMode mode = 316 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count); 317 InstructionCode code = opcode | AddressingModeField::encode(mode); 318 if (node->opcode() == IrOpcode::kPoisonedLoad) { 319 CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison); 320 code |= MiscField::encode(kMemoryAccessPoisoned); 321 } 322 Emit(code, 1, outputs, input_count, inputs); 323 } 324 325 void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); } 326 327 void InstructionSelector::VisitProtectedLoad(Node* node) { 328 // TODO(eholk) 329 UNIMPLEMENTED(); 330 } 331 332 void InstructionSelector::VisitStore(Node* node) { 333 IA32OperandGenerator g(this); 334 Node* base = node->InputAt(0); 335 Node* index = node->InputAt(1); 336 Node* value = node->InputAt(2); 337 338 StoreRepresentation store_rep = StoreRepresentationOf(node->op()); 339 WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind(); 340 MachineRepresentation rep = store_rep.representation(); 341 342 if (write_barrier_kind != kNoWriteBarrier) { 343 DCHECK(CanBeTaggedPointer(rep)); 344 AddressingMode addressing_mode; 345 InstructionOperand inputs[] = { 346 g.UseUniqueRegister(base), 347 g.GetEffectiveIndexOperand(index, &addressing_mode), 348 g.UseUniqueRegister(value)}; 349 RecordWriteMode record_write_mode = RecordWriteMode::kValueIsAny; 350 switch (write_barrier_kind) { 351 case kNoWriteBarrier: 352 UNREACHABLE(); 353 break; 354 case kMapWriteBarrier: 355 record_write_mode = RecordWriteMode::kValueIsMap; 356 break; 357 case kPointerWriteBarrier: 358 record_write_mode = RecordWriteMode::kValueIsPointer; 359 break; 360 case kFullWriteBarrier: 361 record_write_mode = RecordWriteMode::kValueIsAny; 362 break; 363 } 364 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()}; 365 size_t const temp_count = arraysize(temps); 366 InstructionCode code = kArchStoreWithWriteBarrier; 367 code |= AddressingModeField::encode(addressing_mode); 368 code |= MiscField::encode(static_cast<int>(record_write_mode)); 369 Emit(code, 0, nullptr, arraysize(inputs), inputs, temp_count, temps); 370 } else { 371 ArchOpcode opcode = kArchNop; 372 switch (rep) { 373 case MachineRepresentation::kFloat32: 374 opcode = kIA32Movss; 375 break; 376 case MachineRepresentation::kFloat64: 377 opcode = kIA32Movsd; 378 break; 379 case MachineRepresentation::kBit: // Fall through. 380 case MachineRepresentation::kWord8: 381 opcode = kIA32Movb; 382 break; 383 case MachineRepresentation::kWord16: 384 opcode = kIA32Movw; 385 break; 386 case MachineRepresentation::kTaggedSigned: // Fall through. 387 case MachineRepresentation::kTaggedPointer: // Fall through. 388 case MachineRepresentation::kTagged: // Fall through. 389 case MachineRepresentation::kWord32: 390 opcode = kIA32Movl; 391 break; 392 case MachineRepresentation::kSimd128: 393 opcode = kIA32Movdqu; 394 break; 395 case MachineRepresentation::kWord64: // Fall through. 396 case MachineRepresentation::kNone: 397 UNREACHABLE(); 398 return; 399 } 400 401 InstructionOperand val; 402 if (g.CanBeImmediate(value)) { 403 val = g.UseImmediate(value); 404 } else if (rep == MachineRepresentation::kWord8 || 405 rep == MachineRepresentation::kBit) { 406 val = g.UseByteRegister(value); 407 } else { 408 val = g.UseRegister(value); 409 } 410 411 InstructionOperand inputs[4]; 412 size_t input_count = 0; 413 AddressingMode addressing_mode = 414 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count); 415 InstructionCode code = 416 opcode | AddressingModeField::encode(addressing_mode); 417 inputs[input_count++] = val; 418 Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count, 419 inputs); 420 } 421 } 422 423 void InstructionSelector::VisitProtectedStore(Node* node) { 424 // TODO(eholk) 425 UNIMPLEMENTED(); 426 } 427 428 // Architecture supports unaligned access, therefore VisitLoad is used instead 429 void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); } 430 431 // Architecture supports unaligned access, therefore VisitStore is used instead 432 void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); } 433 434 namespace { 435 436 // Shared routine for multiple binary operations. 437 void VisitBinop(InstructionSelector* selector, Node* node, 438 InstructionCode opcode, FlagsContinuation* cont) { 439 IA32OperandGenerator g(selector); 440 Int32BinopMatcher m(node); 441 Node* left = m.left().node(); 442 Node* right = m.right().node(); 443 InstructionOperand inputs[6]; 444 size_t input_count = 0; 445 InstructionOperand outputs[1]; 446 size_t output_count = 0; 447 448 // TODO(turbofan): match complex addressing modes. 449 if (left == right) { 450 // If both inputs refer to the same operand, enforce allocating a register 451 // for both of them to ensure that we don't end up generating code like 452 // this: 453 // 454 // mov eax, [ebp-0x10] 455 // add eax, [ebp-0x10] 456 // jo label 457 InstructionOperand const input = g.UseRegister(left); 458 inputs[input_count++] = input; 459 inputs[input_count++] = input; 460 } else if (g.CanBeImmediate(right)) { 461 inputs[input_count++] = g.UseRegister(left); 462 inputs[input_count++] = g.UseImmediate(right); 463 } else { 464 int effect_level = selector->GetEffectLevel(node); 465 if (cont->IsBranch()) { 466 effect_level = selector->GetEffectLevel( 467 cont->true_block()->PredecessorAt(0)->control_input()); 468 } 469 if (node->op()->HasProperty(Operator::kCommutative) && 470 g.CanBeBetterLeftOperand(right) && 471 (!g.CanBeBetterLeftOperand(left) || 472 !g.CanBeMemoryOperand(opcode, node, right, effect_level))) { 473 std::swap(left, right); 474 } 475 if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) { 476 inputs[input_count++] = g.UseRegister(left); 477 AddressingMode addressing_mode = 478 g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count); 479 opcode |= AddressingModeField::encode(addressing_mode); 480 } else { 481 inputs[input_count++] = g.UseRegister(left); 482 inputs[input_count++] = g.Use(right); 483 } 484 } 485 486 outputs[output_count++] = g.DefineSameAsFirst(node); 487 488 DCHECK_NE(0u, input_count); 489 DCHECK_EQ(1u, output_count); 490 DCHECK_GE(arraysize(inputs), input_count); 491 DCHECK_GE(arraysize(outputs), output_count); 492 493 selector->EmitWithContinuation(opcode, output_count, outputs, input_count, 494 inputs, cont); 495 } 496 497 498 // Shared routine for multiple binary operations. 499 void VisitBinop(InstructionSelector* selector, Node* node, 500 InstructionCode opcode) { 501 FlagsContinuation cont; 502 VisitBinop(selector, node, opcode, &cont); 503 } 504 505 } // namespace 506 507 void InstructionSelector::VisitWord32And(Node* node) { 508 VisitBinop(this, node, kIA32And); 509 } 510 511 512 void InstructionSelector::VisitWord32Or(Node* node) { 513 VisitBinop(this, node, kIA32Or); 514 } 515 516 517 void InstructionSelector::VisitWord32Xor(Node* node) { 518 IA32OperandGenerator g(this); 519 Int32BinopMatcher m(node); 520 if (m.right().Is(-1)) { 521 Emit(kIA32Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node())); 522 } else { 523 VisitBinop(this, node, kIA32Xor); 524 } 525 } 526 527 528 // Shared routine for multiple shift operations. 529 static inline void VisitShift(InstructionSelector* selector, Node* node, 530 ArchOpcode opcode) { 531 IA32OperandGenerator g(selector); 532 Node* left = node->InputAt(0); 533 Node* right = node->InputAt(1); 534 535 if (g.CanBeImmediate(right)) { 536 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left), 537 g.UseImmediate(right)); 538 } else { 539 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left), 540 g.UseFixed(right, ecx)); 541 } 542 } 543 544 545 namespace { 546 547 void VisitMulHigh(InstructionSelector* selector, Node* node, 548 ArchOpcode opcode) { 549 IA32OperandGenerator g(selector); 550 InstructionOperand temps[] = {g.TempRegister(eax)}; 551 selector->Emit( 552 opcode, g.DefineAsFixed(node, edx), g.UseFixed(node->InputAt(0), eax), 553 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps); 554 } 555 556 557 void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) { 558 IA32OperandGenerator g(selector); 559 InstructionOperand temps[] = {g.TempRegister(edx)}; 560 selector->Emit(opcode, g.DefineAsFixed(node, eax), 561 g.UseFixed(node->InputAt(0), eax), 562 g.UseUnique(node->InputAt(1)), arraysize(temps), temps); 563 } 564 565 566 void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) { 567 IA32OperandGenerator g(selector); 568 InstructionOperand temps[] = {g.TempRegister(eax)}; 569 selector->Emit(opcode, g.DefineAsFixed(node, edx), 570 g.UseFixed(node->InputAt(0), eax), 571 g.UseUnique(node->InputAt(1)), arraysize(temps), temps); 572 } 573 574 void EmitLea(InstructionSelector* selector, Node* result, Node* index, 575 int scale, Node* base, Node* displacement, 576 DisplacementMode displacement_mode) { 577 IA32OperandGenerator g(selector); 578 InstructionOperand inputs[4]; 579 size_t input_count = 0; 580 AddressingMode mode = 581 g.GenerateMemoryOperandInputs(index, scale, base, displacement, 582 displacement_mode, inputs, &input_count); 583 584 DCHECK_NE(0u, input_count); 585 DCHECK_GE(arraysize(inputs), input_count); 586 587 InstructionOperand outputs[1]; 588 outputs[0] = g.DefineAsRegister(result); 589 590 InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea; 591 592 selector->Emit(opcode, 1, outputs, input_count, inputs); 593 } 594 595 } // namespace 596 597 598 void InstructionSelector::VisitWord32Shl(Node* node) { 599 Int32ScaleMatcher m(node, true); 600 if (m.matches()) { 601 Node* index = node->InputAt(0); 602 Node* base = m.power_of_two_plus_one() ? index : nullptr; 603 EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement); 604 return; 605 } 606 VisitShift(this, node, kIA32Shl); 607 } 608 609 610 void InstructionSelector::VisitWord32Shr(Node* node) { 611 VisitShift(this, node, kIA32Shr); 612 } 613 614 615 void InstructionSelector::VisitWord32Sar(Node* node) { 616 VisitShift(this, node, kIA32Sar); 617 } 618 619 void InstructionSelector::VisitInt32PairAdd(Node* node) { 620 IA32OperandGenerator g(this); 621 622 Node* projection1 = NodeProperties::FindProjection(node, 1); 623 if (projection1) { 624 // We use UseUniqueRegister here to avoid register sharing with the temp 625 // register. 626 InstructionOperand inputs[] = { 627 g.UseRegister(node->InputAt(0)), g.UseUniqueRegister(node->InputAt(1)), 628 g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))}; 629 630 InstructionOperand outputs[] = {g.DefineSameAsFirst(node), 631 g.DefineAsRegister(projection1)}; 632 633 InstructionOperand temps[] = {g.TempRegister()}; 634 635 Emit(kIA32AddPair, 2, outputs, 4, inputs, 1, temps); 636 } else { 637 // The high word of the result is not used, so we emit the standard 32 bit 638 // instruction. 639 Emit(kIA32Add, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)), 640 g.Use(node->InputAt(2))); 641 } 642 } 643 644 void InstructionSelector::VisitInt32PairSub(Node* node) { 645 IA32OperandGenerator g(this); 646 647 Node* projection1 = NodeProperties::FindProjection(node, 1); 648 if (projection1) { 649 // We use UseUniqueRegister here to avoid register sharing with the temp 650 // register. 651 InstructionOperand inputs[] = { 652 g.UseRegister(node->InputAt(0)), g.UseUniqueRegister(node->InputAt(1)), 653 g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))}; 654 655 InstructionOperand outputs[] = {g.DefineSameAsFirst(node), 656 g.DefineAsRegister(projection1)}; 657 658 InstructionOperand temps[] = {g.TempRegister()}; 659 660 Emit(kIA32SubPair, 2, outputs, 4, inputs, 1, temps); 661 } else { 662 // The high word of the result is not used, so we emit the standard 32 bit 663 // instruction. 664 Emit(kIA32Sub, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)), 665 g.Use(node->InputAt(2))); 666 } 667 } 668 669 void InstructionSelector::VisitInt32PairMul(Node* node) { 670 IA32OperandGenerator g(this); 671 672 Node* projection1 = NodeProperties::FindProjection(node, 1); 673 if (projection1) { 674 // InputAt(3) explicitly shares ecx with OutputRegister(1) to save one 675 // register and one mov instruction. 676 InstructionOperand inputs[] = {g.UseUnique(node->InputAt(0)), 677 g.UseUnique(node->InputAt(1)), 678 g.UseUniqueRegister(node->InputAt(2)), 679 g.UseFixed(node->InputAt(3), ecx)}; 680 681 InstructionOperand outputs[] = { 682 g.DefineAsFixed(node, eax), 683 g.DefineAsFixed(NodeProperties::FindProjection(node, 1), ecx)}; 684 685 InstructionOperand temps[] = {g.TempRegister(edx)}; 686 687 Emit(kIA32MulPair, 2, outputs, 4, inputs, 1, temps); 688 } else { 689 // The high word of the result is not used, so we emit the standard 32 bit 690 // instruction. 691 Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)), 692 g.Use(node->InputAt(2))); 693 } 694 } 695 696 void VisitWord32PairShift(InstructionSelector* selector, InstructionCode opcode, 697 Node* node) { 698 IA32OperandGenerator g(selector); 699 700 Node* shift = node->InputAt(2); 701 InstructionOperand shift_operand; 702 if (g.CanBeImmediate(shift)) { 703 shift_operand = g.UseImmediate(shift); 704 } else { 705 shift_operand = g.UseFixed(shift, ecx); 706 } 707 InstructionOperand inputs[] = {g.UseFixed(node->InputAt(0), eax), 708 g.UseFixed(node->InputAt(1), edx), 709 shift_operand}; 710 711 InstructionOperand outputs[2]; 712 InstructionOperand temps[1]; 713 int32_t output_count = 0; 714 int32_t temp_count = 0; 715 outputs[output_count++] = g.DefineAsFixed(node, eax); 716 Node* projection1 = NodeProperties::FindProjection(node, 1); 717 if (projection1) { 718 outputs[output_count++] = g.DefineAsFixed(projection1, edx); 719 } else { 720 temps[temp_count++] = g.TempRegister(edx); 721 } 722 723 selector->Emit(opcode, output_count, outputs, 3, inputs, temp_count, temps); 724 } 725 726 void InstructionSelector::VisitWord32PairShl(Node* node) { 727 VisitWord32PairShift(this, kIA32ShlPair, node); 728 } 729 730 void InstructionSelector::VisitWord32PairShr(Node* node) { 731 VisitWord32PairShift(this, kIA32ShrPair, node); 732 } 733 734 void InstructionSelector::VisitWord32PairSar(Node* node) { 735 VisitWord32PairShift(this, kIA32SarPair, node); 736 } 737 738 void InstructionSelector::VisitWord32Ror(Node* node) { 739 VisitShift(this, node, kIA32Ror); 740 } 741 742 #define RO_OP_LIST(V) \ 743 V(Word32Clz, kIA32Lzcnt) \ 744 V(Word32Ctz, kIA32Tzcnt) \ 745 V(Word32Popcnt, kIA32Popcnt) \ 746 V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64) \ 747 V(RoundInt32ToFloat32, kSSEInt32ToFloat32) \ 748 V(ChangeInt32ToFloat64, kSSEInt32ToFloat64) \ 749 V(ChangeUint32ToFloat64, kSSEUint32ToFloat64) \ 750 V(TruncateFloat32ToInt32, kSSEFloat32ToInt32) \ 751 V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \ 752 V(ChangeFloat64ToInt32, kSSEFloat64ToInt32) \ 753 V(ChangeFloat64ToUint32, kSSEFloat64ToUint32) \ 754 V(TruncateFloat64ToUint32, kSSEFloat64ToUint32) \ 755 V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32) \ 756 V(RoundFloat64ToInt32, kSSEFloat64ToInt32) \ 757 V(BitcastFloat32ToInt32, kIA32BitcastFI) \ 758 V(BitcastInt32ToFloat32, kIA32BitcastIF) \ 759 V(Float32Sqrt, kSSEFloat32Sqrt) \ 760 V(Float64Sqrt, kSSEFloat64Sqrt) \ 761 V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32) \ 762 V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \ 763 V(SignExtendWord8ToInt32, kIA32Movsxbl) \ 764 V(SignExtendWord16ToInt32, kIA32Movsxwl) 765 766 #define RR_OP_LIST(V) \ 767 V(TruncateFloat64ToWord32, kArchTruncateDoubleToI) \ 768 V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown)) \ 769 V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown)) \ 770 V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp)) \ 771 V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp)) \ 772 V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \ 773 V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \ 774 V(Float32RoundTiesEven, \ 775 kSSEFloat32Round | MiscField::encode(kRoundToNearest)) \ 776 V(Float64RoundTiesEven, kSSEFloat64Round | MiscField::encode(kRoundToNearest)) 777 778 #define RRO_FLOAT_OP_LIST(V) \ 779 V(Float32Add, kAVXFloat32Add, kSSEFloat32Add) \ 780 V(Float64Add, kAVXFloat64Add, kSSEFloat64Add) \ 781 V(Float32Sub, kAVXFloat32Sub, kSSEFloat32Sub) \ 782 V(Float64Sub, kAVXFloat64Sub, kSSEFloat64Sub) \ 783 V(Float32Mul, kAVXFloat32Mul, kSSEFloat32Mul) \ 784 V(Float64Mul, kAVXFloat64Mul, kSSEFloat64Mul) \ 785 V(Float32Div, kAVXFloat32Div, kSSEFloat32Div) \ 786 V(Float64Div, kAVXFloat64Div, kSSEFloat64Div) 787 788 #define FLOAT_UNOP_LIST(V) \ 789 V(Float32Abs, kAVXFloat32Abs, kSSEFloat32Abs) \ 790 V(Float64Abs, kAVXFloat64Abs, kSSEFloat64Abs) \ 791 V(Float32Neg, kAVXFloat32Neg, kSSEFloat32Neg) \ 792 V(Float64Neg, kAVXFloat64Neg, kSSEFloat64Neg) 793 794 #define RO_VISITOR(Name, opcode) \ 795 void InstructionSelector::Visit##Name(Node* node) { \ 796 VisitRO(this, node, opcode); \ 797 } 798 RO_OP_LIST(RO_VISITOR) 799 #undef RO_VISITOR 800 #undef RO_OP_LIST 801 802 #define RR_VISITOR(Name, opcode) \ 803 void InstructionSelector::Visit##Name(Node* node) { \ 804 VisitRR(this, node, opcode); \ 805 } 806 RR_OP_LIST(RR_VISITOR) 807 #undef RR_VISITOR 808 #undef RR_OP_LIST 809 810 #define RRO_FLOAT_VISITOR(Name, avx, sse) \ 811 void InstructionSelector::Visit##Name(Node* node) { \ 812 VisitRROFloat(this, node, avx, sse); \ 813 } 814 RRO_FLOAT_OP_LIST(RRO_FLOAT_VISITOR) 815 #undef RRO_FLOAT_VISITOR 816 #undef RRO_FLOAT_OP_LIST 817 818 #define FLOAT_UNOP_VISITOR(Name, avx, sse) \ 819 void InstructionSelector::Visit##Name(Node* node) { \ 820 VisitFloatUnop(this, node, node->InputAt(0), avx, sse); \ 821 } 822 FLOAT_UNOP_LIST(FLOAT_UNOP_VISITOR) 823 #undef FLOAT_UNOP_VISITOR 824 #undef FLOAT_UNOP_LIST 825 826 void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); } 827 828 void InstructionSelector::VisitWord64ReverseBytes(Node* node) { UNREACHABLE(); } 829 830 void InstructionSelector::VisitWord32ReverseBytes(Node* node) { 831 IA32OperandGenerator g(this); 832 Emit(kIA32Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0))); 833 } 834 835 void InstructionSelector::VisitInt32Add(Node* node) { 836 IA32OperandGenerator g(this); 837 838 // Try to match the Add to a lea pattern 839 BaseWithIndexAndDisplacement32Matcher m(node); 840 if (m.matches() && 841 (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) { 842 InstructionOperand inputs[4]; 843 size_t input_count = 0; 844 AddressingMode mode = g.GenerateMemoryOperandInputs( 845 m.index(), m.scale(), m.base(), m.displacement(), m.displacement_mode(), 846 inputs, &input_count); 847 848 DCHECK_NE(0u, input_count); 849 DCHECK_GE(arraysize(inputs), input_count); 850 851 InstructionOperand outputs[1]; 852 outputs[0] = g.DefineAsRegister(node); 853 854 InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea; 855 Emit(opcode, 1, outputs, input_count, inputs); 856 return; 857 } 858 859 // No lea pattern match, use add 860 VisitBinop(this, node, kIA32Add); 861 } 862 863 864 void InstructionSelector::VisitInt32Sub(Node* node) { 865 IA32OperandGenerator g(this); 866 Int32BinopMatcher m(node); 867 if (m.left().Is(0)) { 868 Emit(kIA32Neg, g.DefineSameAsFirst(node), g.Use(m.right().node())); 869 } else { 870 VisitBinop(this, node, kIA32Sub); 871 } 872 } 873 874 875 void InstructionSelector::VisitInt32Mul(Node* node) { 876 Int32ScaleMatcher m(node, true); 877 if (m.matches()) { 878 Node* index = node->InputAt(0); 879 Node* base = m.power_of_two_plus_one() ? index : nullptr; 880 EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement); 881 return; 882 } 883 IA32OperandGenerator g(this); 884 Node* left = node->InputAt(0); 885 Node* right = node->InputAt(1); 886 if (g.CanBeImmediate(right)) { 887 Emit(kIA32Imul, g.DefineAsRegister(node), g.Use(left), 888 g.UseImmediate(right)); 889 } else { 890 if (g.CanBeBetterLeftOperand(right)) { 891 std::swap(left, right); 892 } 893 Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(left), 894 g.Use(right)); 895 } 896 } 897 898 899 void InstructionSelector::VisitInt32MulHigh(Node* node) { 900 VisitMulHigh(this, node, kIA32ImulHigh); 901 } 902 903 904 void InstructionSelector::VisitUint32MulHigh(Node* node) { 905 VisitMulHigh(this, node, kIA32UmulHigh); 906 } 907 908 909 void InstructionSelector::VisitInt32Div(Node* node) { 910 VisitDiv(this, node, kIA32Idiv); 911 } 912 913 914 void InstructionSelector::VisitUint32Div(Node* node) { 915 VisitDiv(this, node, kIA32Udiv); 916 } 917 918 919 void InstructionSelector::VisitInt32Mod(Node* node) { 920 VisitMod(this, node, kIA32Idiv); 921 } 922 923 924 void InstructionSelector::VisitUint32Mod(Node* node) { 925 VisitMod(this, node, kIA32Udiv); 926 } 927 928 929 void InstructionSelector::VisitRoundUint32ToFloat32(Node* node) { 930 IA32OperandGenerator g(this); 931 InstructionOperand temps[] = {g.TempRegister()}; 932 Emit(kSSEUint32ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)), 933 arraysize(temps), temps); 934 } 935 936 void InstructionSelector::VisitFloat64Mod(Node* node) { 937 IA32OperandGenerator g(this); 938 InstructionOperand temps[] = {g.TempRegister(eax)}; 939 Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node), 940 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), 1, 941 temps); 942 } 943 944 void InstructionSelector::VisitFloat32Max(Node* node) { 945 IA32OperandGenerator g(this); 946 InstructionOperand temps[] = {g.TempRegister()}; 947 Emit(kSSEFloat32Max, g.DefineSameAsFirst(node), 948 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)), 949 arraysize(temps), temps); 950 } 951 952 void InstructionSelector::VisitFloat64Max(Node* node) { 953 IA32OperandGenerator g(this); 954 InstructionOperand temps[] = {g.TempRegister()}; 955 Emit(kSSEFloat64Max, g.DefineSameAsFirst(node), 956 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)), 957 arraysize(temps), temps); 958 } 959 960 void InstructionSelector::VisitFloat32Min(Node* node) { 961 IA32OperandGenerator g(this); 962 InstructionOperand temps[] = {g.TempRegister()}; 963 Emit(kSSEFloat32Min, g.DefineSameAsFirst(node), 964 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)), 965 arraysize(temps), temps); 966 } 967 968 void InstructionSelector::VisitFloat64Min(Node* node) { 969 IA32OperandGenerator g(this); 970 InstructionOperand temps[] = {g.TempRegister()}; 971 Emit(kSSEFloat64Min, g.DefineSameAsFirst(node), 972 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)), 973 arraysize(temps), temps); 974 } 975 976 void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) { 977 UNREACHABLE(); 978 } 979 980 void InstructionSelector::VisitFloat64Ieee754Binop(Node* node, 981 InstructionCode opcode) { 982 IA32OperandGenerator g(this); 983 Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)), 984 g.UseRegister(node->InputAt(1))) 985 ->MarkAsCall(); 986 } 987 988 void InstructionSelector::VisitFloat64Ieee754Unop(Node* node, 989 InstructionCode opcode) { 990 IA32OperandGenerator g(this); 991 Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0))) 992 ->MarkAsCall(); 993 } 994 995 void InstructionSelector::EmitPrepareArguments( 996 ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor, 997 Node* node) { 998 IA32OperandGenerator g(this); 999 1000 // Prepare for C function call. 1001 if (call_descriptor->IsCFunctionCall()) { 1002 InstructionOperand temps[] = {g.TempRegister()}; 1003 size_t const temp_count = arraysize(temps); 1004 Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>( 1005 call_descriptor->ParameterCount())), 1006 0, nullptr, 0, nullptr, temp_count, temps); 1007 1008 // Poke any stack arguments. 1009 for (size_t n = 0; n < arguments->size(); ++n) { 1010 PushParameter input = (*arguments)[n]; 1011 if (input.node) { 1012 int const slot = static_cast<int>(n); 1013 InstructionOperand value = g.CanBeImmediate(node) 1014 ? g.UseImmediate(input.node) 1015 : g.UseRegister(input.node); 1016 Emit(kIA32Poke | MiscField::encode(slot), g.NoOutput(), value); 1017 } 1018 } 1019 } else { 1020 // Push any stack arguments. 1021 int effect_level = GetEffectLevel(node); 1022 for (PushParameter input : base::Reversed(*arguments)) { 1023 // Skip any alignment holes in pushed nodes. 1024 if (input.node == nullptr) continue; 1025 if (g.CanBeMemoryOperand(kIA32Push, node, input.node, effect_level)) { 1026 InstructionOperand outputs[1]; 1027 InstructionOperand inputs[4]; 1028 size_t input_count = 0; 1029 InstructionCode opcode = kIA32Push; 1030 AddressingMode mode = g.GetEffectiveAddressMemoryOperand( 1031 input.node, inputs, &input_count); 1032 opcode |= AddressingModeField::encode(mode); 1033 Emit(opcode, 0, outputs, input_count, inputs); 1034 } else { 1035 InstructionOperand value = 1036 g.CanBeImmediate(input.node) 1037 ? g.UseImmediate(input.node) 1038 : IsSupported(ATOM) || 1039 sequence()->IsFP(GetVirtualRegister(input.node)) 1040 ? g.UseRegister(input.node) 1041 : g.Use(input.node); 1042 if (input.location.GetType() == MachineType::Float32()) { 1043 Emit(kIA32PushFloat32, g.NoOutput(), value); 1044 } else if (input.location.GetType() == MachineType::Float64()) { 1045 Emit(kIA32PushFloat64, g.NoOutput(), value); 1046 } else if (input.location.GetType() == MachineType::Simd128()) { 1047 Emit(kIA32PushSimd128, g.NoOutput(), value); 1048 } else { 1049 Emit(kIA32Push, g.NoOutput(), value); 1050 } 1051 } 1052 } 1053 } 1054 } 1055 1056 void InstructionSelector::EmitPrepareResults( 1057 ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor, 1058 Node* node) { 1059 IA32OperandGenerator g(this); 1060 1061 int reverse_slot = 0; 1062 for (PushParameter output : *results) { 1063 if (!output.location.IsCallerFrameSlot()) continue; 1064 // Skip any alignment holes in nodes. 1065 if (output.node != nullptr) { 1066 DCHECK(!call_descriptor->IsCFunctionCall()); 1067 if (output.location.GetType() == MachineType::Float32()) { 1068 MarkAsFloat32(output.node); 1069 } else if (output.location.GetType() == MachineType::Float64()) { 1070 MarkAsFloat64(output.node); 1071 } 1072 Emit(kIA32Peek, g.DefineAsRegister(output.node), 1073 g.UseImmediate(reverse_slot)); 1074 } 1075 reverse_slot += output.location.GetSizeInPointers(); 1076 } 1077 } 1078 1079 1080 bool InstructionSelector::IsTailCallAddressImmediate() { return true; } 1081 1082 int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 0; } 1083 1084 namespace { 1085 1086 void VisitCompareWithMemoryOperand(InstructionSelector* selector, 1087 InstructionCode opcode, Node* left, 1088 InstructionOperand right, 1089 FlagsContinuation* cont) { 1090 DCHECK_EQ(IrOpcode::kLoad, left->opcode()); 1091 IA32OperandGenerator g(selector); 1092 size_t input_count = 0; 1093 InstructionOperand inputs[4]; 1094 AddressingMode addressing_mode = 1095 g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count); 1096 opcode |= AddressingModeField::encode(addressing_mode); 1097 inputs[input_count++] = right; 1098 1099 selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont); 1100 } 1101 1102 // Shared routine for multiple compare operations. 1103 void VisitCompare(InstructionSelector* selector, InstructionCode opcode, 1104 InstructionOperand left, InstructionOperand right, 1105 FlagsContinuation* cont) { 1106 selector->EmitWithContinuation(opcode, left, right, cont); 1107 } 1108 1109 1110 // Shared routine for multiple compare operations. 1111 void VisitCompare(InstructionSelector* selector, InstructionCode opcode, 1112 Node* left, Node* right, FlagsContinuation* cont, 1113 bool commutative) { 1114 IA32OperandGenerator g(selector); 1115 if (commutative && g.CanBeBetterLeftOperand(right)) { 1116 std::swap(left, right); 1117 } 1118 VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont); 1119 } 1120 1121 MachineType MachineTypeForNarrow(Node* node, Node* hint_node) { 1122 if (hint_node->opcode() == IrOpcode::kLoad) { 1123 MachineType hint = LoadRepresentationOf(hint_node->op()); 1124 if (node->opcode() == IrOpcode::kInt32Constant || 1125 node->opcode() == IrOpcode::kInt64Constant) { 1126 int64_t constant = node->opcode() == IrOpcode::kInt32Constant 1127 ? OpParameter<int32_t>(node->op()) 1128 : OpParameter<int64_t>(node->op()); 1129 if (hint == MachineType::Int8()) { 1130 if (constant >= std::numeric_limits<int8_t>::min() && 1131 constant <= std::numeric_limits<int8_t>::max()) { 1132 return hint; 1133 } 1134 } else if (hint == MachineType::Uint8()) { 1135 if (constant >= std::numeric_limits<uint8_t>::min() && 1136 constant <= std::numeric_limits<uint8_t>::max()) { 1137 return hint; 1138 } 1139 } else if (hint == MachineType::Int16()) { 1140 if (constant >= std::numeric_limits<int16_t>::min() && 1141 constant <= std::numeric_limits<int16_t>::max()) { 1142 return hint; 1143 } 1144 } else if (hint == MachineType::Uint16()) { 1145 if (constant >= std::numeric_limits<uint16_t>::min() && 1146 constant <= std::numeric_limits<uint16_t>::max()) { 1147 return hint; 1148 } 1149 } else if (hint == MachineType::Int32()) { 1150 return hint; 1151 } else if (hint == MachineType::Uint32()) { 1152 if (constant >= 0) return hint; 1153 } 1154 } 1155 } 1156 return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op()) 1157 : MachineType::None(); 1158 } 1159 1160 // Tries to match the size of the given opcode to that of the operands, if 1161 // possible. 1162 InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left, 1163 Node* right, FlagsContinuation* cont) { 1164 // TODO(epertoso): we can probably get some size information out of phi nodes. 1165 // If the load representations don't match, both operands will be 1166 // zero/sign-extended to 32bit. 1167 MachineType left_type = MachineTypeForNarrow(left, right); 1168 MachineType right_type = MachineTypeForNarrow(right, left); 1169 if (left_type == right_type) { 1170 switch (left_type.representation()) { 1171 case MachineRepresentation::kBit: 1172 case MachineRepresentation::kWord8: { 1173 if (opcode == kIA32Test) return kIA32Test8; 1174 if (opcode == kIA32Cmp) { 1175 if (left_type.semantic() == MachineSemantic::kUint32) { 1176 cont->OverwriteUnsignedIfSigned(); 1177 } else { 1178 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic()); 1179 } 1180 return kIA32Cmp8; 1181 } 1182 break; 1183 } 1184 case MachineRepresentation::kWord16: 1185 if (opcode == kIA32Test) return kIA32Test16; 1186 if (opcode == kIA32Cmp) { 1187 if (left_type.semantic() == MachineSemantic::kUint32) { 1188 cont->OverwriteUnsignedIfSigned(); 1189 } else { 1190 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic()); 1191 } 1192 return kIA32Cmp16; 1193 } 1194 break; 1195 default: 1196 break; 1197 } 1198 } 1199 return opcode; 1200 } 1201 1202 // Shared routine for multiple float32 compare operations (inputs commuted). 1203 void VisitFloat32Compare(InstructionSelector* selector, Node* node, 1204 FlagsContinuation* cont) { 1205 Node* const left = node->InputAt(0); 1206 Node* const right = node->InputAt(1); 1207 VisitCompare(selector, kSSEFloat32Cmp, right, left, cont, false); 1208 } 1209 1210 1211 // Shared routine for multiple float64 compare operations (inputs commuted). 1212 void VisitFloat64Compare(InstructionSelector* selector, Node* node, 1213 FlagsContinuation* cont) { 1214 Node* const left = node->InputAt(0); 1215 Node* const right = node->InputAt(1); 1216 VisitCompare(selector, kSSEFloat64Cmp, right, left, cont, false); 1217 } 1218 1219 // Shared routine for multiple word compare operations. 1220 void VisitWordCompare(InstructionSelector* selector, Node* node, 1221 InstructionCode opcode, FlagsContinuation* cont) { 1222 IA32OperandGenerator g(selector); 1223 Node* left = node->InputAt(0); 1224 Node* right = node->InputAt(1); 1225 1226 InstructionCode narrowed_opcode = 1227 TryNarrowOpcodeSize(opcode, left, right, cont); 1228 1229 int effect_level = selector->GetEffectLevel(node); 1230 if (cont->IsBranch()) { 1231 effect_level = selector->GetEffectLevel( 1232 cont->true_block()->PredecessorAt(0)->control_input()); 1233 } 1234 1235 // If one of the two inputs is an immediate, make sure it's on the right, or 1236 // if one of the two inputs is a memory operand, make sure it's on the left. 1237 if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) || 1238 (g.CanBeMemoryOperand(narrowed_opcode, node, right, effect_level) && 1239 !g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level))) { 1240 if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute(); 1241 std::swap(left, right); 1242 } 1243 1244 // Match immediates on right side of comparison. 1245 if (g.CanBeImmediate(right)) { 1246 if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) { 1247 return VisitCompareWithMemoryOperand(selector, narrowed_opcode, left, 1248 g.UseImmediate(right), cont); 1249 } 1250 return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right), 1251 cont); 1252 } 1253 1254 // Match memory operands on left side of comparison. 1255 if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) { 1256 bool needs_byte_register = 1257 narrowed_opcode == kIA32Test8 || narrowed_opcode == kIA32Cmp8; 1258 return VisitCompareWithMemoryOperand( 1259 selector, narrowed_opcode, left, 1260 needs_byte_register ? g.UseByteRegister(right) : g.UseRegister(right), 1261 cont); 1262 } 1263 1264 return VisitCompare(selector, opcode, left, right, cont, 1265 node->op()->HasProperty(Operator::kCommutative)); 1266 } 1267 1268 void VisitWordCompare(InstructionSelector* selector, Node* node, 1269 FlagsContinuation* cont) { 1270 StackCheckMatcher<Int32BinopMatcher, IrOpcode::kUint32LessThan> m( 1271 selector->isolate(), node); 1272 if (m.Matched()) { 1273 // Compare(Load(js_stack_limit), LoadStackPointer) 1274 if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute(); 1275 InstructionCode opcode = cont->Encode(kIA32StackCheck); 1276 CHECK(cont->IsBranch()); 1277 selector->EmitWithContinuation(opcode, cont); 1278 return; 1279 } 1280 WasmStackCheckMatcher<Int32BinopMatcher, IrOpcode::kUint32LessThan> wasm_m( 1281 node); 1282 if (wasm_m.Matched()) { 1283 // This is a wasm stack check. By structure, we know that we can use the 1284 // stack pointer directly, as wasm code does not modify the stack at points 1285 // where stack checks are performed. 1286 Node* left = node->InputAt(0); 1287 LocationOperand esp(InstructionOperand::EXPLICIT, LocationOperand::REGISTER, 1288 InstructionSequence::DefaultRepresentation(), 1289 RegisterCode::kRegCode_esp); 1290 return VisitCompareWithMemoryOperand(selector, kIA32Cmp, left, esp, cont); 1291 } 1292 VisitWordCompare(selector, node, kIA32Cmp, cont); 1293 } 1294 1295 void VisitAtomicExchange(InstructionSelector* selector, Node* node, 1296 ArchOpcode opcode, MachineRepresentation rep) { 1297 IA32OperandGenerator g(selector); 1298 Node* base = node->InputAt(0); 1299 Node* index = node->InputAt(1); 1300 Node* value = node->InputAt(2); 1301 1302 AddressingMode addressing_mode; 1303 InstructionOperand value_operand = (rep == MachineRepresentation::kWord8) 1304 ? g.UseFixed(value, edx) 1305 : g.UseUniqueRegister(value); 1306 InstructionOperand inputs[] = { 1307 value_operand, g.UseUniqueRegister(base), 1308 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1309 InstructionOperand outputs[] = { 1310 (rep == MachineRepresentation::kWord8) 1311 // Using DefineSameAsFirst requires the register to be unallocated. 1312 ? g.DefineAsFixed(node, edx) 1313 : g.DefineSameAsFirst(node)}; 1314 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1315 selector->Emit(code, 1, outputs, arraysize(inputs), inputs); 1316 } 1317 1318 void VisitAtomicBinOp(InstructionSelector* selector, Node* node, 1319 ArchOpcode opcode, MachineRepresentation rep) { 1320 AddressingMode addressing_mode; 1321 IA32OperandGenerator g(selector); 1322 Node* base = node->InputAt(0); 1323 Node* index = node->InputAt(1); 1324 Node* value = node->InputAt(2); 1325 InstructionOperand inputs[] = { 1326 g.UseUniqueRegister(value), g.UseUniqueRegister(base), 1327 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1328 InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)}; 1329 InstructionOperand temp[] = {(rep == MachineRepresentation::kWord8) 1330 ? g.UseByteRegister(node) 1331 : g.TempRegister()}; 1332 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1333 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, 1334 arraysize(temp), temp); 1335 } 1336 1337 void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node, 1338 ArchOpcode opcode) { 1339 IA32OperandGenerator g(selector); 1340 Node* base = node->InputAt(0); 1341 Node* index = node->InputAt(1); 1342 Node* value = node->InputAt(2); 1343 // For Word64 operations, the value input is split into the a high node, 1344 // and a low node in the int64-lowering phase. 1345 Node* value_high = node->InputAt(3); 1346 1347 // Wasm lives in 32-bit address space, so we do not need to worry about 1348 // base/index lowering. This will need to be fixed for Wasm64. 1349 AddressingMode addressing_mode; 1350 InstructionOperand inputs[] = { 1351 g.UseFixed(value, ebx), g.UseFixed(value_high, ecx), 1352 g.UseUniqueRegister(base), 1353 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1354 InstructionOperand outputs[] = { 1355 g.DefineAsFixed(NodeProperties::FindProjection(node, 0), eax), 1356 g.DefineAsFixed(NodeProperties::FindProjection(node, 1), edx)}; 1357 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1358 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs); 1359 } 1360 1361 void VisitNarrowAtomicBinOp(InstructionSelector* selector, Node* node, 1362 ArchOpcode opcode, MachineType type) { 1363 IA32OperandGenerator g(selector); 1364 Node* base = node->InputAt(0); 1365 Node* index = node->InputAt(1); 1366 Node* value = node->InputAt(2); 1367 1368 // Wasm lives in 32-bit address space, so we do not need to worry about 1369 // base/index lowering. This will need to be fixed for Wasm64. 1370 AddressingMode addressing_mode; 1371 InstructionOperand inputs[] = { 1372 g.UseUniqueRegister(value), g.UseUniqueRegister(base), 1373 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1374 InstructionOperand outputs[] = { 1375 g.DefineAsFixed(NodeProperties::FindProjection(node, 0), eax), 1376 g.DefineAsFixed(NodeProperties::FindProjection(node, 1), edx)}; 1377 InstructionOperand temp[] = {(type == MachineType::Uint8()) 1378 ? g.UseByteRegister(node) 1379 : g.TempRegister()}; 1380 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1381 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, 1382 arraysize(temp), temp); 1383 } 1384 1385 } // namespace 1386 1387 // Shared routine for word comparison with zero. 1388 void InstructionSelector::VisitWordCompareZero(Node* user, Node* value, 1389 FlagsContinuation* cont) { 1390 // Try to combine with comparisons against 0 by simply inverting the branch. 1391 while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) { 1392 Int32BinopMatcher m(value); 1393 if (!m.right().Is(0)) break; 1394 1395 user = value; 1396 value = m.left().node(); 1397 cont->Negate(); 1398 } 1399 1400 if (CanCover(user, value)) { 1401 switch (value->opcode()) { 1402 case IrOpcode::kWord32Equal: 1403 cont->OverwriteAndNegateIfEqual(kEqual); 1404 return VisitWordCompare(this, value, cont); 1405 case IrOpcode::kInt32LessThan: 1406 cont->OverwriteAndNegateIfEqual(kSignedLessThan); 1407 return VisitWordCompare(this, value, cont); 1408 case IrOpcode::kInt32LessThanOrEqual: 1409 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual); 1410 return VisitWordCompare(this, value, cont); 1411 case IrOpcode::kUint32LessThan: 1412 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan); 1413 return VisitWordCompare(this, value, cont); 1414 case IrOpcode::kUint32LessThanOrEqual: 1415 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual); 1416 return VisitWordCompare(this, value, cont); 1417 case IrOpcode::kFloat32Equal: 1418 cont->OverwriteAndNegateIfEqual(kUnorderedEqual); 1419 return VisitFloat32Compare(this, value, cont); 1420 case IrOpcode::kFloat32LessThan: 1421 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan); 1422 return VisitFloat32Compare(this, value, cont); 1423 case IrOpcode::kFloat32LessThanOrEqual: 1424 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual); 1425 return VisitFloat32Compare(this, value, cont); 1426 case IrOpcode::kFloat64Equal: 1427 cont->OverwriteAndNegateIfEqual(kUnorderedEqual); 1428 return VisitFloat64Compare(this, value, cont); 1429 case IrOpcode::kFloat64LessThan: 1430 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan); 1431 return VisitFloat64Compare(this, value, cont); 1432 case IrOpcode::kFloat64LessThanOrEqual: 1433 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual); 1434 return VisitFloat64Compare(this, value, cont); 1435 case IrOpcode::kProjection: 1436 // Check if this is the overflow output projection of an 1437 // <Operation>WithOverflow node. 1438 if (ProjectionIndexOf(value->op()) == 1u) { 1439 // We cannot combine the <Operation>WithOverflow with this branch 1440 // unless the 0th projection (the use of the actual value of the 1441 // <Operation> is either nullptr, which means there's no use of the 1442 // actual value, or was already defined, which means it is scheduled 1443 // *AFTER* this branch). 1444 Node* const node = value->InputAt(0); 1445 Node* const result = NodeProperties::FindProjection(node, 0); 1446 if (result == nullptr || IsDefined(result)) { 1447 switch (node->opcode()) { 1448 case IrOpcode::kInt32AddWithOverflow: 1449 cont->OverwriteAndNegateIfEqual(kOverflow); 1450 return VisitBinop(this, node, kIA32Add, cont); 1451 case IrOpcode::kInt32SubWithOverflow: 1452 cont->OverwriteAndNegateIfEqual(kOverflow); 1453 return VisitBinop(this, node, kIA32Sub, cont); 1454 case IrOpcode::kInt32MulWithOverflow: 1455 cont->OverwriteAndNegateIfEqual(kOverflow); 1456 return VisitBinop(this, node, kIA32Imul, cont); 1457 default: 1458 break; 1459 } 1460 } 1461 } 1462 break; 1463 case IrOpcode::kInt32Sub: 1464 return VisitWordCompare(this, value, cont); 1465 case IrOpcode::kWord32And: 1466 return VisitWordCompare(this, value, kIA32Test, cont); 1467 default: 1468 break; 1469 } 1470 } 1471 1472 // Continuation could not be combined with a compare, emit compare against 0. 1473 IA32OperandGenerator g(this); 1474 VisitCompare(this, kIA32Cmp, g.Use(value), g.TempImmediate(0), cont); 1475 } 1476 1477 void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) { 1478 IA32OperandGenerator g(this); 1479 InstructionOperand value_operand = g.UseRegister(node->InputAt(0)); 1480 1481 // Emit either ArchTableSwitch or ArchLookupSwitch. 1482 if (enable_switch_jump_table_ == kEnableSwitchJumpTable) { 1483 static const size_t kMaxTableSwitchValueRange = 2 << 16; 1484 size_t table_space_cost = 4 + sw.value_range(); 1485 size_t table_time_cost = 3; 1486 size_t lookup_space_cost = 3 + 2 * sw.case_count(); 1487 size_t lookup_time_cost = sw.case_count(); 1488 if (sw.case_count() > 4 && 1489 table_space_cost + 3 * table_time_cost <= 1490 lookup_space_cost + 3 * lookup_time_cost && 1491 sw.min_value() > std::numeric_limits<int32_t>::min() && 1492 sw.value_range() <= kMaxTableSwitchValueRange) { 1493 InstructionOperand index_operand = value_operand; 1494 if (sw.min_value()) { 1495 index_operand = g.TempRegister(); 1496 Emit(kIA32Lea | AddressingModeField::encode(kMode_MRI), index_operand, 1497 value_operand, g.TempImmediate(-sw.min_value())); 1498 } 1499 // Generate a table lookup. 1500 return EmitTableSwitch(sw, index_operand); 1501 } 1502 } 1503 1504 // Generate a tree of conditional jumps. 1505 return EmitBinarySearchSwitch(sw, value_operand); 1506 } 1507 1508 1509 void InstructionSelector::VisitWord32Equal(Node* const node) { 1510 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node); 1511 Int32BinopMatcher m(node); 1512 if (m.right().Is(0)) { 1513 return VisitWordCompareZero(m.node(), m.left().node(), &cont); 1514 } 1515 VisitWordCompare(this, node, &cont); 1516 } 1517 1518 1519 void InstructionSelector::VisitInt32LessThan(Node* node) { 1520 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node); 1521 VisitWordCompare(this, node, &cont); 1522 } 1523 1524 1525 void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) { 1526 FlagsContinuation cont = 1527 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node); 1528 VisitWordCompare(this, node, &cont); 1529 } 1530 1531 1532 void InstructionSelector::VisitUint32LessThan(Node* node) { 1533 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node); 1534 VisitWordCompare(this, node, &cont); 1535 } 1536 1537 1538 void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) { 1539 FlagsContinuation cont = 1540 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node); 1541 VisitWordCompare(this, node, &cont); 1542 } 1543 1544 1545 void InstructionSelector::VisitInt32AddWithOverflow(Node* node) { 1546 if (Node* ovf = NodeProperties::FindProjection(node, 1)) { 1547 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf); 1548 return VisitBinop(this, node, kIA32Add, &cont); 1549 } 1550 FlagsContinuation cont; 1551 VisitBinop(this, node, kIA32Add, &cont); 1552 } 1553 1554 1555 void InstructionSelector::VisitInt32SubWithOverflow(Node* node) { 1556 if (Node* ovf = NodeProperties::FindProjection(node, 1)) { 1557 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf); 1558 return VisitBinop(this, node, kIA32Sub, &cont); 1559 } 1560 FlagsContinuation cont; 1561 VisitBinop(this, node, kIA32Sub, &cont); 1562 } 1563 1564 void InstructionSelector::VisitInt32MulWithOverflow(Node* node) { 1565 if (Node* ovf = NodeProperties::FindProjection(node, 1)) { 1566 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf); 1567 return VisitBinop(this, node, kIA32Imul, &cont); 1568 } 1569 FlagsContinuation cont; 1570 VisitBinop(this, node, kIA32Imul, &cont); 1571 } 1572 1573 void InstructionSelector::VisitFloat32Equal(Node* node) { 1574 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node); 1575 VisitFloat32Compare(this, node, &cont); 1576 } 1577 1578 1579 void InstructionSelector::VisitFloat32LessThan(Node* node) { 1580 FlagsContinuation cont = 1581 FlagsContinuation::ForSet(kUnsignedGreaterThan, node); 1582 VisitFloat32Compare(this, node, &cont); 1583 } 1584 1585 1586 void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) { 1587 FlagsContinuation cont = 1588 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node); 1589 VisitFloat32Compare(this, node, &cont); 1590 } 1591 1592 1593 void InstructionSelector::VisitFloat64Equal(Node* node) { 1594 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node); 1595 VisitFloat64Compare(this, node, &cont); 1596 } 1597 1598 1599 void InstructionSelector::VisitFloat64LessThan(Node* node) { 1600 FlagsContinuation cont = 1601 FlagsContinuation::ForSet(kUnsignedGreaterThan, node); 1602 VisitFloat64Compare(this, node, &cont); 1603 } 1604 1605 1606 void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) { 1607 FlagsContinuation cont = 1608 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node); 1609 VisitFloat64Compare(this, node, &cont); 1610 } 1611 1612 1613 1614 void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) { 1615 IA32OperandGenerator g(this); 1616 Node* left = node->InputAt(0); 1617 Node* right = node->InputAt(1); 1618 Float64Matcher mleft(left); 1619 if (mleft.HasValue() && (bit_cast<uint64_t>(mleft.Value()) >> 32) == 0u) { 1620 Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right)); 1621 return; 1622 } 1623 Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node), 1624 g.UseRegister(left), g.Use(right)); 1625 } 1626 1627 1628 void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) { 1629 IA32OperandGenerator g(this); 1630 Node* left = node->InputAt(0); 1631 Node* right = node->InputAt(1); 1632 Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node), 1633 g.UseRegister(left), g.Use(right)); 1634 } 1635 1636 void InstructionSelector::VisitFloat64SilenceNaN(Node* node) { 1637 IA32OperandGenerator g(this); 1638 Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node), 1639 g.UseRegister(node->InputAt(0))); 1640 } 1641 1642 void InstructionSelector::VisitWord32AtomicLoad(Node* node) { 1643 LoadRepresentation load_rep = LoadRepresentationOf(node->op()); 1644 DCHECK(load_rep.representation() == MachineRepresentation::kWord8 || 1645 load_rep.representation() == MachineRepresentation::kWord16 || 1646 load_rep.representation() == MachineRepresentation::kWord32); 1647 USE(load_rep); 1648 VisitLoad(node); 1649 } 1650 1651 void InstructionSelector::VisitWord32AtomicStore(Node* node) { 1652 IA32OperandGenerator g(this); 1653 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op()); 1654 ArchOpcode opcode = kArchNop; 1655 switch (rep) { 1656 case MachineRepresentation::kWord8: 1657 opcode = kWord32AtomicExchangeInt8; 1658 break; 1659 case MachineRepresentation::kWord16: 1660 opcode = kWord32AtomicExchangeInt16; 1661 break; 1662 case MachineRepresentation::kWord32: 1663 opcode = kWord32AtomicExchangeWord32; 1664 break; 1665 default: 1666 UNREACHABLE(); 1667 break; 1668 } 1669 VisitAtomicExchange(this, node, opcode, rep); 1670 } 1671 1672 void InstructionSelector::VisitWord32AtomicExchange(Node* node) { 1673 IA32OperandGenerator g(this); 1674 MachineType type = AtomicOpType(node->op()); 1675 ArchOpcode opcode = kArchNop; 1676 if (type == MachineType::Int8()) { 1677 opcode = kWord32AtomicExchangeInt8; 1678 } else if (type == MachineType::Uint8()) { 1679 opcode = kWord32AtomicExchangeUint8; 1680 } else if (type == MachineType::Int16()) { 1681 opcode = kWord32AtomicExchangeInt16; 1682 } else if (type == MachineType::Uint16()) { 1683 opcode = kWord32AtomicExchangeUint16; 1684 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) { 1685 opcode = kWord32AtomicExchangeWord32; 1686 } else { 1687 UNREACHABLE(); 1688 return; 1689 } 1690 VisitAtomicExchange(this, node, opcode, type.representation()); 1691 } 1692 1693 void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) { 1694 IA32OperandGenerator g(this); 1695 Node* base = node->InputAt(0); 1696 Node* index = node->InputAt(1); 1697 Node* old_value = node->InputAt(2); 1698 Node* new_value = node->InputAt(3); 1699 1700 MachineType type = AtomicOpType(node->op()); 1701 ArchOpcode opcode = kArchNop; 1702 if (type == MachineType::Int8()) { 1703 opcode = kWord32AtomicCompareExchangeInt8; 1704 } else if (type == MachineType::Uint8()) { 1705 opcode = kWord32AtomicCompareExchangeUint8; 1706 } else if (type == MachineType::Int16()) { 1707 opcode = kWord32AtomicCompareExchangeInt16; 1708 } else if (type == MachineType::Uint16()) { 1709 opcode = kWord32AtomicCompareExchangeUint16; 1710 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) { 1711 opcode = kWord32AtomicCompareExchangeWord32; 1712 } else { 1713 UNREACHABLE(); 1714 return; 1715 } 1716 AddressingMode addressing_mode; 1717 InstructionOperand new_val_operand = 1718 (type.representation() == MachineRepresentation::kWord8) 1719 ? g.UseByteRegister(new_value) 1720 : g.UseUniqueRegister(new_value); 1721 InstructionOperand inputs[] = { 1722 g.UseFixed(old_value, eax), new_val_operand, g.UseUniqueRegister(base), 1723 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1724 InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)}; 1725 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1726 Emit(code, 1, outputs, arraysize(inputs), inputs); 1727 } 1728 1729 void InstructionSelector::VisitWord32AtomicBinaryOperation( 1730 Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op, 1731 ArchOpcode uint16_op, ArchOpcode word32_op) { 1732 MachineType type = AtomicOpType(node->op()); 1733 ArchOpcode opcode = kArchNop; 1734 if (type == MachineType::Int8()) { 1735 opcode = int8_op; 1736 } else if (type == MachineType::Uint8()) { 1737 opcode = uint8_op; 1738 } else if (type == MachineType::Int16()) { 1739 opcode = int16_op; 1740 } else if (type == MachineType::Uint16()) { 1741 opcode = uint16_op; 1742 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) { 1743 opcode = word32_op; 1744 } else { 1745 UNREACHABLE(); 1746 return; 1747 } 1748 VisitAtomicBinOp(this, node, opcode, type.representation()); 1749 } 1750 1751 #define VISIT_ATOMIC_BINOP(op) \ 1752 void InstructionSelector::VisitWord32Atomic##op(Node* node) { \ 1753 VisitWord32AtomicBinaryOperation( \ 1754 node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \ 1755 kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \ 1756 kWord32Atomic##op##Word32); \ 1757 } 1758 VISIT_ATOMIC_BINOP(Add) 1759 VISIT_ATOMIC_BINOP(Sub) 1760 VISIT_ATOMIC_BINOP(And) 1761 VISIT_ATOMIC_BINOP(Or) 1762 VISIT_ATOMIC_BINOP(Xor) 1763 #undef VISIT_ATOMIC_BINOP 1764 1765 void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) { 1766 IA32OperandGenerator g(this); 1767 AddressingMode mode; 1768 Node* base = node->InputAt(0); 1769 Node* index = node->InputAt(1); 1770 InstructionOperand inputs[] = {g.UseUniqueRegister(base), 1771 g.GetEffectiveIndexOperand(index, &mode)}; 1772 InstructionOperand temps[] = {g.TempDoubleRegister()}; 1773 InstructionOperand outputs[] = { 1774 g.DefineAsRegister(NodeProperties::FindProjection(node, 0)), 1775 g.DefineAsRegister(NodeProperties::FindProjection(node, 1))}; 1776 InstructionCode code = 1777 kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode); 1778 Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, 1779 arraysize(temps), temps); 1780 } 1781 1782 void InstructionSelector::VisitWord32AtomicPairStore(Node* node) { 1783 IA32OperandGenerator g(this); 1784 Node* base = node->InputAt(0); 1785 Node* index = node->InputAt(1); 1786 Node* value = node->InputAt(2); 1787 Node* value_high = node->InputAt(3); 1788 1789 AddressingMode addressing_mode; 1790 InstructionOperand inputs[] = { 1791 g.UseFixed(value, ebx), g.UseFixed(value_high, ecx), 1792 g.UseUniqueRegister(base), 1793 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1794 // Allocating temp registers here as stores are performed using an atomic 1795 // exchange, the output of which is stored in edx:eax, which should be saved 1796 // and restored at the end of the instruction. 1797 InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)}; 1798 InstructionCode code = 1799 kIA32Word32AtomicPairStore | AddressingModeField::encode(addressing_mode); 1800 Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps); 1801 } 1802 1803 void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) { 1804 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAdd); 1805 } 1806 1807 void InstructionSelector::VisitWord32AtomicPairSub(Node* node) { 1808 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairSub); 1809 } 1810 1811 void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) { 1812 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAnd); 1813 } 1814 1815 void InstructionSelector::VisitWord32AtomicPairOr(Node* node) { 1816 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairOr); 1817 } 1818 1819 void InstructionSelector::VisitWord32AtomicPairXor(Node* node) { 1820 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairXor); 1821 } 1822 1823 void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) { 1824 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairExchange); 1825 } 1826 1827 void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) { 1828 IA32OperandGenerator g(this); 1829 Node* index = node->InputAt(1); 1830 AddressingMode addressing_mode; 1831 InstructionOperand inputs[] = { 1832 // High, Low values of old value 1833 g.UseFixed(node->InputAt(2), eax), g.UseFixed(node->InputAt(3), edx), 1834 // High, Low values of new value 1835 g.UseFixed(node->InputAt(4), ebx), g.UseFixed(node->InputAt(5), ecx), 1836 // InputAt(0) => base 1837 g.UseUniqueRegister(node->InputAt(0)), 1838 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1839 InstructionOperand outputs[] = { 1840 g.DefineAsFixed(NodeProperties::FindProjection(node, 0), eax), 1841 g.DefineAsFixed(NodeProperties::FindProjection(node, 1), edx)}; 1842 InstructionCode code = kIA32Word32AtomicPairCompareExchange | 1843 AddressingModeField::encode(addressing_mode); 1844 Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs); 1845 } 1846 1847 void InstructionSelector::VisitWord64AtomicNarrowBinop(Node* node, 1848 ArchOpcode uint8_op, 1849 ArchOpcode uint16_op, 1850 ArchOpcode uint32_op) { 1851 MachineType type = AtomicOpType(node->op()); 1852 DCHECK(type != MachineType::Uint64()); 1853 ArchOpcode opcode = kArchNop; 1854 if (type == MachineType::Uint32()) { 1855 opcode = uint32_op; 1856 } else if (type == MachineType::Uint16()) { 1857 opcode = uint16_op; 1858 } else if (type == MachineType::Uint8()) { 1859 opcode = uint8_op; 1860 } else { 1861 UNREACHABLE(); 1862 return; 1863 } 1864 VisitNarrowAtomicBinOp(this, node, opcode, type); 1865 } 1866 1867 #define VISIT_ATOMIC_BINOP(op) \ 1868 void InstructionSelector::VisitWord64AtomicNarrow##op(Node* node) { \ 1869 VisitWord64AtomicNarrowBinop(node, kIA32Word64AtomicNarrow##op##Uint8, \ 1870 kIA32Word64AtomicNarrow##op##Uint16, \ 1871 kIA32Word64AtomicNarrow##op##Uint32); \ 1872 } 1873 VISIT_ATOMIC_BINOP(Add) 1874 VISIT_ATOMIC_BINOP(Sub) 1875 VISIT_ATOMIC_BINOP(And) 1876 VISIT_ATOMIC_BINOP(Or) 1877 VISIT_ATOMIC_BINOP(Xor) 1878 #undef VISIT_ATOMIC_BINOP 1879 1880 void InstructionSelector::VisitWord64AtomicNarrowExchange(Node* node) { 1881 MachineType type = AtomicOpType(node->op()); 1882 DCHECK(type != MachineType::Uint64()); 1883 ArchOpcode opcode = kArchNop; 1884 if (type == MachineType::Uint32()) { 1885 opcode = kIA32Word64AtomicNarrowExchangeUint32; 1886 } else if (type == MachineType::Uint16()) { 1887 opcode = kIA32Word64AtomicNarrowExchangeUint16; 1888 } else if (type == MachineType::Uint8()) { 1889 opcode = kIA32Word64AtomicNarrowExchangeUint8; 1890 } else { 1891 UNREACHABLE(); 1892 return; 1893 } 1894 IA32OperandGenerator g(this); 1895 Node* base = node->InputAt(0); 1896 Node* index = node->InputAt(1); 1897 Node* value = node->InputAt(2); 1898 AddressingMode addressing_mode; 1899 InstructionOperand value_operand = 1900 (type.representation() == MachineRepresentation::kWord8) 1901 ? g.UseFixed(value, edx) 1902 : g.UseUniqueRegister(value); 1903 InstructionOperand inputs[] = { 1904 value_operand, g.UseUniqueRegister(base), 1905 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1906 InstructionOperand outputs[2]; 1907 if (type.representation() == MachineRepresentation::kWord8) { 1908 // Using DefineSameAsFirst requires the register to be unallocated. 1909 outputs[0] = g.DefineAsFixed(NodeProperties::FindProjection(node, 0), edx); 1910 } else { 1911 outputs[0] = g.DefineSameAsFirst(NodeProperties::FindProjection(node, 0)); 1912 } 1913 outputs[1] = g.DefineAsRegister(NodeProperties::FindProjection(node, 1)); 1914 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1915 Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs); 1916 } 1917 1918 void InstructionSelector::VisitWord64AtomicNarrowCompareExchange(Node* node) { 1919 MachineType type = AtomicOpType(node->op()); 1920 DCHECK(type != MachineType::Uint64()); 1921 ArchOpcode opcode = kArchNop; 1922 if (type == MachineType::Uint32()) { 1923 opcode = kIA32Word64AtomicNarrowCompareExchangeUint32; 1924 } else if (type == MachineType::Uint16()) { 1925 opcode = kIA32Word64AtomicNarrowCompareExchangeUint16; 1926 } else if (type == MachineType::Uint8()) { 1927 opcode = kIA32Word64AtomicNarrowCompareExchangeUint8; 1928 } else { 1929 UNREACHABLE(); 1930 return; 1931 } 1932 IA32OperandGenerator g(this); 1933 Node* base = node->InputAt(0); 1934 Node* index = node->InputAt(1); 1935 Node* old_value = node->InputAt(2); 1936 Node* new_value = node->InputAt(3); 1937 AddressingMode addressing_mode; 1938 InstructionOperand new_value_operand = 1939 (type.representation() == MachineRepresentation::kWord8) 1940 ? g.UseByteRegister(new_value) 1941 : g.UseUniqueRegister(new_value); 1942 InstructionOperand inputs[] = { 1943 g.UseFixed(old_value, eax), new_value_operand, g.UseUniqueRegister(base), 1944 g.GetEffectiveIndexOperand(index, &addressing_mode)}; 1945 InstructionOperand outputs[] = { 1946 g.DefineAsFixed(NodeProperties::FindProjection(node, 0), eax), 1947 g.DefineAsRegister(NodeProperties::FindProjection(node, 1))}; 1948 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); 1949 Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs); 1950 } 1951 1952 #define SIMD_INT_TYPES(V) \ 1953 V(I32x4) \ 1954 V(I16x8) \ 1955 V(I8x16) 1956 1957 #define SIMD_BINOP_LIST(V) \ 1958 V(F32x4Add) \ 1959 V(F32x4AddHoriz) \ 1960 V(F32x4Sub) \ 1961 V(F32x4Mul) \ 1962 V(F32x4Min) \ 1963 V(F32x4Max) \ 1964 V(F32x4Eq) \ 1965 V(F32x4Ne) \ 1966 V(F32x4Lt) \ 1967 V(F32x4Le) \ 1968 V(I32x4Add) \ 1969 V(I32x4AddHoriz) \ 1970 V(I32x4Sub) \ 1971 V(I32x4Mul) \ 1972 V(I32x4MinS) \ 1973 V(I32x4MaxS) \ 1974 V(I32x4Eq) \ 1975 V(I32x4Ne) \ 1976 V(I32x4GtS) \ 1977 V(I32x4GeS) \ 1978 V(I32x4MinU) \ 1979 V(I32x4MaxU) \ 1980 V(I32x4GtU) \ 1981 V(I32x4GeU) \ 1982 V(I16x8SConvertI32x4) \ 1983 V(I16x8Add) \ 1984 V(I16x8AddSaturateS) \ 1985 V(I16x8AddHoriz) \ 1986 V(I16x8Sub) \ 1987 V(I16x8SubSaturateS) \ 1988 V(I16x8Mul) \ 1989 V(I16x8MinS) \ 1990 V(I16x8MaxS) \ 1991 V(I16x8Eq) \ 1992 V(I16x8Ne) \ 1993 V(I16x8GtS) \ 1994 V(I16x8GeS) \ 1995 V(I16x8AddSaturateU) \ 1996 V(I16x8SubSaturateU) \ 1997 V(I16x8MinU) \ 1998 V(I16x8MaxU) \ 1999 V(I16x8GtU) \ 2000 V(I16x8GeU) \ 2001 V(I8x16SConvertI16x8) \ 2002 V(I8x16Add) \ 2003 V(I8x16AddSaturateS) \ 2004 V(I8x16Sub) \ 2005 V(I8x16SubSaturateS) \ 2006 V(I8x16MinS) \ 2007 V(I8x16MaxS) \ 2008 V(I8x16Eq) \ 2009 V(I8x16Ne) \ 2010 V(I8x16GtS) \ 2011 V(I8x16GeS) \ 2012 V(I8x16AddSaturateU) \ 2013 V(I8x16SubSaturateU) \ 2014 V(I8x16MinU) \ 2015 V(I8x16MaxU) \ 2016 V(I8x16GtU) \ 2017 V(I8x16GeU) \ 2018 V(S128And) \ 2019 V(S128Or) \ 2020 V(S128Xor) 2021 2022 #define SIMD_UNOP_LIST(V) \ 2023 V(F32x4SConvertI32x4) \ 2024 V(F32x4RecipApprox) \ 2025 V(F32x4RecipSqrtApprox) \ 2026 V(I32x4SConvertI16x8Low) \ 2027 V(I32x4SConvertI16x8High) \ 2028 V(I32x4Neg) \ 2029 V(I32x4UConvertI16x8Low) \ 2030 V(I32x4UConvertI16x8High) \ 2031 V(I16x8SConvertI8x16Low) \ 2032 V(I16x8SConvertI8x16High) \ 2033 V(I16x8Neg) \ 2034 V(I16x8UConvertI8x16Low) \ 2035 V(I16x8UConvertI8x16High) \ 2036 V(I8x16Neg) 2037 2038 #define SIMD_UNOP_PREFIX_LIST(V) \ 2039 V(F32x4Abs) \ 2040 V(F32x4Neg) \ 2041 V(S128Not) 2042 2043 #define SIMD_ANYTRUE_LIST(V) \ 2044 V(S1x4AnyTrue) \ 2045 V(S1x8AnyTrue) \ 2046 V(S1x16AnyTrue) 2047 2048 #define SIMD_ALLTRUE_LIST(V) \ 2049 V(S1x4AllTrue) \ 2050 V(S1x8AllTrue) \ 2051 V(S1x16AllTrue) 2052 2053 #define SIMD_SHIFT_OPCODES(V) \ 2054 V(I32x4Shl) \ 2055 V(I32x4ShrS) \ 2056 V(I32x4ShrU) \ 2057 V(I16x8Shl) \ 2058 V(I16x8ShrS) \ 2059 V(I16x8ShrU) \ 2060 V(I8x16Shl) 2061 2062 #define SIMD_I8X16_RIGHT_SHIFT_OPCODES(V) \ 2063 V(I8x16ShrS) \ 2064 V(I8x16ShrU) 2065 2066 void InstructionSelector::VisitF32x4Splat(Node* node) { 2067 VisitRRSimd(this, node, kAVXF32x4Splat, kSSEF32x4Splat); 2068 } 2069 2070 void InstructionSelector::VisitF32x4ExtractLane(Node* node) { 2071 VisitRRISimd(this, node, kAVXF32x4ExtractLane, kSSEF32x4ExtractLane); 2072 } 2073 2074 void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) { 2075 VisitRRSimd(this, node, kAVXF32x4UConvertI32x4, kSSEF32x4UConvertI32x4); 2076 } 2077 2078 void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) { 2079 VisitRRSimd(this, node, kAVXI32x4SConvertF32x4, kSSEI32x4SConvertF32x4); 2080 } 2081 2082 void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) { 2083 IA32OperandGenerator g(this); 2084 InstructionOperand temps[] = {g.TempSimd128Register()}; 2085 InstructionCode opcode = 2086 IsSupported(AVX) ? kAVXI32x4UConvertF32x4 : kSSEI32x4UConvertF32x4; 2087 Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)), 2088 arraysize(temps), temps); 2089 } 2090 2091 void InstructionSelector::VisitI8x16Mul(Node* node) { 2092 IA32OperandGenerator g(this); 2093 InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0)); 2094 InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1)); 2095 InstructionOperand temps[] = {g.TempSimd128Register()}; 2096 if (IsSupported(AVX)) { 2097 Emit(kAVXI8x16Mul, g.DefineAsRegister(node), operand0, operand1, 2098 arraysize(temps), temps); 2099 } else { 2100 Emit(kSSEI8x16Mul, g.DefineSameAsFirst(node), operand0, operand1, 2101 arraysize(temps), temps); 2102 } 2103 } 2104 2105 void InstructionSelector::VisitS128Zero(Node* node) { 2106 IA32OperandGenerator g(this); 2107 Emit(kIA32S128Zero, g.DefineAsRegister(node)); 2108 } 2109 2110 void InstructionSelector::VisitS128Select(Node* node) { 2111 IA32OperandGenerator g(this); 2112 InstructionOperand operand2 = g.UseRegister(node->InputAt(2)); 2113 if (IsSupported(AVX)) { 2114 Emit(kAVXS128Select, g.DefineAsRegister(node), g.Use(node->InputAt(0)), 2115 g.Use(node->InputAt(1)), operand2); 2116 } else { 2117 Emit(kSSES128Select, g.DefineSameAsFirst(node), 2118 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), 2119 operand2); 2120 } 2121 } 2122 2123 #define VISIT_SIMD_SPLAT(Type) \ 2124 void InstructionSelector::Visit##Type##Splat(Node* node) { \ 2125 VisitRO(this, node, kIA32##Type##Splat); \ 2126 } 2127 SIMD_INT_TYPES(VISIT_SIMD_SPLAT) 2128 #undef VISIT_SIMD_SPLAT 2129 2130 #define VISIT_SIMD_EXTRACT_LANE(Type) \ 2131 void InstructionSelector::Visit##Type##ExtractLane(Node* node) { \ 2132 VisitRRISimd(this, node, kIA32##Type##ExtractLane); \ 2133 } 2134 SIMD_INT_TYPES(VISIT_SIMD_EXTRACT_LANE) 2135 #undef VISIT_SIMD_EXTRACT_LANE 2136 2137 #define VISIT_SIMD_REPLACE_LANE(Type) \ 2138 void InstructionSelector::Visit##Type##ReplaceLane(Node* node) { \ 2139 IA32OperandGenerator g(this); \ 2140 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); \ 2141 InstructionOperand operand1 = \ 2142 g.UseImmediate(OpParameter<int32_t>(node->op())); \ 2143 InstructionOperand operand2 = g.Use(node->InputAt(1)); \ 2144 if (IsSupported(AVX)) { \ 2145 Emit(kAVX##Type##ReplaceLane, g.DefineAsRegister(node), operand0, \ 2146 operand1, operand2); \ 2147 } else { \ 2148 Emit(kSSE##Type##ReplaceLane, g.DefineSameAsFirst(node), operand0, \ 2149 operand1, operand2); \ 2150 } \ 2151 } 2152 SIMD_INT_TYPES(VISIT_SIMD_REPLACE_LANE) 2153 VISIT_SIMD_REPLACE_LANE(F32x4) 2154 #undef VISIT_SIMD_REPLACE_LANE 2155 #undef SIMD_INT_TYPES 2156 2157 #define VISIT_SIMD_SHIFT(Opcode) \ 2158 void InstructionSelector::Visit##Opcode(Node* node) { \ 2159 VisitRRISimd(this, node, kAVX##Opcode, kSSE##Opcode); \ 2160 } 2161 SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT) 2162 #undef VISIT_SIMD_SHIFT 2163 #undef SIMD_SHIFT_OPCODES 2164 2165 #define VISIT_SIMD_I8X16_RIGHT_SHIFT(Op) \ 2166 void InstructionSelector::Visit##Op(Node* node) { \ 2167 VisitRRISimd(this, node, kIA32##Op); \ 2168 } 2169 2170 SIMD_I8X16_RIGHT_SHIFT_OPCODES(VISIT_SIMD_I8X16_RIGHT_SHIFT) 2171 #undef SIMD_I8X16_RIGHT_SHIFT_OPCODES 2172 #undef VISIT_SIMD_I8X16_RIGHT_SHIFT 2173 2174 #define VISIT_SIMD_UNOP(Opcode) \ 2175 void InstructionSelector::Visit##Opcode(Node* node) { \ 2176 IA32OperandGenerator g(this); \ 2177 Emit(kIA32##Opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); \ 2178 } 2179 SIMD_UNOP_LIST(VISIT_SIMD_UNOP) 2180 #undef VISIT_SIMD_UNOP 2181 #undef SIMD_UNOP_LIST 2182 2183 #define VISIT_SIMD_UNOP_PREFIX(Opcode) \ 2184 void InstructionSelector::Visit##Opcode(Node* node) { \ 2185 IA32OperandGenerator g(this); \ 2186 InstructionCode opcode = IsSupported(AVX) ? kAVX##Opcode : kSSE##Opcode; \ 2187 Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); \ 2188 } 2189 SIMD_UNOP_PREFIX_LIST(VISIT_SIMD_UNOP_PREFIX) 2190 #undef VISIT_SIMD_UNOP_PREFIX 2191 #undef SIMD_UNOP_PREFIX_LIST 2192 2193 #define VISIT_SIMD_ANYTRUE(Opcode) \ 2194 void InstructionSelector::Visit##Opcode(Node* node) { \ 2195 IA32OperandGenerator g(this); \ 2196 InstructionOperand temps[] = {g.TempRegister()}; \ 2197 Emit(kIA32##Opcode, g.DefineAsRegister(node), \ 2198 g.UseRegister(node->InputAt(0)), arraysize(temps), temps); \ 2199 } 2200 SIMD_ANYTRUE_LIST(VISIT_SIMD_ANYTRUE) 2201 #undef VISIT_SIMD_ANYTRUE 2202 #undef SIMD_ANYTRUE_LIST 2203 2204 #define VISIT_SIMD_ALLTRUE(Opcode) \ 2205 void InstructionSelector::Visit##Opcode(Node* node) { \ 2206 IA32OperandGenerator g(this); \ 2207 InstructionOperand temps[] = {g.TempRegister()}; \ 2208 Emit(kIA32##Opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)), \ 2209 arraysize(temps), temps); \ 2210 } 2211 SIMD_ALLTRUE_LIST(VISIT_SIMD_ALLTRUE) 2212 #undef VISIT_SIMD_ALLTRUE 2213 #undef SIMD_ALLTRUE_LIST 2214 2215 #define VISIT_SIMD_BINOP(Opcode) \ 2216 void InstructionSelector::Visit##Opcode(Node* node) { \ 2217 VisitRROFloat(this, node, kAVX##Opcode, kSSE##Opcode); \ 2218 } 2219 SIMD_BINOP_LIST(VISIT_SIMD_BINOP) 2220 #undef VISIT_SIMD_BINOP 2221 #undef SIMD_BINOP_LIST 2222 2223 void VisitPack(InstructionSelector* selector, Node* node, ArchOpcode avx_opcode, 2224 ArchOpcode sse_opcode) { 2225 IA32OperandGenerator g(selector); 2226 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); 2227 InstructionOperand operand1 = g.Use(node->InputAt(1)); 2228 if (selector->IsSupported(AVX)) { 2229 selector->Emit(avx_opcode, g.DefineSameAsFirst(node), operand0, operand1); 2230 } else { 2231 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1); 2232 } 2233 } 2234 2235 void InstructionSelector::VisitI16x8UConvertI32x4(Node* node) { 2236 VisitPack(this, node, kAVXI16x8UConvertI32x4, kSSEI16x8UConvertI32x4); 2237 } 2238 2239 void InstructionSelector::VisitI8x16UConvertI16x8(Node* node) { 2240 VisitPack(this, node, kAVXI8x16UConvertI16x8, kSSEI8x16UConvertI16x8); 2241 } 2242 2243 void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) { 2244 UNREACHABLE(); 2245 } 2246 2247 void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) { 2248 UNREACHABLE(); 2249 } 2250 2251 namespace { 2252 2253 // Packs a 4 lane shuffle into a single imm8 suitable for use by pshufd, 2254 // pshuflw, and pshufhw. 2255 uint8_t PackShuffle4(uint8_t* shuffle) { 2256 return (shuffle[0] & 3) | ((shuffle[1] & 3) << 2) | ((shuffle[2] & 3) << 4) | 2257 ((shuffle[3] & 3) << 6); 2258 } 2259 2260 // Gets an 8 bit lane mask suitable for 16x8 pblendw. 2261 uint8_t PackBlend8(const uint8_t* shuffle16x8) { 2262 int8_t result = 0; 2263 for (int i = 0; i < 8; ++i) { 2264 result |= (shuffle16x8[i] >= 8 ? 1 : 0) << i; 2265 } 2266 return result; 2267 } 2268 2269 // Gets an 8 bit lane mask suitable for 32x4 pblendw. 2270 uint8_t PackBlend4(const uint8_t* shuffle32x4) { 2271 int8_t result = 0; 2272 for (int i = 0; i < 4; ++i) { 2273 result |= (shuffle32x4[i] >= 4 ? 0x3 : 0) << (i * 2); 2274 } 2275 return result; 2276 } 2277 2278 // Returns true if shuffle can be decomposed into two 16x4 half shuffles 2279 // followed by a 16x8 blend. 2280 // E.g. [3 2 1 0 15 14 13 12]. 2281 bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) { 2282 *blend_mask = 0; 2283 for (int i = 0; i < 8; i++) { 2284 if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false; 2285 *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i; 2286 } 2287 return true; 2288 } 2289 2290 struct ShuffleEntry { 2291 uint8_t shuffle[kSimd128Size]; 2292 ArchOpcode opcode; 2293 ArchOpcode avx_opcode; 2294 bool src0_needs_reg; 2295 bool src1_needs_reg; 2296 }; 2297 2298 // Shuffles that map to architecture-specific instruction sequences. These are 2299 // matched very early, so we shouldn't include shuffles that match better in 2300 // later tests, like 32x4 and 16x8 shuffles. In general, these patterns should 2301 // map to either a single instruction, or be finer grained, such as zip/unzip or 2302 // transpose patterns. 2303 static const ShuffleEntry arch_shuffles[] = { 2304 {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23}, 2305 kIA32S64x2UnpackLow, 2306 kIA32S64x2UnpackLow, 2307 true, 2308 false}, 2309 {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}, 2310 kIA32S64x2UnpackHigh, 2311 kIA32S64x2UnpackHigh, 2312 true, 2313 false}, 2314 {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}, 2315 kIA32S32x4UnpackLow, 2316 kIA32S32x4UnpackLow, 2317 true, 2318 false}, 2319 {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31}, 2320 kIA32S32x4UnpackHigh, 2321 kIA32S32x4UnpackHigh, 2322 true, 2323 false}, 2324 {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23}, 2325 kIA32S16x8UnpackLow, 2326 kIA32S16x8UnpackLow, 2327 true, 2328 false}, 2329 {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31}, 2330 kIA32S16x8UnpackHigh, 2331 kIA32S16x8UnpackHigh, 2332 true, 2333 false}, 2334 {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23}, 2335 kIA32S8x16UnpackLow, 2336 kIA32S8x16UnpackLow, 2337 true, 2338 false}, 2339 {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31}, 2340 kIA32S8x16UnpackHigh, 2341 kIA32S8x16UnpackHigh, 2342 true, 2343 false}, 2344 2345 {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29}, 2346 kSSES16x8UnzipLow, 2347 kAVXS16x8UnzipLow, 2348 true, 2349 false}, 2350 {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31}, 2351 kSSES16x8UnzipHigh, 2352 kAVXS16x8UnzipHigh, 2353 true, 2354 true}, 2355 {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30}, 2356 kSSES8x16UnzipLow, 2357 kAVXS8x16UnzipLow, 2358 true, 2359 true}, 2360 {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31}, 2361 kSSES8x16UnzipHigh, 2362 kAVXS8x16UnzipHigh, 2363 true, 2364 true}, 2365 2366 {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30}, 2367 kSSES8x16TransposeLow, 2368 kAVXS8x16TransposeLow, 2369 true, 2370 true}, 2371 {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31}, 2372 kSSES8x16TransposeHigh, 2373 kAVXS8x16TransposeHigh, 2374 true, 2375 true}, 2376 {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8}, 2377 kSSES8x8Reverse, 2378 kAVXS8x8Reverse, 2379 false, 2380 false}, 2381 {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12}, 2382 kSSES8x4Reverse, 2383 kAVXS8x4Reverse, 2384 false, 2385 false}, 2386 {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}, 2387 kSSES8x2Reverse, 2388 kAVXS8x2Reverse, 2389 true, 2390 true}}; 2391 2392 bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table, 2393 size_t num_entries, bool is_swizzle, 2394 const ShuffleEntry** arch_shuffle) { 2395 uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1; 2396 for (size_t i = 0; i < num_entries; ++i) { 2397 const ShuffleEntry& entry = table[i]; 2398 int j = 0; 2399 for (; j < kSimd128Size; ++j) { 2400 if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) { 2401 break; 2402 } 2403 } 2404 if (j == kSimd128Size) { 2405 *arch_shuffle = &entry; 2406 return true; 2407 } 2408 } 2409 return false; 2410 } 2411 2412 } // namespace 2413 2414 void InstructionSelector::VisitS8x16Shuffle(Node* node) { 2415 uint8_t shuffle[kSimd128Size]; 2416 bool is_swizzle; 2417 CanonicalizeShuffle(node, shuffle, &is_swizzle); 2418 2419 int imm_count = 0; 2420 static const int kMaxImms = 6; 2421 uint32_t imms[kMaxImms]; 2422 int temp_count = 0; 2423 static const int kMaxTemps = 2; 2424 InstructionOperand temps[kMaxTemps]; 2425 2426 IA32OperandGenerator g(this); 2427 bool use_avx = CpuFeatures::IsSupported(AVX); 2428 // AVX and swizzles don't generally need DefineSameAsFirst to avoid a move. 2429 bool no_same_as_first = use_avx || is_swizzle; 2430 // We generally need UseRegister for input0, Use for input1. 2431 bool src0_needs_reg = true; 2432 bool src1_needs_reg = false; 2433 ArchOpcode opcode = kIA32S8x16Shuffle; // general shuffle is the default 2434 2435 uint8_t offset; 2436 uint8_t shuffle32x4[4]; 2437 uint8_t shuffle16x8[8]; 2438 int index; 2439 const ShuffleEntry* arch_shuffle; 2440 if (TryMatchConcat(shuffle, &offset)) { 2441 // Swap inputs from the normal order for (v)palignr. 2442 SwapShuffleInputs(node); 2443 is_swizzle = false; // It's simpler to just handle the general case. 2444 no_same_as_first = use_avx; // SSE requires same-as-first. 2445 opcode = kIA32S8x16Alignr; 2446 // palignr takes a single imm8 offset. 2447 imms[imm_count++] = offset; 2448 } else if (TryMatchArchShuffle(shuffle, arch_shuffles, 2449 arraysize(arch_shuffles), is_swizzle, 2450 &arch_shuffle)) { 2451 opcode = use_avx ? arch_shuffle->avx_opcode : arch_shuffle->opcode; 2452 src0_needs_reg = !use_avx || arch_shuffle->src0_needs_reg; 2453 // SSE can't take advantage of both operands in registers and needs 2454 // same-as-first. 2455 src1_needs_reg = use_avx && arch_shuffle->src1_needs_reg; 2456 no_same_as_first = use_avx; 2457 } else if (TryMatch32x4Shuffle(shuffle, shuffle32x4)) { 2458 uint8_t shuffle_mask = PackShuffle4(shuffle32x4); 2459 if (is_swizzle) { 2460 if (TryMatchIdentity(shuffle)) { 2461 // Bypass normal shuffle code generation in this case. 2462 EmitIdentity(node); 2463 return; 2464 } else { 2465 // pshufd takes a single imm8 shuffle mask. 2466 opcode = kIA32S32x4Swizzle; 2467 no_same_as_first = true; 2468 src0_needs_reg = false; 2469 imms[imm_count++] = shuffle_mask; 2470 } 2471 } else { 2472 // 2 operand shuffle 2473 // A blend is more efficient than a general 32x4 shuffle; try it first. 2474 if (TryMatchBlend(shuffle)) { 2475 opcode = kIA32S16x8Blend; 2476 uint8_t blend_mask = PackBlend4(shuffle32x4); 2477 imms[imm_count++] = blend_mask; 2478 } else { 2479 opcode = kIA32S32x4Shuffle; 2480 no_same_as_first = true; 2481 src0_needs_reg = false; 2482 imms[imm_count++] = shuffle_mask; 2483 int8_t blend_mask = PackBlend4(shuffle32x4); 2484 imms[imm_count++] = blend_mask; 2485 } 2486 } 2487 } else if (TryMatch16x8Shuffle(shuffle, shuffle16x8)) { 2488 uint8_t blend_mask; 2489 if (TryMatchBlend(shuffle)) { 2490 opcode = kIA32S16x8Blend; 2491 blend_mask = PackBlend8(shuffle16x8); 2492 imms[imm_count++] = blend_mask; 2493 } else if (TryMatchDup<8>(shuffle, &index)) { 2494 opcode = kIA32S16x8Dup; 2495 src0_needs_reg = false; 2496 imms[imm_count++] = index; 2497 } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) { 2498 opcode = is_swizzle ? kIA32S16x8HalfShuffle1 : kIA32S16x8HalfShuffle2; 2499 // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0). 2500 no_same_as_first = true; 2501 src0_needs_reg = false; 2502 uint8_t mask_lo = PackShuffle4(shuffle16x8); 2503 uint8_t mask_hi = PackShuffle4(shuffle16x8 + 4); 2504 imms[imm_count++] = mask_lo; 2505 imms[imm_count++] = mask_hi; 2506 if (!is_swizzle) imms[imm_count++] = blend_mask; 2507 } 2508 } else if (TryMatchDup<16>(shuffle, &index)) { 2509 opcode = kIA32S8x16Dup; 2510 no_same_as_first = use_avx; 2511 src0_needs_reg = true; 2512 imms[imm_count++] = index; 2513 } 2514 if (opcode == kIA32S8x16Shuffle) { 2515 // Use same-as-first for general swizzle, but not shuffle. 2516 no_same_as_first = !is_swizzle; 2517 src0_needs_reg = !no_same_as_first; 2518 imms[imm_count++] = Pack4Lanes(shuffle); 2519 imms[imm_count++] = Pack4Lanes(shuffle + 4); 2520 imms[imm_count++] = Pack4Lanes(shuffle + 8); 2521 imms[imm_count++] = Pack4Lanes(shuffle + 12); 2522 temps[temp_count++] = g.TempRegister(); 2523 } 2524 2525 // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra 2526 // move instruction in the CodeGenerator. 2527 Node* input0 = node->InputAt(0); 2528 InstructionOperand dst = 2529 no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node); 2530 InstructionOperand src0 = 2531 src0_needs_reg ? g.UseRegister(input0) : g.Use(input0); 2532 2533 int input_count = 0; 2534 InstructionOperand inputs[2 + kMaxImms + kMaxTemps]; 2535 inputs[input_count++] = src0; 2536 if (!is_swizzle) { 2537 Node* input1 = node->InputAt(1); 2538 inputs[input_count++] = 2539 src1_needs_reg ? g.UseRegister(input1) : g.Use(input1); 2540 } 2541 for (int i = 0; i < imm_count; ++i) { 2542 inputs[input_count++] = g.UseImmediate(imms[i]); 2543 } 2544 Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps); 2545 } 2546 2547 // static 2548 MachineOperatorBuilder::Flags 2549 InstructionSelector::SupportedMachineOperatorFlags() { 2550 MachineOperatorBuilder::Flags flags = 2551 MachineOperatorBuilder::kWord32ShiftIsSafe | 2552 MachineOperatorBuilder::kWord32Ctz | 2553 MachineOperatorBuilder::kSpeculationFence; 2554 if (CpuFeatures::IsSupported(POPCNT)) { 2555 flags |= MachineOperatorBuilder::kWord32Popcnt; 2556 } 2557 if (CpuFeatures::IsSupported(SSE4_1)) { 2558 flags |= MachineOperatorBuilder::kFloat32RoundDown | 2559 MachineOperatorBuilder::kFloat64RoundDown | 2560 MachineOperatorBuilder::kFloat32RoundUp | 2561 MachineOperatorBuilder::kFloat64RoundUp | 2562 MachineOperatorBuilder::kFloat32RoundTruncate | 2563 MachineOperatorBuilder::kFloat64RoundTruncate | 2564 MachineOperatorBuilder::kFloat32RoundTiesEven | 2565 MachineOperatorBuilder::kFloat64RoundTiesEven; 2566 } 2567 return flags; 2568 } 2569 2570 // static 2571 MachineOperatorBuilder::AlignmentRequirements 2572 InstructionSelector::AlignmentRequirements() { 2573 return MachineOperatorBuilder::AlignmentRequirements:: 2574 FullUnalignedAccessSupport(); 2575 } 2576 2577 } // namespace compiler 2578 } // namespace internal 2579 } // namespace v8 2580