1 // Copyright 2016 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/simd-scalar-lowering.h" 6 7 #include "src/compiler/diamond.h" 8 #include "src/compiler/linkage.h" 9 #include "src/compiler/node-matchers.h" 10 #include "src/compiler/node-properties.h" 11 #include "src/compiler/node.h" 12 #include "src/compiler/wasm-compiler.h" 13 14 namespace v8 { 15 namespace internal { 16 namespace compiler { 17 18 namespace { 19 static const int kNumLanes32 = 4; 20 static const int kNumLanes16 = 8; 21 static const int kNumLanes8 = 16; 22 static const int32_t kMask16 = 0xFFFF; 23 static const int32_t kMask8 = 0xFF; 24 static const int32_t kShift16 = 16; 25 static const int32_t kShift8 = 24; 26 } // anonymous 27 28 SimdScalarLowering::SimdScalarLowering( 29 MachineGraph* mcgraph, Signature<MachineRepresentation>* signature) 30 : mcgraph_(mcgraph), 31 state_(mcgraph->graph(), 3), 32 stack_(mcgraph_->zone()), 33 replacements_(nullptr), 34 signature_(signature), 35 placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"), 36 graph()->start())), 37 parameter_count_after_lowering_(-1) { 38 DCHECK_NOT_NULL(graph()); 39 DCHECK_NOT_NULL(graph()->end()); 40 replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount()); 41 memset(replacements_, 0, sizeof(Replacement) * graph()->NodeCount()); 42 } 43 44 void SimdScalarLowering::LowerGraph() { 45 stack_.push_back({graph()->end(), 0}); 46 state_.Set(graph()->end(), State::kOnStack); 47 replacements_[graph()->end()->id()].type = SimdType::kInt32x4; 48 49 while (!stack_.empty()) { 50 NodeState& top = stack_.back(); 51 if (top.input_index == top.node->InputCount()) { 52 // All inputs of top have already been lowered, now lower top. 53 stack_.pop_back(); 54 state_.Set(top.node, State::kVisited); 55 LowerNode(top.node); 56 } else { 57 // Push the next input onto the stack. 58 Node* input = top.node->InputAt(top.input_index++); 59 if (state_.Get(input) == State::kUnvisited) { 60 SetLoweredType(input, top.node); 61 if (input->opcode() == IrOpcode::kPhi) { 62 // To break cycles with phi nodes we push phis on a separate stack so 63 // that they are processed after all other nodes. 64 PreparePhiReplacement(input); 65 stack_.push_front({input, 0}); 66 } else if (input->opcode() == IrOpcode::kEffectPhi || 67 input->opcode() == IrOpcode::kLoop) { 68 stack_.push_front({input, 0}); 69 } else { 70 stack_.push_back({input, 0}); 71 } 72 state_.Set(input, State::kOnStack); 73 } 74 } 75 } 76 } 77 78 #define FOREACH_INT32X4_OPCODE(V) \ 79 V(I32x4Splat) \ 80 V(I32x4ExtractLane) \ 81 V(I32x4ReplaceLane) \ 82 V(I32x4SConvertF32x4) \ 83 V(I32x4UConvertF32x4) \ 84 V(I32x4SConvertI16x8Low) \ 85 V(I32x4SConvertI16x8High) \ 86 V(I32x4Neg) \ 87 V(I32x4Shl) \ 88 V(I32x4ShrS) \ 89 V(I32x4Add) \ 90 V(I32x4AddHoriz) \ 91 V(I32x4Sub) \ 92 V(I32x4Mul) \ 93 V(I32x4MinS) \ 94 V(I32x4MaxS) \ 95 V(I32x4ShrU) \ 96 V(I32x4MinU) \ 97 V(I32x4MaxU) \ 98 V(I32x4Eq) \ 99 V(I32x4Ne) \ 100 V(I32x4LtS) \ 101 V(I32x4LeS) \ 102 V(I32x4GtS) \ 103 V(I32x4GeS) \ 104 V(I32x4UConvertI16x8Low) \ 105 V(I32x4UConvertI16x8High) \ 106 V(I32x4LtU) \ 107 V(I32x4LeU) \ 108 V(I32x4GtU) \ 109 V(I32x4GeU) \ 110 V(S128And) \ 111 V(S128Or) \ 112 V(S128Xor) \ 113 V(S128Not) \ 114 V(S1x4AnyTrue) \ 115 V(S1x4AllTrue) \ 116 V(S1x8AnyTrue) \ 117 V(S1x8AllTrue) \ 118 V(S1x16AnyTrue) \ 119 V(S1x16AllTrue) 120 121 #define FOREACH_FLOAT32X4_OPCODE(V) \ 122 V(F32x4Splat) \ 123 V(F32x4ExtractLane) \ 124 V(F32x4ReplaceLane) \ 125 V(F32x4SConvertI32x4) \ 126 V(F32x4UConvertI32x4) \ 127 V(F32x4Abs) \ 128 V(F32x4Neg) \ 129 V(F32x4RecipApprox) \ 130 V(F32x4RecipSqrtApprox) \ 131 V(F32x4Add) \ 132 V(F32x4AddHoriz) \ 133 V(F32x4Sub) \ 134 V(F32x4Mul) \ 135 V(F32x4Min) \ 136 V(F32x4Max) 137 138 #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \ 139 V(F32x4Eq) \ 140 V(F32x4Ne) \ 141 V(F32x4Lt) \ 142 V(F32x4Le) \ 143 V(F32x4Gt) \ 144 V(F32x4Ge) 145 146 #define FOREACH_INT16X8_OPCODE(V) \ 147 V(I16x8Splat) \ 148 V(I16x8ExtractLane) \ 149 V(I16x8ReplaceLane) \ 150 V(I16x8SConvertI8x16Low) \ 151 V(I16x8SConvertI8x16High) \ 152 V(I16x8Neg) \ 153 V(I16x8Shl) \ 154 V(I16x8ShrS) \ 155 V(I16x8SConvertI32x4) \ 156 V(I16x8Add) \ 157 V(I16x8AddSaturateS) \ 158 V(I16x8AddHoriz) \ 159 V(I16x8Sub) \ 160 V(I16x8SubSaturateS) \ 161 V(I16x8Mul) \ 162 V(I16x8MinS) \ 163 V(I16x8MaxS) \ 164 V(I16x8UConvertI8x16Low) \ 165 V(I16x8UConvertI8x16High) \ 166 V(I16x8ShrU) \ 167 V(I16x8UConvertI32x4) \ 168 V(I16x8AddSaturateU) \ 169 V(I16x8SubSaturateU) \ 170 V(I16x8MinU) \ 171 V(I16x8MaxU) \ 172 V(I16x8Eq) \ 173 V(I16x8Ne) \ 174 V(I16x8LtS) \ 175 V(I16x8LeS) \ 176 V(I16x8LtU) \ 177 V(I16x8LeU) 178 179 #define FOREACH_INT8X16_OPCODE(V) \ 180 V(I8x16Splat) \ 181 V(I8x16ExtractLane) \ 182 V(I8x16ReplaceLane) \ 183 V(I8x16SConvertI16x8) \ 184 V(I8x16Neg) \ 185 V(I8x16Shl) \ 186 V(I8x16ShrS) \ 187 V(I8x16Add) \ 188 V(I8x16AddSaturateS) \ 189 V(I8x16Sub) \ 190 V(I8x16SubSaturateS) \ 191 V(I8x16Mul) \ 192 V(I8x16MinS) \ 193 V(I8x16MaxS) \ 194 V(I8x16ShrU) \ 195 V(I8x16UConvertI16x8) \ 196 V(I8x16AddSaturateU) \ 197 V(I8x16SubSaturateU) \ 198 V(I8x16MinU) \ 199 V(I8x16MaxU) \ 200 V(I8x16Eq) \ 201 V(I8x16Ne) \ 202 V(I8x16LtS) \ 203 V(I8x16LeS) \ 204 V(I8x16LtU) \ 205 V(I8x16LeU) \ 206 V(S8x16Shuffle) 207 208 MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) { 209 switch (simdType) { 210 case SimdType::kFloat32x4: 211 return MachineType::Float32(); 212 case SimdType::kInt32x4: 213 return MachineType::Int32(); 214 case SimdType::kInt16x8: 215 return MachineType::Int16(); 216 case SimdType::kInt8x16: 217 return MachineType::Int8(); 218 } 219 return MachineType::None(); 220 } 221 222 void SimdScalarLowering::SetLoweredType(Node* node, Node* output) { 223 switch (node->opcode()) { 224 #define CASE_STMT(name) case IrOpcode::k##name: 225 FOREACH_INT32X4_OPCODE(CASE_STMT) 226 case IrOpcode::kReturn: 227 case IrOpcode::kParameter: 228 case IrOpcode::kCall: { 229 replacements_[node->id()].type = SimdType::kInt32x4; 230 break; 231 } 232 FOREACH_FLOAT32X4_OPCODE(CASE_STMT) { 233 replacements_[node->id()].type = SimdType::kFloat32x4; 234 break; 235 } 236 FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) { 237 replacements_[node->id()].type = SimdType::kInt32x4; 238 break; 239 } 240 FOREACH_INT16X8_OPCODE(CASE_STMT) { 241 replacements_[node->id()].type = SimdType::kInt16x8; 242 break; 243 } 244 FOREACH_INT8X16_OPCODE(CASE_STMT) { 245 replacements_[node->id()].type = SimdType::kInt8x16; 246 break; 247 } 248 default: { 249 switch (output->opcode()) { 250 case IrOpcode::kF32x4SConvertI32x4: 251 case IrOpcode::kF32x4UConvertI32x4: 252 case IrOpcode::kI16x8SConvertI32x4: 253 case IrOpcode::kI16x8UConvertI32x4: { 254 replacements_[node->id()].type = SimdType::kInt32x4; 255 break; 256 } 257 case IrOpcode::kI8x16SConvertI16x8: 258 case IrOpcode::kI8x16UConvertI16x8: 259 case IrOpcode::kI32x4SConvertI16x8Low: 260 case IrOpcode::kI32x4SConvertI16x8High: 261 case IrOpcode::kI32x4UConvertI16x8Low: 262 case IrOpcode::kI32x4UConvertI16x8High: { 263 replacements_[node->id()].type = SimdType::kInt16x8; 264 break; 265 } 266 case IrOpcode::kI16x8SConvertI8x16Low: 267 case IrOpcode::kI16x8SConvertI8x16High: 268 case IrOpcode::kI16x8UConvertI8x16Low: 269 case IrOpcode::kI16x8UConvertI8x16High: { 270 replacements_[node->id()].type = SimdType::kInt8x16; 271 break; 272 } 273 FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) 274 case IrOpcode::kI32x4SConvertF32x4: 275 case IrOpcode::kI32x4UConvertF32x4: { 276 replacements_[node->id()].type = SimdType::kFloat32x4; 277 break; 278 } 279 case IrOpcode::kS128Select: { 280 replacements_[node->id()].type = SimdType::kInt32x4; 281 break; 282 } 283 default: { 284 replacements_[node->id()].type = replacements_[output->id()].type; 285 } 286 } 287 } 288 #undef CASE_STMT 289 } 290 } 291 292 static int GetParameterIndexAfterLoweringSimd128( 293 Signature<MachineRepresentation>* signature, int old_index) { 294 // In function calls, the simd128 types are passed as 4 Int32 types. The 295 // parameters are typecast to the types as needed for various operations. 296 int result = old_index; 297 for (int i = 0; i < old_index; ++i) { 298 if (signature->GetParam(i) == MachineRepresentation::kSimd128) { 299 result += 3; 300 } 301 } 302 return result; 303 } 304 305 int SimdScalarLowering::GetParameterCountAfterLowering() { 306 if (parameter_count_after_lowering_ == -1) { 307 // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the 308 // parameter count after lowering. 309 parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128( 310 signature(), static_cast<int>(signature()->parameter_count())); 311 } 312 return parameter_count_after_lowering_; 313 } 314 315 static int GetReturnCountAfterLoweringSimd128( 316 Signature<MachineRepresentation>* signature) { 317 int result = static_cast<int>(signature->return_count()); 318 for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) { 319 if (signature->GetReturn(i) == MachineRepresentation::kSimd128) { 320 result += 3; 321 } 322 } 323 return result; 324 } 325 326 int SimdScalarLowering::NumLanes(SimdType type) { 327 int num_lanes = 0; 328 if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) { 329 num_lanes = kNumLanes32; 330 } else if (type == SimdType::kInt16x8) { 331 num_lanes = kNumLanes16; 332 } else if (type == SimdType::kInt8x16) { 333 num_lanes = kNumLanes8; 334 } else { 335 UNREACHABLE(); 336 } 337 return num_lanes; 338 } 339 340 constexpr int SimdScalarLowering::kLaneOffsets[]; 341 342 void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices, 343 SimdType type) { 344 int num_lanes = NumLanes(type); 345 int lane_width = kSimd128Size / num_lanes; 346 int laneIndex = kLaneOffsets[0] / lane_width; 347 new_indices[laneIndex] = index; 348 for (int i = 1; i < num_lanes; ++i) { 349 laneIndex = kLaneOffsets[i * lane_width] / lane_width; 350 new_indices[laneIndex] = graph()->NewNode( 351 machine()->Int32Add(), index, 352 graph()->NewNode( 353 common()->Int32Constant(static_cast<int>(i) * lane_width))); 354 } 355 } 356 357 void SimdScalarLowering::LowerLoadOp(Node* node, SimdType type) { 358 MachineRepresentation rep = LoadRepresentationOf(node->op()).representation(); 359 const Operator* load_op; 360 switch (node->opcode()) { 361 case IrOpcode::kLoad: 362 load_op = machine()->Load(MachineTypeFrom(type)); 363 break; 364 case IrOpcode::kUnalignedLoad: 365 load_op = machine()->UnalignedLoad(MachineTypeFrom(type)); 366 break; 367 case IrOpcode::kProtectedLoad: 368 load_op = machine()->ProtectedLoad(MachineTypeFrom(type)); 369 break; 370 default: 371 UNREACHABLE(); 372 } 373 if (rep == MachineRepresentation::kSimd128) { 374 Node* base = node->InputAt(0); 375 Node* index = node->InputAt(1); 376 int num_lanes = NumLanes(type); 377 Node** indices = zone()->NewArray<Node*>(num_lanes); 378 GetIndexNodes(index, indices, type); 379 Node** rep_nodes = zone()->NewArray<Node*>(num_lanes); 380 rep_nodes[0] = node; 381 rep_nodes[0]->ReplaceInput(1, indices[0]); 382 NodeProperties::ChangeOp(rep_nodes[0], load_op); 383 if (node->InputCount() > 2) { 384 DCHECK_LT(3, node->InputCount()); 385 Node* effect_input = node->InputAt(2); 386 Node* control_input = node->InputAt(3); 387 for (int i = num_lanes - 1; i > 0; --i) { 388 rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input, 389 control_input); 390 effect_input = rep_nodes[i]; 391 } 392 rep_nodes[0]->ReplaceInput(2, rep_nodes[1]); 393 } else { 394 for (int i = 1; i < num_lanes; ++i) { 395 rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]); 396 } 397 } 398 ReplaceNode(node, rep_nodes, num_lanes); 399 } else { 400 DefaultLowering(node); 401 } 402 } 403 404 void SimdScalarLowering::LowerStoreOp(Node* node) { 405 // For store operation, use replacement type of its input instead of the 406 // one of its effected node. 407 DCHECK_LT(2, node->InputCount()); 408 SimdType rep_type = ReplacementType(node->InputAt(2)); 409 replacements_[node->id()].type = rep_type; 410 const Operator* store_op; 411 MachineRepresentation rep; 412 switch (node->opcode()) { 413 case IrOpcode::kStore: { 414 rep = StoreRepresentationOf(node->op()).representation(); 415 WriteBarrierKind write_barrier_kind = 416 StoreRepresentationOf(node->op()).write_barrier_kind(); 417 store_op = machine()->Store(StoreRepresentation( 418 MachineTypeFrom(rep_type).representation(), write_barrier_kind)); 419 break; 420 } 421 case IrOpcode::kUnalignedStore: { 422 rep = UnalignedStoreRepresentationOf(node->op()); 423 store_op = 424 machine()->UnalignedStore(MachineTypeFrom(rep_type).representation()); 425 break; 426 } 427 case IrOpcode::kProtectedStore: { 428 rep = StoreRepresentationOf(node->op()).representation(); 429 store_op = 430 machine()->ProtectedStore(MachineTypeFrom(rep_type).representation()); 431 break; 432 } 433 default: 434 UNREACHABLE(); 435 } 436 if (rep == MachineRepresentation::kSimd128) { 437 Node* base = node->InputAt(0); 438 Node* index = node->InputAt(1); 439 int num_lanes = NumLanes(rep_type); 440 Node** indices = zone()->NewArray<Node*>(num_lanes); 441 GetIndexNodes(index, indices, rep_type); 442 Node* value = node->InputAt(2); 443 DCHECK(HasReplacement(1, value)); 444 Node** rep_nodes = zone()->NewArray<Node*>(num_lanes); 445 rep_nodes[0] = node; 446 Node** rep_inputs = GetReplacementsWithType(value, rep_type); 447 rep_nodes[0]->ReplaceInput(2, rep_inputs[0]); 448 rep_nodes[0]->ReplaceInput(1, indices[0]); 449 NodeProperties::ChangeOp(node, store_op); 450 if (node->InputCount() > 3) { 451 DCHECK_LT(4, node->InputCount()); 452 Node* effect_input = node->InputAt(3); 453 Node* control_input = node->InputAt(4); 454 for (int i = num_lanes - 1; i > 0; --i) { 455 rep_nodes[i] = 456 graph()->NewNode(store_op, base, indices[i], rep_inputs[i], 457 effect_input, control_input); 458 effect_input = rep_nodes[i]; 459 } 460 rep_nodes[0]->ReplaceInput(3, rep_nodes[1]); 461 } else { 462 for (int i = 1; i < num_lanes; ++i) { 463 rep_nodes[i] = 464 graph()->NewNode(store_op, base, indices[i], rep_inputs[i]); 465 } 466 } 467 ReplaceNode(node, rep_nodes, num_lanes); 468 } else { 469 DefaultLowering(node); 470 } 471 } 472 473 void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type, 474 const Operator* op, 475 bool not_horizontal) { 476 DCHECK_EQ(2, node->InputCount()); 477 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type); 478 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type); 479 int num_lanes = NumLanes(input_rep_type); 480 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 481 if (not_horizontal) { 482 for (int i = 0; i < num_lanes; ++i) { 483 rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]); 484 } 485 } else { 486 for (int i = 0; i < num_lanes / 2; ++i) { 487 rep_node[i] = graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]); 488 rep_node[i + num_lanes / 2] = 489 graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]); 490 } 491 } 492 ReplaceNode(node, rep_node, num_lanes); 493 } 494 495 void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type, 496 const Operator* op, 497 bool invert_inputs) { 498 DCHECK_EQ(2, node->InputCount()); 499 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type); 500 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type); 501 int num_lanes = NumLanes(input_rep_type); 502 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 503 for (int i = 0; i < num_lanes; ++i) { 504 Node* cmp_result = nullptr; 505 if (invert_inputs) { 506 cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]); 507 } else { 508 cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]); 509 } 510 Diamond d_cmp(graph(), common(), 511 graph()->NewNode(machine()->Word32Equal(), cmp_result, 512 mcgraph_->Int32Constant(0))); 513 MachineRepresentation rep = 514 (input_rep_type == SimdType::kFloat32x4) 515 ? MachineRepresentation::kWord32 516 : MachineTypeFrom(input_rep_type).representation(); 517 rep_node[i] = 518 d_cmp.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1)); 519 } 520 ReplaceNode(node, rep_node, num_lanes); 521 } 522 523 Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) { 524 return graph()->NewNode(machine()->Word32Sar(), 525 graph()->NewNode(machine()->Word32Shl(), input, 526 mcgraph_->Int32Constant(shift)), 527 mcgraph_->Int32Constant(shift)); 528 } 529 530 void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node, 531 SimdType input_rep_type, 532 const Operator* op, 533 bool not_horizontal) { 534 DCHECK_EQ(2, node->InputCount()); 535 DCHECK(input_rep_type == SimdType::kInt16x8 || 536 input_rep_type == SimdType::kInt8x16); 537 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type); 538 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type); 539 int num_lanes = NumLanes(input_rep_type); 540 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 541 int32_t shift_val = 542 (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8; 543 if (not_horizontal) { 544 for (int i = 0; i < num_lanes; ++i) { 545 rep_node[i] = FixUpperBits( 546 graph()->NewNode(op, rep_left[i], rep_right[i]), shift_val); 547 } 548 } else { 549 for (int i = 0; i < num_lanes / 2; ++i) { 550 rep_node[i] = FixUpperBits( 551 graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]), 552 shift_val); 553 rep_node[i + num_lanes / 2] = FixUpperBits( 554 graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]), 555 shift_val); 556 } 557 } 558 ReplaceNode(node, rep_node, num_lanes); 559 } 560 561 Node* SimdScalarLowering::Mask(Node* input, int32_t mask) { 562 return graph()->NewNode(machine()->Word32And(), input, 563 mcgraph_->Int32Constant(mask)); 564 } 565 566 void SimdScalarLowering::LowerSaturateBinaryOp(Node* node, 567 SimdType input_rep_type, 568 const Operator* op, 569 bool is_signed) { 570 DCHECK_EQ(2, node->InputCount()); 571 DCHECK(input_rep_type == SimdType::kInt16x8 || 572 input_rep_type == SimdType::kInt8x16); 573 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type); 574 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type); 575 int32_t min = 0; 576 int32_t max = 0; 577 int32_t mask = 0; 578 int32_t shift_val = 0; 579 MachineRepresentation phi_rep; 580 if (input_rep_type == SimdType::kInt16x8) { 581 if (is_signed) { 582 min = std::numeric_limits<int16_t>::min(); 583 max = std::numeric_limits<int16_t>::max(); 584 } else { 585 min = std::numeric_limits<uint16_t>::min(); 586 max = std::numeric_limits<uint16_t>::max(); 587 } 588 mask = kMask16; 589 shift_val = kShift16; 590 phi_rep = MachineRepresentation::kWord16; 591 } else { 592 if (is_signed) { 593 min = std::numeric_limits<int8_t>::min(); 594 max = std::numeric_limits<int8_t>::max(); 595 } else { 596 min = std::numeric_limits<uint8_t>::min(); 597 max = std::numeric_limits<uint8_t>::max(); 598 } 599 mask = kMask8; 600 shift_val = kShift8; 601 phi_rep = MachineRepresentation::kWord8; 602 } 603 int num_lanes = NumLanes(input_rep_type); 604 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 605 for (int i = 0; i < num_lanes; ++i) { 606 Node* op_result = nullptr; 607 Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask); 608 Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask); 609 op_result = graph()->NewNode(op, left, right); 610 Diamond d_min(graph(), common(), 611 graph()->NewNode(machine()->Int32LessThan(), op_result, 612 mcgraph_->Int32Constant(min))); 613 rep_node[i] = d_min.Phi(phi_rep, mcgraph_->Int32Constant(min), op_result); 614 Diamond d_max(graph(), common(), 615 graph()->NewNode(machine()->Int32LessThan(), 616 mcgraph_->Int32Constant(max), rep_node[i])); 617 rep_node[i] = d_max.Phi(phi_rep, mcgraph_->Int32Constant(max), rep_node[i]); 618 rep_node[i] = 619 is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val); 620 } 621 ReplaceNode(node, rep_node, num_lanes); 622 } 623 624 void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type, 625 const Operator* op) { 626 DCHECK_EQ(1, node->InputCount()); 627 Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type); 628 int num_lanes = NumLanes(input_rep_type); 629 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 630 for (int i = 0; i < num_lanes; ++i) { 631 rep_node[i] = graph()->NewNode(op, rep[i]); 632 } 633 ReplaceNode(node, rep_node, num_lanes); 634 } 635 636 void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op, 637 bool is_max, SimdType type) { 638 DCHECK_EQ(2, node->InputCount()); 639 Node** rep_left = GetReplacementsWithType(node->InputAt(0), type); 640 Node** rep_right = GetReplacementsWithType(node->InputAt(1), type); 641 int num_lanes = NumLanes(type); 642 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 643 MachineRepresentation rep = MachineRepresentation::kNone; 644 if (type == SimdType::kInt32x4) { 645 rep = MachineRepresentation::kWord32; 646 } else if (type == SimdType::kInt16x8) { 647 rep = MachineRepresentation::kWord16; 648 } else if (type == SimdType::kInt8x16) { 649 rep = MachineRepresentation::kWord8; 650 } else { 651 UNREACHABLE(); 652 } 653 for (int i = 0; i < num_lanes; ++i) { 654 Diamond d(graph(), common(), 655 graph()->NewNode(op, rep_left[i], rep_right[i])); 656 if (is_max) { 657 rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]); 658 } else { 659 rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]); 660 } 661 } 662 ReplaceNode(node, rep_node, num_lanes); 663 } 664 665 Node* SimdScalarLowering::BuildF64Trunc(Node* input) { 666 if (machine()->Float64RoundTruncate().IsSupported()) { 667 return graph()->NewNode(machine()->Float64RoundTruncate().op(), input); 668 } else { 669 ExternalReference ref = ExternalReference::wasm_f64_trunc(); 670 Node* stack_slot = 671 graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64)); 672 const Operator* store_op = machine()->Store( 673 StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier)); 674 Node* effect = 675 graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(0), 676 input, graph()->start(), graph()->start()); 677 Node* function = graph()->NewNode(common()->ExternalConstant(ref)); 678 Node** args = zone()->NewArray<Node*>(4); 679 args[0] = function; 680 args[1] = stack_slot; 681 args[2] = effect; 682 args[3] = graph()->start(); 683 Signature<MachineType>::Builder sig_builder(zone(), 0, 1); 684 sig_builder.AddParam(MachineType::Pointer()); 685 auto call_descriptor = 686 Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build()); 687 Node* call = graph()->NewNode(common()->Call(call_descriptor), 4, args); 688 return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()), 689 stack_slot, mcgraph_->Int32Constant(0), call, 690 graph()->start()); 691 } 692 } 693 694 void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) { 695 DCHECK_EQ(1, node->InputCount()); 696 Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4); 697 Node* rep_node[kNumLanes32]; 698 Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0)); 699 Node* min = graph()->NewNode( 700 common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0))); 701 Node* max = graph()->NewNode(common()->Float64Constant( 702 static_cast<double>(is_signed ? kMaxInt : 0xFFFFFFFFu))); 703 for (int i = 0; i < kNumLanes32; ++i) { 704 Node* double_rep = 705 graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]); 706 Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(), 707 double_rep, double_rep)); 708 Node* temp = 709 nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero); 710 Diamond min_d(graph(), common(), 711 graph()->NewNode(machine()->Float64LessThan(), temp, min)); 712 temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp); 713 Diamond max_d(graph(), common(), 714 graph()->NewNode(machine()->Float64LessThan(), max, temp)); 715 temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp); 716 Node* trunc = BuildF64Trunc(temp); 717 if (is_signed) { 718 rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc); 719 } else { 720 rep_node[i] = 721 graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc); 722 } 723 } 724 ReplaceNode(node, rep_node, kNumLanes32); 725 } 726 727 void SimdScalarLowering::LowerConvertFromInt(Node* node, 728 SimdType input_rep_type, 729 SimdType output_rep_type, 730 bool is_signed, int start_index) { 731 DCHECK_EQ(1, node->InputCount()); 732 Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type); 733 734 int32_t mask = 0; 735 if (input_rep_type == SimdType::kInt16x8) { 736 DCHECK_EQ(output_rep_type, SimdType::kInt32x4); 737 mask = kMask16; 738 } else { 739 DCHECK_EQ(output_rep_type, SimdType::kInt16x8); 740 DCHECK_EQ(input_rep_type, SimdType::kInt8x16); 741 mask = kMask8; 742 } 743 744 int num_lanes = NumLanes(output_rep_type); 745 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 746 for (int i = 0; i < num_lanes; ++i) { 747 rep_node[i] = 748 is_signed ? rep[i + start_index] : Mask(rep[i + start_index], mask); 749 } 750 751 ReplaceNode(node, rep_node, num_lanes); 752 } 753 754 void SimdScalarLowering::LowerPack(Node* node, SimdType input_rep_type, 755 SimdType output_rep_type, bool is_signed) { 756 DCHECK_EQ(2, node->InputCount()); 757 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type); 758 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type); 759 const Operator* less_op = 760 is_signed ? machine()->Int32LessThan() : machine()->Uint32LessThan(); 761 Node* min = nullptr; 762 Node* max = nullptr; 763 int32_t shift_val = 0; 764 MachineRepresentation phi_rep; 765 if (output_rep_type == SimdType::kInt16x8) { 766 DCHECK(input_rep_type == SimdType::kInt32x4); 767 if (is_signed) { 768 min = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::min()); 769 max = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::max()); 770 } else { 771 max = mcgraph_->Uint32Constant(std::numeric_limits<uint16_t>::max()); 772 shift_val = kShift16; 773 } 774 phi_rep = MachineRepresentation::kWord16; 775 } else { 776 DCHECK(output_rep_type == SimdType::kInt8x16 && 777 input_rep_type == SimdType::kInt16x8); 778 if (is_signed) { 779 min = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::min()); 780 max = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::max()); 781 } else { 782 max = mcgraph_->Uint32Constant(std::numeric_limits<uint8_t>::max()); 783 shift_val = kShift8; 784 } 785 phi_rep = MachineRepresentation::kWord8; 786 } 787 int num_lanes = NumLanes(output_rep_type); 788 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 789 for (int i = 0; i < num_lanes; ++i) { 790 Node* input = nullptr; 791 if (i < num_lanes / 2) 792 input = rep_left[i]; 793 else 794 input = rep_right[i - num_lanes / 2]; 795 if (is_signed) { 796 Diamond d_min(graph(), common(), graph()->NewNode(less_op, input, min)); 797 input = d_min.Phi(phi_rep, min, input); 798 } 799 Diamond d_max(graph(), common(), graph()->NewNode(less_op, max, input)); 800 rep_node[i] = d_max.Phi(phi_rep, max, input); 801 rep_node[i] = 802 is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val); 803 } 804 ReplaceNode(node, rep_node, num_lanes); 805 } 806 807 void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) { 808 DCHECK_EQ(1, node->InputCount()); 809 int32_t shift_amount = OpParameter<int32_t>(node->op()); 810 Node* shift_node = graph()->NewNode(common()->Int32Constant(shift_amount)); 811 Node** rep = GetReplacementsWithType(node->InputAt(0), type); 812 int num_lanes = NumLanes(type); 813 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 814 for (int i = 0; i < num_lanes; ++i) { 815 rep_node[i] = rep[i]; 816 switch (node->opcode()) { 817 case IrOpcode::kI8x16ShrU: 818 rep_node[i] = Mask(rep_node[i], kMask8); 819 rep_node[i] = 820 graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node); 821 break; 822 case IrOpcode::kI16x8ShrU: 823 rep_node[i] = Mask(rep_node[i], kMask16); 824 V8_FALLTHROUGH; 825 case IrOpcode::kI32x4ShrU: 826 rep_node[i] = 827 graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node); 828 break; 829 case IrOpcode::kI32x4Shl: 830 rep_node[i] = 831 graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node); 832 break; 833 case IrOpcode::kI16x8Shl: 834 rep_node[i] = 835 graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node); 836 rep_node[i] = FixUpperBits(rep_node[i], kShift16); 837 break; 838 case IrOpcode::kI8x16Shl: 839 rep_node[i] = 840 graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node); 841 rep_node[i] = FixUpperBits(rep_node[i], kShift8); 842 break; 843 case IrOpcode::kI32x4ShrS: 844 case IrOpcode::kI16x8ShrS: 845 case IrOpcode::kI8x16ShrS: 846 rep_node[i] = 847 graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node); 848 break; 849 default: 850 UNREACHABLE(); 851 } 852 } 853 ReplaceNode(node, rep_node, num_lanes); 854 } 855 856 void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type, 857 const Operator* op) { 858 DCHECK_EQ(2, node->InputCount()); 859 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type); 860 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type); 861 int num_lanes = NumLanes(input_rep_type); 862 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 863 for (int i = 0; i < num_lanes; ++i) { 864 Diamond d(graph(), common(), 865 graph()->NewNode(op, rep_left[i], rep_right[i])); 866 MachineRepresentation rep = 867 (input_rep_type == SimdType::kFloat32x4) 868 ? MachineRepresentation::kWord32 869 : MachineTypeFrom(input_rep_type).representation(); 870 rep_node[i] = 871 d.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1)); 872 } 873 ReplaceNode(node, rep_node, num_lanes); 874 } 875 876 void SimdScalarLowering::LowerNode(Node* node) { 877 SimdType rep_type = ReplacementType(node); 878 int num_lanes = NumLanes(rep_type); 879 switch (node->opcode()) { 880 case IrOpcode::kStart: { 881 int parameter_count = GetParameterCountAfterLowering(); 882 // Only exchange the node if the parameter count actually changed. 883 if (parameter_count != static_cast<int>(signature()->parameter_count())) { 884 int delta = 885 parameter_count - static_cast<int>(signature()->parameter_count()); 886 int new_output_count = node->op()->ValueOutputCount() + delta; 887 NodeProperties::ChangeOp(node, common()->Start(new_output_count)); 888 } 889 break; 890 } 891 case IrOpcode::kParameter: { 892 DCHECK_EQ(1, node->InputCount()); 893 // Only exchange the node if the parameter count actually changed. We do 894 // not even have to do the default lowering because the the start node, 895 // the only input of a parameter node, only changes if the parameter count 896 // changes. 897 if (GetParameterCountAfterLowering() != 898 static_cast<int>(signature()->parameter_count())) { 899 int old_index = ParameterIndexOf(node->op()); 900 int new_index = 901 GetParameterIndexAfterLoweringSimd128(signature(), old_index); 902 if (old_index == new_index) { 903 NodeProperties::ChangeOp(node, common()->Parameter(new_index)); 904 905 Node* new_node[kNumLanes32]; 906 for (int i = 0; i < kNumLanes32; ++i) { 907 new_node[i] = nullptr; 908 } 909 new_node[0] = node; 910 if (signature()->GetParam(old_index) == 911 MachineRepresentation::kSimd128) { 912 for (int i = 1; i < kNumLanes32; ++i) { 913 new_node[i] = graph()->NewNode(common()->Parameter(new_index + i), 914 graph()->start()); 915 } 916 } 917 ReplaceNode(node, new_node, kNumLanes32); 918 } 919 } 920 break; 921 } 922 case IrOpcode::kLoad: 923 case IrOpcode::kUnalignedLoad: 924 case IrOpcode::kProtectedLoad: { 925 LowerLoadOp(node, rep_type); 926 break; 927 } 928 case IrOpcode::kStore: 929 case IrOpcode::kUnalignedStore: 930 case IrOpcode::kProtectedStore: { 931 LowerStoreOp(node); 932 break; 933 } 934 case IrOpcode::kReturn: { 935 DefaultLowering(node); 936 int new_return_count = GetReturnCountAfterLoweringSimd128(signature()); 937 if (static_cast<int>(signature()->return_count()) != new_return_count) { 938 NodeProperties::ChangeOp(node, common()->Return(new_return_count)); 939 } 940 break; 941 } 942 case IrOpcode::kCall: { 943 // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor. 944 auto call_descriptor = 945 const_cast<CallDescriptor*>(CallDescriptorOf(node->op())); 946 if (DefaultLowering(node) || 947 (call_descriptor->ReturnCount() == 1 && 948 call_descriptor->GetReturnType(0) == MachineType::Simd128())) { 949 // We have to adjust the call descriptor. 950 const Operator* op = common()->Call( 951 GetI32WasmCallDescriptorForSimd(zone(), call_descriptor)); 952 NodeProperties::ChangeOp(node, op); 953 } 954 if (call_descriptor->ReturnCount() == 1 && 955 call_descriptor->GetReturnType(0) == MachineType::Simd128()) { 956 // We access the additional return values through projections. 957 Node* rep_node[kNumLanes32]; 958 for (int i = 0; i < kNumLanes32; ++i) { 959 rep_node[i] = 960 graph()->NewNode(common()->Projection(i), node, graph()->start()); 961 } 962 ReplaceNode(node, rep_node, kNumLanes32); 963 } 964 break; 965 } 966 case IrOpcode::kPhi: { 967 MachineRepresentation rep = PhiRepresentationOf(node->op()); 968 if (rep == MachineRepresentation::kSimd128) { 969 // The replacement nodes have already been created, we only have to 970 // replace placeholder nodes. 971 Node** rep_node = GetReplacements(node); 972 for (int i = 0; i < node->op()->ValueInputCount(); ++i) { 973 Node** rep_input = 974 GetReplacementsWithType(node->InputAt(i), rep_type); 975 for (int j = 0; j < num_lanes; j++) { 976 rep_node[j]->ReplaceInput(i, rep_input[j]); 977 } 978 } 979 } else { 980 DefaultLowering(node); 981 } 982 break; 983 } 984 #define I32X4_BINOP_CASE(opcode, instruction) \ 985 case IrOpcode::opcode: { \ 986 LowerBinaryOp(node, rep_type, machine()->instruction()); \ 987 break; \ 988 } 989 I32X4_BINOP_CASE(kI32x4Add, Int32Add) 990 I32X4_BINOP_CASE(kI32x4Sub, Int32Sub) 991 I32X4_BINOP_CASE(kI32x4Mul, Int32Mul) 992 I32X4_BINOP_CASE(kS128And, Word32And) 993 I32X4_BINOP_CASE(kS128Or, Word32Or) 994 I32X4_BINOP_CASE(kS128Xor, Word32Xor) 995 #undef I32X4_BINOP_CASE 996 case IrOpcode::kI32x4AddHoriz: { 997 LowerBinaryOp(node, rep_type, machine()->Int32Add(), false); 998 break; 999 } 1000 case IrOpcode::kI16x8AddHoriz: { 1001 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add(), false); 1002 break; 1003 } 1004 case IrOpcode::kI16x8Add: 1005 case IrOpcode::kI8x16Add: { 1006 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add()); 1007 break; 1008 } 1009 case IrOpcode::kI16x8Sub: 1010 case IrOpcode::kI8x16Sub: { 1011 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub()); 1012 break; 1013 } 1014 case IrOpcode::kI16x8Mul: 1015 case IrOpcode::kI8x16Mul: { 1016 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul()); 1017 break; 1018 } 1019 case IrOpcode::kI16x8AddSaturateS: 1020 case IrOpcode::kI8x16AddSaturateS: { 1021 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true); 1022 break; 1023 } 1024 case IrOpcode::kI16x8SubSaturateS: 1025 case IrOpcode::kI8x16SubSaturateS: { 1026 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true); 1027 break; 1028 } 1029 case IrOpcode::kI16x8AddSaturateU: 1030 case IrOpcode::kI8x16AddSaturateU: { 1031 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false); 1032 break; 1033 } 1034 case IrOpcode::kI16x8SubSaturateU: 1035 case IrOpcode::kI8x16SubSaturateU: { 1036 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false); 1037 break; 1038 } 1039 case IrOpcode::kI32x4MaxS: 1040 case IrOpcode::kI16x8MaxS: 1041 case IrOpcode::kI8x16MaxS: { 1042 LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type); 1043 break; 1044 } 1045 case IrOpcode::kI32x4MinS: 1046 case IrOpcode::kI16x8MinS: 1047 case IrOpcode::kI8x16MinS: { 1048 LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type); 1049 break; 1050 } 1051 case IrOpcode::kI32x4MaxU: 1052 case IrOpcode::kI16x8MaxU: 1053 case IrOpcode::kI8x16MaxU: { 1054 LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type); 1055 break; 1056 } 1057 case IrOpcode::kI32x4MinU: 1058 case IrOpcode::kI16x8MinU: 1059 case IrOpcode::kI8x16MinU: { 1060 LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type); 1061 break; 1062 } 1063 case IrOpcode::kI32x4Neg: 1064 case IrOpcode::kI16x8Neg: 1065 case IrOpcode::kI8x16Neg: { 1066 DCHECK_EQ(1, node->InputCount()); 1067 Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type); 1068 int num_lanes = NumLanes(rep_type); 1069 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1070 Node* zero = graph()->NewNode(common()->Int32Constant(0)); 1071 for (int i = 0; i < num_lanes; ++i) { 1072 rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]); 1073 if (node->opcode() == IrOpcode::kI16x8Neg) { 1074 rep_node[i] = FixUpperBits(rep_node[i], kShift16); 1075 } else if (node->opcode() == IrOpcode::kI8x16Neg) { 1076 rep_node[i] = FixUpperBits(rep_node[i], kShift8); 1077 } 1078 } 1079 ReplaceNode(node, rep_node, num_lanes); 1080 break; 1081 } 1082 case IrOpcode::kS128Not: { 1083 DCHECK_EQ(1, node->InputCount()); 1084 Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type); 1085 Node* rep_node[kNumLanes32]; 1086 Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF)); 1087 for (int i = 0; i < kNumLanes32; ++i) { 1088 rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask); 1089 } 1090 ReplaceNode(node, rep_node, kNumLanes32); 1091 break; 1092 } 1093 case IrOpcode::kI32x4SConvertF32x4: { 1094 LowerConvertFromFloat(node, true); 1095 break; 1096 } 1097 case IrOpcode::kI32x4UConvertF32x4: { 1098 LowerConvertFromFloat(node, false); 1099 break; 1100 } 1101 case IrOpcode::kI32x4SConvertI16x8Low: { 1102 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true, 1103 0); 1104 break; 1105 } 1106 case IrOpcode::kI32x4SConvertI16x8High: { 1107 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true, 1108 4); 1109 break; 1110 } 1111 case IrOpcode::kI32x4UConvertI16x8Low: { 1112 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false, 1113 0); 1114 break; 1115 } 1116 case IrOpcode::kI32x4UConvertI16x8High: { 1117 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false, 1118 4); 1119 break; 1120 } 1121 case IrOpcode::kI16x8SConvertI8x16Low: { 1122 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true, 1123 0); 1124 break; 1125 } 1126 case IrOpcode::kI16x8SConvertI8x16High: { 1127 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true, 1128 8); 1129 break; 1130 } 1131 case IrOpcode::kI16x8UConvertI8x16Low: { 1132 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false, 1133 0); 1134 break; 1135 } 1136 case IrOpcode::kI16x8UConvertI8x16High: { 1137 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false, 1138 8); 1139 break; 1140 } 1141 case IrOpcode::kI16x8SConvertI32x4: { 1142 LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, true); 1143 break; 1144 } 1145 case IrOpcode::kI16x8UConvertI32x4: { 1146 LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, false); 1147 break; 1148 } 1149 case IrOpcode::kI8x16SConvertI16x8: { 1150 LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, true); 1151 break; 1152 } 1153 case IrOpcode::kI8x16UConvertI16x8: { 1154 LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, false); 1155 break; 1156 } 1157 case IrOpcode::kI32x4Shl: 1158 case IrOpcode::kI16x8Shl: 1159 case IrOpcode::kI8x16Shl: 1160 case IrOpcode::kI32x4ShrS: 1161 case IrOpcode::kI16x8ShrS: 1162 case IrOpcode::kI8x16ShrS: 1163 case IrOpcode::kI32x4ShrU: 1164 case IrOpcode::kI16x8ShrU: 1165 case IrOpcode::kI8x16ShrU: { 1166 LowerShiftOp(node, rep_type); 1167 break; 1168 } 1169 case IrOpcode::kF32x4AddHoriz: { 1170 LowerBinaryOp(node, rep_type, machine()->Float32Add(), false); 1171 break; 1172 } 1173 #define F32X4_BINOP_CASE(name) \ 1174 case IrOpcode::kF32x4##name: { \ 1175 LowerBinaryOp(node, rep_type, machine()->Float32##name()); \ 1176 break; \ 1177 } 1178 F32X4_BINOP_CASE(Add) 1179 F32X4_BINOP_CASE(Sub) 1180 F32X4_BINOP_CASE(Mul) 1181 F32X4_BINOP_CASE(Min) 1182 F32X4_BINOP_CASE(Max) 1183 #undef F32X4_BINOP_CASE 1184 #define F32X4_UNOP_CASE(name) \ 1185 case IrOpcode::kF32x4##name: { \ 1186 LowerUnaryOp(node, rep_type, machine()->Float32##name()); \ 1187 break; \ 1188 } 1189 F32X4_UNOP_CASE(Abs) 1190 F32X4_UNOP_CASE(Neg) 1191 #undef F32x4_UNOP_CASE 1192 case IrOpcode::kF32x4RecipApprox: 1193 case IrOpcode::kF32x4RecipSqrtApprox: { 1194 DCHECK_EQ(1, node->InputCount()); 1195 Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type); 1196 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1197 Node* float_one = graph()->NewNode(common()->Float32Constant(1.0)); 1198 for (int i = 0; i < num_lanes; ++i) { 1199 Node* tmp = rep[i]; 1200 if (node->opcode() == IrOpcode::kF32x4RecipSqrtApprox) { 1201 tmp = graph()->NewNode(machine()->Float32Sqrt(), rep[i]); 1202 } 1203 rep_node[i] = graph()->NewNode(machine()->Float32Div(), float_one, tmp); 1204 } 1205 ReplaceNode(node, rep_node, num_lanes); 1206 break; 1207 } 1208 case IrOpcode::kF32x4SConvertI32x4: { 1209 LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32()); 1210 break; 1211 } 1212 case IrOpcode::kF32x4UConvertI32x4: { 1213 LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32()); 1214 break; 1215 } 1216 case IrOpcode::kI32x4Splat: 1217 case IrOpcode::kF32x4Splat: 1218 case IrOpcode::kI16x8Splat: 1219 case IrOpcode::kI8x16Splat: { 1220 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1221 for (int i = 0; i < num_lanes; ++i) { 1222 if (HasReplacement(0, node->InputAt(0))) { 1223 rep_node[i] = GetReplacements(node->InputAt(0))[0]; 1224 } else { 1225 rep_node[i] = node->InputAt(0); 1226 } 1227 } 1228 ReplaceNode(node, rep_node, num_lanes); 1229 break; 1230 } 1231 case IrOpcode::kI32x4ExtractLane: 1232 case IrOpcode::kF32x4ExtractLane: 1233 case IrOpcode::kI16x8ExtractLane: 1234 case IrOpcode::kI8x16ExtractLane: { 1235 int32_t lane = OpParameter<int32_t>(node->op()); 1236 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1237 rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane]; 1238 for (int i = 1; i < num_lanes; ++i) { 1239 rep_node[i] = nullptr; 1240 } 1241 ReplaceNode(node, rep_node, num_lanes); 1242 break; 1243 } 1244 case IrOpcode::kI32x4ReplaceLane: 1245 case IrOpcode::kF32x4ReplaceLane: 1246 case IrOpcode::kI16x8ReplaceLane: 1247 case IrOpcode::kI8x16ReplaceLane: { 1248 DCHECK_EQ(2, node->InputCount()); 1249 Node* repNode = node->InputAt(1); 1250 int32_t lane = OpParameter<int32_t>(node->op()); 1251 Node** old_rep_node = GetReplacementsWithType(node->InputAt(0), rep_type); 1252 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1253 for (int i = 0; i < num_lanes; ++i) { 1254 rep_node[i] = old_rep_node[i]; 1255 } 1256 if (HasReplacement(0, repNode)) { 1257 rep_node[lane] = GetReplacements(repNode)[0]; 1258 } else { 1259 rep_node[lane] = repNode; 1260 } 1261 ReplaceNode(node, rep_node, num_lanes); 1262 break; 1263 } 1264 #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \ 1265 case IrOpcode::simd_op: { \ 1266 LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \ 1267 break; \ 1268 } 1269 COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false) 1270 COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false) 1271 COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false) 1272 COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true) 1273 COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true) 1274 COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false) 1275 COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false) 1276 COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false) 1277 COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true) 1278 COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true) 1279 COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false) 1280 COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false) 1281 COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true) 1282 COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true) 1283 COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false) 1284 COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false) 1285 COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false) 1286 COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true) 1287 COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true) 1288 COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false) 1289 COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false) 1290 COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true) 1291 COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true) 1292 COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false) 1293 COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false) 1294 COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false) 1295 COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true) 1296 COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true) 1297 COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false) 1298 COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false) 1299 COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true) 1300 COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true) 1301 #undef COMPARISON_CASE 1302 case IrOpcode::kF32x4Ne: { 1303 LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal()); 1304 break; 1305 } 1306 case IrOpcode::kI32x4Ne: { 1307 LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal()); 1308 break; 1309 } 1310 case IrOpcode::kI16x8Ne: { 1311 LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal()); 1312 break; 1313 } 1314 case IrOpcode::kI8x16Ne: { 1315 LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal()); 1316 break; 1317 } 1318 case IrOpcode::kS128Select: { 1319 DCHECK_EQ(3, node->InputCount()); 1320 DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 || 1321 ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 || 1322 ReplacementType(node->InputAt(0)) == SimdType::kInt8x16); 1323 Node** boolean_input = GetReplacements(node->InputAt(0)); 1324 Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type); 1325 Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type); 1326 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1327 for (int i = 0; i < num_lanes; ++i) { 1328 Node* tmp1 = 1329 graph()->NewNode(machine()->Word32Xor(), rep_left[i], rep_right[i]); 1330 Node* tmp2 = 1331 graph()->NewNode(machine()->Word32And(), boolean_input[i], tmp1); 1332 rep_node[i] = 1333 graph()->NewNode(machine()->Word32Xor(), rep_right[i], tmp2); 1334 } 1335 ReplaceNode(node, rep_node, num_lanes); 1336 break; 1337 } 1338 case IrOpcode::kS8x16Shuffle: { 1339 DCHECK_EQ(2, node->InputCount()); 1340 const uint8_t* shuffle = OpParameter<uint8_t*>(node->op()); 1341 Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type); 1342 Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type); 1343 Node** rep_node = zone()->NewArray<Node*>(16); 1344 for (int i = 0; i < 16; i++) { 1345 int lane = shuffle[i]; 1346 rep_node[i] = lane < 16 ? rep_left[lane] : rep_right[lane - 16]; 1347 } 1348 ReplaceNode(node, rep_node, 16); 1349 break; 1350 } 1351 case IrOpcode::kS1x4AnyTrue: 1352 case IrOpcode::kS1x4AllTrue: 1353 case IrOpcode::kS1x8AnyTrue: 1354 case IrOpcode::kS1x8AllTrue: 1355 case IrOpcode::kS1x16AnyTrue: 1356 case IrOpcode::kS1x16AllTrue: { 1357 DCHECK_EQ(1, node->InputCount()); 1358 SimdType input_rep_type = ReplacementType(node->InputAt(0)); 1359 int input_num_lanes = NumLanes(input_rep_type); 1360 Node** rep = GetReplacements(node->InputAt(0)); 1361 Node** rep_node = zone()->NewArray<Node*>(num_lanes); 1362 Node* true_node = mcgraph_->Int32Constant(-1); 1363 Node* false_node = mcgraph_->Int32Constant(0); 1364 Node* tmp_result = false_node; 1365 if (node->opcode() == IrOpcode::kS1x4AllTrue || 1366 node->opcode() == IrOpcode::kS1x8AllTrue || 1367 node->opcode() == IrOpcode::kS1x16AllTrue) { 1368 tmp_result = true_node; 1369 } 1370 for (int i = 0; i < input_num_lanes; ++i) { 1371 Diamond is_false( 1372 graph(), common(), 1373 graph()->NewNode(machine()->Word32Equal(), rep[i], false_node)); 1374 if (node->opcode() == IrOpcode::kS1x4AllTrue || 1375 node->opcode() == IrOpcode::kS1x8AllTrue || 1376 node->opcode() == IrOpcode::kS1x16AllTrue) { 1377 tmp_result = is_false.Phi(MachineRepresentation::kWord32, false_node, 1378 tmp_result); 1379 } else { 1380 tmp_result = is_false.Phi(MachineRepresentation::kWord32, tmp_result, 1381 true_node); 1382 } 1383 } 1384 rep_node[0] = tmp_result; 1385 for (int i = 1; i < num_lanes; ++i) { 1386 rep_node[i] = nullptr; 1387 } 1388 ReplaceNode(node, rep_node, num_lanes); 1389 break; 1390 } 1391 default: { DefaultLowering(node); } 1392 } 1393 } 1394 1395 bool SimdScalarLowering::DefaultLowering(Node* node) { 1396 bool something_changed = false; 1397 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) { 1398 Node* input = node->InputAt(i); 1399 if (HasReplacement(0, input)) { 1400 something_changed = true; 1401 node->ReplaceInput(i, GetReplacements(input)[0]); 1402 } 1403 if (HasReplacement(1, input)) { 1404 something_changed = true; 1405 for (int j = 1; j < ReplacementCount(input); ++j) { 1406 node->InsertInput(zone(), i + j, GetReplacements(input)[j]); 1407 } 1408 } 1409 } 1410 return something_changed; 1411 } 1412 1413 void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) { 1414 replacements_[old->id()].node = zone()->NewArray<Node*>(count); 1415 for (int i = 0; i < count; ++i) { 1416 replacements_[old->id()].node[i] = new_nodes[i]; 1417 } 1418 replacements_[old->id()].num_replacements = count; 1419 } 1420 1421 bool SimdScalarLowering::HasReplacement(size_t index, Node* node) { 1422 return replacements_[node->id()].node != nullptr && 1423 replacements_[node->id()].node[index] != nullptr; 1424 } 1425 1426 SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) { 1427 return replacements_[node->id()].type; 1428 } 1429 1430 Node** SimdScalarLowering::GetReplacements(Node* node) { 1431 Node** result = replacements_[node->id()].node; 1432 DCHECK(result); 1433 return result; 1434 } 1435 1436 int SimdScalarLowering::ReplacementCount(Node* node) { 1437 return replacements_[node->id()].num_replacements; 1438 } 1439 1440 void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) { 1441 for (int i = 0; i < kNumLanes32; ++i) { 1442 if (replacements[i] != nullptr) { 1443 result[i] = 1444 graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]); 1445 } else { 1446 result[i] = nullptr; 1447 } 1448 } 1449 } 1450 1451 void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) { 1452 for (int i = 0; i < kNumLanes32; ++i) { 1453 if (replacements[i] != nullptr) { 1454 result[i] = 1455 graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]); 1456 } else { 1457 result[i] = nullptr; 1458 } 1459 } 1460 } 1461 1462 template <typename T> 1463 void SimdScalarLowering::Int32ToSmallerInt(Node** replacements, Node** result) { 1464 const int num_ints = sizeof(int32_t) / sizeof(T); 1465 const int bit_size = sizeof(T) * 8; 1466 const Operator* sign_extend; 1467 switch (sizeof(T)) { 1468 case 1: 1469 sign_extend = machine()->SignExtendWord8ToInt32(); 1470 break; 1471 case 2: 1472 sign_extend = machine()->SignExtendWord16ToInt32(); 1473 break; 1474 default: 1475 UNREACHABLE(); 1476 } 1477 1478 for (int i = 0; i < kNumLanes32; i++) { 1479 if (replacements[i] != nullptr) { 1480 for (int j = 0; j < num_ints; j++) { 1481 result[num_ints * i + j] = graph()->NewNode( 1482 sign_extend, 1483 graph()->NewNode(machine()->Word32Sar(), replacements[i], 1484 mcgraph_->Int32Constant(j * bit_size))); 1485 } 1486 } else { 1487 for (int j = 0; j < num_ints; j++) { 1488 result[num_ints * i + j] = nullptr; 1489 } 1490 } 1491 } 1492 } 1493 1494 template <typename T> 1495 void SimdScalarLowering::SmallerIntToInt32(Node** replacements, Node** result) { 1496 const int num_ints = sizeof(int32_t) / sizeof(T); 1497 const int bit_size = sizeof(T) * 8; 1498 const int bit_mask = (1 << bit_size) - 1; 1499 1500 for (int i = 0; i < kNumLanes32; ++i) { 1501 result[i] = mcgraph_->Int32Constant(0); 1502 for (int j = 0; j < num_ints; j++) { 1503 if (replacements[num_ints * i + j] != nullptr) { 1504 Node* clean_bits = graph()->NewNode(machine()->Word32And(), 1505 replacements[num_ints * i + j], 1506 mcgraph_->Int32Constant(bit_mask)); 1507 Node* shift = graph()->NewNode(machine()->Word32Shl(), clean_bits, 1508 mcgraph_->Int32Constant(j * bit_size)); 1509 result[i] = graph()->NewNode(machine()->Word32Or(), result[i], shift); 1510 } 1511 } 1512 } 1513 } 1514 1515 Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) { 1516 Node** replacements = GetReplacements(node); 1517 if (ReplacementType(node) == type) { 1518 return GetReplacements(node); 1519 } 1520 int num_lanes = NumLanes(type); 1521 Node** result = zone()->NewArray<Node*>(num_lanes); 1522 if (type == SimdType::kInt32x4) { 1523 if (ReplacementType(node) == SimdType::kFloat32x4) { 1524 Float32ToInt32(replacements, result); 1525 } else if (ReplacementType(node) == SimdType::kInt16x8) { 1526 SmallerIntToInt32<int16_t>(replacements, result); 1527 } else if (ReplacementType(node) == SimdType::kInt8x16) { 1528 SmallerIntToInt32<int8_t>(replacements, result); 1529 } else { 1530 UNREACHABLE(); 1531 } 1532 } else if (type == SimdType::kFloat32x4) { 1533 if (ReplacementType(node) == SimdType::kInt32x4) { 1534 Int32ToFloat32(replacements, result); 1535 } else if (ReplacementType(node) == SimdType::kInt16x8) { 1536 UNIMPLEMENTED(); 1537 } else { 1538 UNREACHABLE(); 1539 } 1540 } else if (type == SimdType::kInt16x8) { 1541 if (ReplacementType(node) == SimdType::kInt32x4) { 1542 Int32ToSmallerInt<int16_t>(replacements, result); 1543 } else if (ReplacementType(node) == SimdType::kFloat32x4) { 1544 UNIMPLEMENTED(); 1545 } else { 1546 UNREACHABLE(); 1547 } 1548 } else if (type == SimdType::kInt8x16) { 1549 if (ReplacementType(node) == SimdType::kInt32x4) { 1550 Int32ToSmallerInt<int8_t>(replacements, result); 1551 } else { 1552 UNIMPLEMENTED(); 1553 } 1554 } else { 1555 UNREACHABLE(); 1556 } 1557 return result; 1558 } 1559 1560 void SimdScalarLowering::PreparePhiReplacement(Node* phi) { 1561 MachineRepresentation rep = PhiRepresentationOf(phi->op()); 1562 if (rep == MachineRepresentation::kSimd128) { 1563 // We have to create the replacements for a phi node before we actually 1564 // lower the phi to break potential cycles in the graph. The replacements of 1565 // input nodes do not exist yet, so we use a placeholder node to pass the 1566 // graph verifier. 1567 int value_count = phi->op()->ValueInputCount(); 1568 SimdType type = ReplacementType(phi); 1569 int num_lanes = NumLanes(type); 1570 Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes); 1571 for (int i = 0; i < num_lanes; ++i) { 1572 inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1); 1573 inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0); 1574 } 1575 for (int i = 0; i < value_count; ++i) { 1576 for (int j = 0; j < num_lanes; ++j) { 1577 inputs_rep[j][i] = placeholder_; 1578 } 1579 } 1580 Node** rep_nodes = zone()->NewArray<Node*>(num_lanes); 1581 for (int i = 0; i < num_lanes; ++i) { 1582 rep_nodes[i] = graph()->NewNode( 1583 common()->Phi(MachineTypeFrom(type).representation(), value_count), 1584 value_count + 1, inputs_rep[i], false); 1585 } 1586 ReplaceNode(phi, rep_nodes, num_lanes); 1587 } 1588 } 1589 } // namespace compiler 1590 } // namespace internal 1591 } // namespace v8 1592
