1 // Copyright 2016 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/simd-scalar-lowering.h" 6 #include "src/compiler/diamond.h" 7 #include "src/compiler/linkage.h" 8 #include "src/compiler/node-matchers.h" 9 #include "src/compiler/node-properties.h" 10 11 #include "src/compiler/node.h" 12 #include "src/objects-inl.h" 13 #include "src/wasm/wasm-module.h" 14 15 namespace v8 { 16 namespace internal { 17 namespace compiler { 18 19 SimdScalarLowering::SimdScalarLowering( 20 Graph* graph, MachineOperatorBuilder* machine, 21 CommonOperatorBuilder* common, Zone* zone, 22 Signature<MachineRepresentation>* signature) 23 : zone_(zone), 24 graph_(graph), 25 machine_(machine), 26 common_(common), 27 state_(graph, 3), 28 stack_(zone), 29 replacements_(nullptr), 30 signature_(signature), 31 placeholder_( 32 graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())), 33 parameter_count_after_lowering_(-1) { 34 DCHECK_NOT_NULL(graph); 35 DCHECK_NOT_NULL(graph->end()); 36 replacements_ = zone->NewArray<Replacement>(graph->NodeCount()); 37 memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount()); 38 } 39 40 void SimdScalarLowering::LowerGraph() { 41 stack_.push_back({graph()->end(), 0}); 42 state_.Set(graph()->end(), State::kOnStack); 43 replacements_[graph()->end()->id()].type = SimdType::kInt32; 44 45 while (!stack_.empty()) { 46 NodeState& top = stack_.back(); 47 if (top.input_index == top.node->InputCount()) { 48 // All inputs of top have already been lowered, now lower top. 49 stack_.pop_back(); 50 state_.Set(top.node, State::kVisited); 51 LowerNode(top.node); 52 } else { 53 // Push the next input onto the stack. 54 Node* input = top.node->InputAt(top.input_index++); 55 if (state_.Get(input) == State::kUnvisited) { 56 SetLoweredType(input, top.node); 57 if (input->opcode() == IrOpcode::kPhi) { 58 // To break cycles with phi nodes we push phis on a separate stack so 59 // that they are processed after all other nodes. 60 PreparePhiReplacement(input); 61 stack_.push_front({input, 0}); 62 } else if (input->opcode() == IrOpcode::kEffectPhi || 63 input->opcode() == IrOpcode::kLoop) { 64 stack_.push_front({input, 0}); 65 } else { 66 stack_.push_back({input, 0}); 67 } 68 state_.Set(input, State::kOnStack); 69 } 70 } 71 } 72 } 73 74 #define FOREACH_INT32X4_OPCODE(V) \ 75 V(Int32x4Add) \ 76 V(Int32x4ExtractLane) \ 77 V(CreateInt32x4) \ 78 V(Int32x4ReplaceLane) 79 80 #define FOREACH_FLOAT32X4_OPCODE(V) \ 81 V(Float32x4Add) \ 82 V(Float32x4ExtractLane) \ 83 V(CreateFloat32x4) \ 84 V(Float32x4ReplaceLane) 85 86 void SimdScalarLowering::SetLoweredType(Node* node, Node* output) { 87 switch (node->opcode()) { 88 #define CASE_STMT(name) case IrOpcode::k##name: 89 FOREACH_INT32X4_OPCODE(CASE_STMT) 90 case IrOpcode::kReturn: 91 case IrOpcode::kParameter: 92 case IrOpcode::kCall: { 93 replacements_[node->id()].type = SimdType::kInt32; 94 break; 95 } 96 FOREACH_FLOAT32X4_OPCODE(CASE_STMT) { 97 replacements_[node->id()].type = SimdType::kFloat32; 98 break; 99 } 100 #undef CASE_STMT 101 default: 102 replacements_[node->id()].type = replacements_[output->id()].type; 103 } 104 } 105 106 static int GetParameterIndexAfterLowering( 107 Signature<MachineRepresentation>* signature, int old_index) { 108 // In function calls, the simd128 types are passed as 4 Int32 types. The 109 // parameters are typecast to the types as needed for various operations. 110 int result = old_index; 111 for (int i = 0; i < old_index; ++i) { 112 if (signature->GetParam(i) == MachineRepresentation::kSimd128) { 113 result += 3; 114 } 115 } 116 return result; 117 } 118 119 int SimdScalarLowering::GetParameterCountAfterLowering() { 120 if (parameter_count_after_lowering_ == -1) { 121 // GetParameterIndexAfterLowering(parameter_count) returns the parameter 122 // count after lowering. 123 parameter_count_after_lowering_ = GetParameterIndexAfterLowering( 124 signature(), static_cast<int>(signature()->parameter_count())); 125 } 126 return parameter_count_after_lowering_; 127 } 128 129 static int GetReturnCountAfterLowering( 130 Signature<MachineRepresentation>* signature) { 131 int result = static_cast<int>(signature->return_count()); 132 for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) { 133 if (signature->GetReturn(i) == MachineRepresentation::kSimd128) { 134 result += 3; 135 } 136 } 137 return result; 138 } 139 140 void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices) { 141 new_indices[0] = index; 142 for (size_t i = 1; i < kMaxLanes; ++i) { 143 new_indices[i] = graph()->NewNode(machine()->Int32Add(), index, 144 graph()->NewNode(common()->Int32Constant( 145 static_cast<int>(i) * kLaneWidth))); 146 } 147 } 148 149 void SimdScalarLowering::LowerLoadOp(MachineRepresentation rep, Node* node, 150 const Operator* load_op) { 151 if (rep == MachineRepresentation::kSimd128) { 152 Node* base = node->InputAt(0); 153 Node* index = node->InputAt(1); 154 Node* indices[kMaxLanes]; 155 GetIndexNodes(index, indices); 156 Node* rep_nodes[kMaxLanes]; 157 rep_nodes[0] = node; 158 NodeProperties::ChangeOp(rep_nodes[0], load_op); 159 if (node->InputCount() > 2) { 160 DCHECK(node->InputCount() > 3); 161 Node* effect_input = node->InputAt(2); 162 Node* control_input = node->InputAt(3); 163 rep_nodes[3] = graph()->NewNode(load_op, base, indices[3], effect_input, 164 control_input); 165 rep_nodes[2] = graph()->NewNode(load_op, base, indices[2], rep_nodes[3], 166 control_input); 167 rep_nodes[1] = graph()->NewNode(load_op, base, indices[1], rep_nodes[2], 168 control_input); 169 rep_nodes[0]->ReplaceInput(2, rep_nodes[1]); 170 } else { 171 for (size_t i = 1; i < kMaxLanes; ++i) { 172 rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]); 173 } 174 } 175 ReplaceNode(node, rep_nodes); 176 } else { 177 DefaultLowering(node); 178 } 179 } 180 181 void SimdScalarLowering::LowerStoreOp(MachineRepresentation rep, Node* node, 182 const Operator* store_op, 183 SimdType rep_type) { 184 if (rep == MachineRepresentation::kSimd128) { 185 Node* base = node->InputAt(0); 186 Node* index = node->InputAt(1); 187 Node* indices[kMaxLanes]; 188 GetIndexNodes(index, indices); 189 DCHECK(node->InputCount() > 2); 190 Node* value = node->InputAt(2); 191 DCHECK(HasReplacement(1, value)); 192 Node* rep_nodes[kMaxLanes]; 193 rep_nodes[0] = node; 194 Node** rep_inputs = GetReplacementsWithType(value, rep_type); 195 rep_nodes[0]->ReplaceInput(2, rep_inputs[0]); 196 NodeProperties::ChangeOp(node, store_op); 197 if (node->InputCount() > 3) { 198 DCHECK(node->InputCount() > 4); 199 Node* effect_input = node->InputAt(3); 200 Node* control_input = node->InputAt(4); 201 rep_nodes[3] = graph()->NewNode(store_op, base, indices[3], rep_inputs[3], 202 effect_input, control_input); 203 rep_nodes[2] = graph()->NewNode(store_op, base, indices[2], rep_inputs[2], 204 rep_nodes[3], control_input); 205 rep_nodes[1] = graph()->NewNode(store_op, base, indices[1], rep_inputs[1], 206 rep_nodes[2], control_input); 207 rep_nodes[0]->ReplaceInput(3, rep_nodes[1]); 208 209 } else { 210 for (size_t i = 1; i < kMaxLanes; ++i) { 211 rep_nodes[i] = 212 graph()->NewNode(store_op, base, indices[i], rep_inputs[i]); 213 } 214 } 215 216 ReplaceNode(node, rep_nodes); 217 } else { 218 DefaultLowering(node); 219 } 220 } 221 222 void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType rep_type, 223 const Operator* op) { 224 DCHECK(node->InputCount() == 2); 225 Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type); 226 Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type); 227 Node* rep_node[kMaxLanes]; 228 for (int i = 0; i < kMaxLanes; ++i) { 229 rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]); 230 } 231 ReplaceNode(node, rep_node); 232 } 233 234 void SimdScalarLowering::LowerNode(Node* node) { 235 SimdType rep_type = ReplacementType(node); 236 switch (node->opcode()) { 237 case IrOpcode::kStart: { 238 int parameter_count = GetParameterCountAfterLowering(); 239 // Only exchange the node if the parameter count actually changed. 240 if (parameter_count != static_cast<int>(signature()->parameter_count())) { 241 int delta = 242 parameter_count - static_cast<int>(signature()->parameter_count()); 243 int new_output_count = node->op()->ValueOutputCount() + delta; 244 NodeProperties::ChangeOp(node, common()->Start(new_output_count)); 245 } 246 break; 247 } 248 case IrOpcode::kParameter: { 249 DCHECK(node->InputCount() == 1); 250 // Only exchange the node if the parameter count actually changed. We do 251 // not even have to do the default lowering because the the start node, 252 // the only input of a parameter node, only changes if the parameter count 253 // changes. 254 if (GetParameterCountAfterLowering() != 255 static_cast<int>(signature()->parameter_count())) { 256 int old_index = ParameterIndexOf(node->op()); 257 int new_index = GetParameterIndexAfterLowering(signature(), old_index); 258 if (old_index == new_index) { 259 NodeProperties::ChangeOp(node, common()->Parameter(new_index)); 260 261 Node* new_node[kMaxLanes]; 262 for (int i = 0; i < kMaxLanes; ++i) { 263 new_node[i] = nullptr; 264 } 265 new_node[0] = node; 266 if (signature()->GetParam(old_index) == 267 MachineRepresentation::kSimd128) { 268 for (int i = 1; i < kMaxLanes; ++i) { 269 new_node[i] = graph()->NewNode(common()->Parameter(new_index + i), 270 graph()->start()); 271 } 272 } 273 ReplaceNode(node, new_node); 274 } 275 } 276 break; 277 } 278 case IrOpcode::kLoad: { 279 MachineRepresentation rep = 280 LoadRepresentationOf(node->op()).representation(); 281 const Operator* load_op; 282 if (rep_type == SimdType::kInt32) { 283 load_op = machine()->Load(MachineType::Int32()); 284 } else if (rep_type == SimdType::kFloat32) { 285 load_op = machine()->Load(MachineType::Float32()); 286 } 287 LowerLoadOp(rep, node, load_op); 288 break; 289 } 290 case IrOpcode::kUnalignedLoad: { 291 MachineRepresentation rep = 292 UnalignedLoadRepresentationOf(node->op()).representation(); 293 const Operator* load_op; 294 if (rep_type == SimdType::kInt32) { 295 load_op = machine()->UnalignedLoad(MachineType::Int32()); 296 } else if (rep_type == SimdType::kFloat32) { 297 load_op = machine()->UnalignedLoad(MachineType::Float32()); 298 } 299 LowerLoadOp(rep, node, load_op); 300 break; 301 } 302 case IrOpcode::kStore: { 303 MachineRepresentation rep = 304 StoreRepresentationOf(node->op()).representation(); 305 WriteBarrierKind write_barrier_kind = 306 StoreRepresentationOf(node->op()).write_barrier_kind(); 307 const Operator* store_op; 308 if (rep_type == SimdType::kInt32) { 309 store_op = machine()->Store(StoreRepresentation( 310 MachineRepresentation::kWord32, write_barrier_kind)); 311 } else { 312 store_op = machine()->Store(StoreRepresentation( 313 MachineRepresentation::kFloat32, write_barrier_kind)); 314 } 315 LowerStoreOp(rep, node, store_op, rep_type); 316 break; 317 } 318 case IrOpcode::kUnalignedStore: { 319 MachineRepresentation rep = UnalignedStoreRepresentationOf(node->op()); 320 const Operator* store_op; 321 if (rep_type == SimdType::kInt32) { 322 store_op = machine()->UnalignedStore(MachineRepresentation::kWord32); 323 } else { 324 store_op = machine()->UnalignedStore(MachineRepresentation::kFloat32); 325 } 326 LowerStoreOp(rep, node, store_op, rep_type); 327 break; 328 } 329 case IrOpcode::kReturn: { 330 DefaultLowering(node); 331 int new_return_count = GetReturnCountAfterLowering(signature()); 332 if (static_cast<int>(signature()->return_count()) != new_return_count) { 333 NodeProperties::ChangeOp(node, common()->Return(new_return_count)); 334 } 335 break; 336 } 337 case IrOpcode::kCall: { 338 // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor. 339 CallDescriptor* descriptor = 340 const_cast<CallDescriptor*>(CallDescriptorOf(node->op())); 341 if (DefaultLowering(node) || 342 (descriptor->ReturnCount() == 1 && 343 descriptor->GetReturnType(0) == MachineType::Simd128())) { 344 // We have to adjust the call descriptor. 345 const Operator* op = 346 common()->Call(wasm::ModuleEnv::GetI32WasmCallDescriptorForSimd( 347 zone(), descriptor)); 348 NodeProperties::ChangeOp(node, op); 349 } 350 if (descriptor->ReturnCount() == 1 && 351 descriptor->GetReturnType(0) == MachineType::Simd128()) { 352 // We access the additional return values through projections. 353 Node* rep_node[kMaxLanes]; 354 for (int i = 0; i < kMaxLanes; ++i) { 355 rep_node[i] = 356 graph()->NewNode(common()->Projection(i), node, graph()->start()); 357 } 358 ReplaceNode(node, rep_node); 359 } 360 break; 361 } 362 case IrOpcode::kPhi: { 363 MachineRepresentation rep = PhiRepresentationOf(node->op()); 364 if (rep == MachineRepresentation::kSimd128) { 365 // The replacement nodes have already been created, we only have to 366 // replace placeholder nodes. 367 Node** rep_node = GetReplacements(node); 368 for (int i = 0; i < node->op()->ValueInputCount(); ++i) { 369 Node** rep_input = 370 GetReplacementsWithType(node->InputAt(i), rep_type); 371 for (int j = 0; j < kMaxLanes; j++) { 372 rep_node[j]->ReplaceInput(i, rep_input[j]); 373 } 374 } 375 } else { 376 DefaultLowering(node); 377 } 378 break; 379 } 380 case IrOpcode::kInt32x4Add: { 381 LowerBinaryOp(node, rep_type, machine()->Int32Add()); 382 break; 383 } 384 case IrOpcode::kFloat32x4Add: { 385 LowerBinaryOp(node, rep_type, machine()->Float32Add()); 386 break; 387 } 388 case IrOpcode::kCreateInt32x4: 389 case IrOpcode::kCreateFloat32x4: { 390 Node* rep_node[kMaxLanes]; 391 for (int i = 0; i < kMaxLanes; ++i) { 392 if (HasReplacement(0, node->InputAt(i))) { 393 rep_node[i] = GetReplacements(node->InputAt(i))[0]; 394 } else { 395 rep_node[i] = node->InputAt(i); 396 } 397 } 398 ReplaceNode(node, rep_node); 399 break; 400 } 401 case IrOpcode::kInt32x4ExtractLane: 402 case IrOpcode::kFloat32x4ExtractLane: { 403 int32_t lane = OpParameter<int32_t>(node); 404 Node* rep_node[kMaxLanes] = { 405 GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr, 406 nullptr, nullptr}; 407 ReplaceNode(node, rep_node); 408 break; 409 } 410 case IrOpcode::kInt32x4ReplaceLane: 411 case IrOpcode::kFloat32x4ReplaceLane: { 412 DCHECK_EQ(2, node->InputCount()); 413 Node* repNode = node->InputAt(1); 414 int32_t lane = OpParameter<int32_t>(node); 415 DCHECK(lane >= 0 && lane <= 3); 416 Node** rep_node = GetReplacementsWithType(node->InputAt(0), rep_type); 417 if (HasReplacement(0, repNode)) { 418 rep_node[lane] = GetReplacements(repNode)[0]; 419 } else { 420 rep_node[lane] = repNode; 421 } 422 ReplaceNode(node, rep_node); 423 break; 424 } 425 default: { DefaultLowering(node); } 426 } 427 } 428 429 bool SimdScalarLowering::DefaultLowering(Node* node) { 430 bool something_changed = false; 431 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) { 432 Node* input = node->InputAt(i); 433 if (HasReplacement(0, input)) { 434 something_changed = true; 435 node->ReplaceInput(i, GetReplacements(input)[0]); 436 } 437 if (HasReplacement(1, input)) { 438 something_changed = true; 439 for (int j = 1; j < kMaxLanes; j++) { 440 node->InsertInput(zone(), i + j, GetReplacements(input)[j]); 441 } 442 } 443 } 444 return something_changed; 445 } 446 447 void SimdScalarLowering::ReplaceNode(Node* old, Node** new_node) { 448 // if new_low == nullptr, then also new_high == nullptr. 449 DCHECK(new_node[0] != nullptr || 450 (new_node[1] == nullptr && new_node[2] == nullptr && 451 new_node[3] == nullptr)); 452 for (int i = 0; i < kMaxLanes; ++i) { 453 replacements_[old->id()].node[i] = new_node[i]; 454 } 455 } 456 457 bool SimdScalarLowering::HasReplacement(size_t index, Node* node) { 458 return replacements_[node->id()].node[index] != nullptr; 459 } 460 461 SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) { 462 return replacements_[node->id()].type; 463 } 464 465 Node** SimdScalarLowering::GetReplacements(Node* node) { 466 Node** result = replacements_[node->id()].node; 467 DCHECK(result); 468 return result; 469 } 470 471 Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) { 472 Node** replacements = GetReplacements(node); 473 if (ReplacementType(node) == type) { 474 return GetReplacements(node); 475 } 476 Node** result = zone()->NewArray<Node*>(kMaxLanes); 477 if (ReplacementType(node) == SimdType::kInt32 && type == SimdType::kFloat32) { 478 for (int i = 0; i < kMaxLanes; ++i) { 479 if (replacements[i] != nullptr) { 480 result[i] = graph()->NewNode(machine()->BitcastInt32ToFloat32(), 481 replacements[i]); 482 } else { 483 result[i] = nullptr; 484 } 485 } 486 } else { 487 for (int i = 0; i < kMaxLanes; ++i) { 488 if (replacements[i] != nullptr) { 489 result[i] = graph()->NewNode(machine()->BitcastFloat32ToInt32(), 490 replacements[i]); 491 } else { 492 result[i] = nullptr; 493 } 494 } 495 } 496 return result; 497 } 498 499 void SimdScalarLowering::PreparePhiReplacement(Node* phi) { 500 MachineRepresentation rep = PhiRepresentationOf(phi->op()); 501 if (rep == MachineRepresentation::kSimd128) { 502 // We have to create the replacements for a phi node before we actually 503 // lower the phi to break potential cycles in the graph. The replacements of 504 // input nodes do not exist yet, so we use a placeholder node to pass the 505 // graph verifier. 506 int value_count = phi->op()->ValueInputCount(); 507 SimdType type = ReplacementType(phi); 508 Node** inputs_rep[kMaxLanes]; 509 for (int i = 0; i < kMaxLanes; ++i) { 510 inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1); 511 inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0); 512 } 513 for (int i = 0; i < value_count; ++i) { 514 for (int j = 0; j < kMaxLanes; j++) { 515 inputs_rep[j][i] = placeholder_; 516 } 517 } 518 Node* rep_nodes[kMaxLanes]; 519 for (int i = 0; i < kMaxLanes; ++i) { 520 if (type == SimdType::kInt32) { 521 rep_nodes[i] = graph()->NewNode( 522 common()->Phi(MachineRepresentation::kWord32, value_count), 523 value_count + 1, inputs_rep[i], false); 524 } else if (type == SimdType::kFloat32) { 525 rep_nodes[i] = graph()->NewNode( 526 common()->Phi(MachineRepresentation::kFloat32, value_count), 527 value_count + 1, inputs_rep[i], false); 528 } else { 529 UNREACHABLE(); 530 } 531 } 532 ReplaceNode(phi, rep_nodes); 533 } 534 } 535 } // namespace compiler 536 } // namespace internal 537 } // namespace v8 538
