1 // Copyright 2014 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/int64-lowering.h" 6 7 #include "src/compiler/common-operator.h" 8 #include "src/compiler/diamond.h" 9 #include "src/compiler/graph.h" 10 #include "src/compiler/linkage.h" 11 #include "src/compiler/machine-operator.h" 12 #include "src/compiler/node-matchers.h" 13 #include "src/compiler/node-properties.h" 14 #include "src/compiler/node.h" 15 #include "src/compiler/wasm-compiler.h" 16 // TODO(wasm): Remove this include. 17 #include "src/wasm/wasm-linkage.h" 18 #include "src/zone/zone.h" 19 20 namespace v8 { 21 namespace internal { 22 namespace compiler { 23 24 Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine, 25 CommonOperatorBuilder* common, Zone* zone, 26 Signature<MachineRepresentation>* signature) 27 : zone_(zone), 28 graph_(graph), 29 machine_(machine), 30 common_(common), 31 state_(graph, 3), 32 stack_(zone), 33 replacements_(nullptr), 34 signature_(signature), 35 placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"), 36 graph->start())) { 37 DCHECK_NOT_NULL(graph); 38 DCHECK_NOT_NULL(graph->end()); 39 replacements_ = zone->NewArray<Replacement>(graph->NodeCount()); 40 memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount()); 41 } 42 43 void Int64Lowering::LowerGraph() { 44 if (!machine()->Is32()) { 45 return; 46 } 47 stack_.push_back({graph()->end(), 0}); 48 state_.Set(graph()->end(), State::kOnStack); 49 50 while (!stack_.empty()) { 51 NodeState& top = stack_.back(); 52 if (top.input_index == top.node->InputCount()) { 53 // All inputs of top have already been lowered, now lower top. 54 stack_.pop_back(); 55 state_.Set(top.node, State::kVisited); 56 LowerNode(top.node); 57 } else { 58 // Push the next input onto the stack. 59 Node* input = top.node->InputAt(top.input_index++); 60 if (state_.Get(input) == State::kUnvisited) { 61 if (input->opcode() == IrOpcode::kPhi) { 62 // To break cycles with phi nodes we push phis on a separate stack so 63 // that they are processed after all other nodes. 64 PreparePhiReplacement(input); 65 stack_.push_front({input, 0}); 66 } else if (input->opcode() == IrOpcode::kEffectPhi || 67 input->opcode() == IrOpcode::kLoop) { 68 stack_.push_front({input, 0}); 69 } else { 70 stack_.push_back({input, 0}); 71 } 72 state_.Set(input, State::kOnStack); 73 } 74 } 75 } 76 } 77 78 namespace { 79 80 int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor, 81 int old_index) { 82 int result = old_index; 83 for (int i = 0; i < old_index; i++) { 84 if (call_descriptor->GetReturnType(i).representation() == 85 MachineRepresentation::kWord64) { 86 result++; 87 } 88 } 89 return result; 90 } 91 92 int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) { 93 return GetReturnIndexAfterLowering( 94 call_descriptor, static_cast<int>(call_descriptor->ReturnCount())); 95 } 96 97 int GetParameterIndexAfterLowering( 98 Signature<MachineRepresentation>* signature, int old_index) { 99 int result = old_index; 100 for (int i = 0; i < old_index; i++) { 101 if (signature->GetParam(i) == MachineRepresentation::kWord64) { 102 result++; 103 } 104 } 105 return result; 106 } 107 108 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) { 109 int result = static_cast<int>(signature->return_count()); 110 for (int i = 0; i < static_cast<int>(signature->return_count()); i++) { 111 if (signature->GetReturn(i) == MachineRepresentation::kWord64) { 112 result++; 113 } 114 } 115 return result; 116 } 117 118 } // namespace 119 120 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) { 121 DCHECK_EQ(5, node->InputCount()); 122 Node* value = node->InputAt(2); 123 node->ReplaceInput(2, GetReplacementLow(value)); 124 node->InsertInput(zone(), 3, GetReplacementHigh(value)); 125 NodeProperties::ChangeOp(node, op); 126 ReplaceNodeWithProjections(node); 127 } 128 129 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) { 130 DefaultLowering(node, true); 131 NodeProperties::ChangeOp(node, op); 132 ReplaceNodeWithProjections(node); 133 } 134 135 // static 136 int Int64Lowering::GetParameterCountAfterLowering( 137 Signature<MachineRepresentation>* signature) { 138 // GetParameterIndexAfterLowering(parameter_count) returns the parameter count 139 // after lowering. 140 return GetParameterIndexAfterLowering( 141 signature, static_cast<int>(signature->parameter_count())); 142 } 143 144 void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low, 145 Node*& index_high) { 146 if (HasReplacementLow(index)) { 147 index = GetReplacementLow(index); 148 } 149 #if defined(V8_TARGET_LITTLE_ENDIAN) 150 index_low = index; 151 index_high = graph()->NewNode(machine()->Int32Add(), index, 152 graph()->NewNode(common()->Int32Constant(4))); 153 #elif defined(V8_TARGET_BIG_ENDIAN) 154 index_low = graph()->NewNode(machine()->Int32Add(), index, 155 graph()->NewNode(common()->Int32Constant(4))); 156 index_high = index; 157 #endif 158 } 159 160 void Int64Lowering::LowerNode(Node* node) { 161 switch (node->opcode()) { 162 case IrOpcode::kInt64Constant: { 163 int64_t value = OpParameter<int64_t>(node->op()); 164 Node* low_node = graph()->NewNode( 165 common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF))); 166 Node* high_node = graph()->NewNode( 167 common()->Int32Constant(static_cast<int32_t>(value >> 32))); 168 ReplaceNode(node, low_node, high_node); 169 break; 170 } 171 case IrOpcode::kLoad: 172 case IrOpcode::kUnalignedLoad: { 173 MachineRepresentation rep; 174 if (node->opcode() == IrOpcode::kLoad) { 175 rep = LoadRepresentationOf(node->op()).representation(); 176 } else { 177 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode()); 178 rep = LoadRepresentationOf(node->op()).representation(); 179 } 180 181 if (rep == MachineRepresentation::kWord64) { 182 Node* base = node->InputAt(0); 183 Node* index = node->InputAt(1); 184 Node* index_low; 185 Node* index_high; 186 GetIndexNodes(index, index_low, index_high); 187 const Operator* load_op; 188 189 if (node->opcode() == IrOpcode::kLoad) { 190 load_op = machine()->Load(MachineType::Int32()); 191 } else { 192 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode()); 193 load_op = machine()->UnalignedLoad(MachineType::Int32()); 194 } 195 196 Node* high_node; 197 if (node->InputCount() > 2) { 198 Node* effect_high = node->InputAt(2); 199 Node* control_high = node->InputAt(3); 200 high_node = graph()->NewNode(load_op, base, index_high, effect_high, 201 control_high); 202 // change the effect change from old_node --> old_effect to 203 // old_node --> high_node --> old_effect. 204 node->ReplaceInput(2, high_node); 205 } else { 206 high_node = graph()->NewNode(load_op, base, index_high); 207 } 208 node->ReplaceInput(1, index_low); 209 NodeProperties::ChangeOp(node, load_op); 210 ReplaceNode(node, node, high_node); 211 } else { 212 DefaultLowering(node); 213 } 214 break; 215 } 216 case IrOpcode::kStore: 217 case IrOpcode::kUnalignedStore: { 218 MachineRepresentation rep; 219 if (node->opcode() == IrOpcode::kStore) { 220 rep = StoreRepresentationOf(node->op()).representation(); 221 } else { 222 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode()); 223 rep = UnalignedStoreRepresentationOf(node->op()); 224 } 225 226 if (rep == MachineRepresentation::kWord64) { 227 // We change the original store node to store the low word, and create 228 // a new store node to store the high word. The effect and control edges 229 // are copied from the original store to the new store node, the effect 230 // edge of the original store is redirected to the new store. 231 Node* base = node->InputAt(0); 232 Node* index = node->InputAt(1); 233 Node* index_low; 234 Node* index_high; 235 GetIndexNodes(index, index_low, index_high); 236 Node* value = node->InputAt(2); 237 DCHECK(HasReplacementLow(value)); 238 DCHECK(HasReplacementHigh(value)); 239 240 const Operator* store_op; 241 if (node->opcode() == IrOpcode::kStore) { 242 WriteBarrierKind write_barrier_kind = 243 StoreRepresentationOf(node->op()).write_barrier_kind(); 244 store_op = machine()->Store(StoreRepresentation( 245 MachineRepresentation::kWord32, write_barrier_kind)); 246 } else { 247 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode()); 248 store_op = machine()->UnalignedStore(MachineRepresentation::kWord32); 249 } 250 251 Node* high_node; 252 if (node->InputCount() > 3) { 253 Node* effect_high = node->InputAt(3); 254 Node* control_high = node->InputAt(4); 255 high_node = graph()->NewNode(store_op, base, index_high, 256 GetReplacementHigh(value), effect_high, 257 control_high); 258 node->ReplaceInput(3, high_node); 259 260 } else { 261 high_node = graph()->NewNode(store_op, base, index_high, 262 GetReplacementHigh(value)); 263 } 264 265 node->ReplaceInput(1, index_low); 266 node->ReplaceInput(2, GetReplacementLow(value)); 267 NodeProperties::ChangeOp(node, store_op); 268 ReplaceNode(node, node, high_node); 269 } else { 270 DefaultLowering(node, true); 271 } 272 break; 273 } 274 case IrOpcode::kStart: { 275 int parameter_count = GetParameterCountAfterLowering(signature()); 276 // Only exchange the node if the parameter count actually changed. 277 if (parameter_count != static_cast<int>(signature()->parameter_count())) { 278 int delta = 279 parameter_count - static_cast<int>(signature()->parameter_count()); 280 int new_output_count = node->op()->ValueOutputCount() + delta; 281 NodeProperties::ChangeOp(node, common()->Start(new_output_count)); 282 } 283 break; 284 } 285 case IrOpcode::kParameter: { 286 DCHECK_EQ(1, node->InputCount()); 287 // Only exchange the node if the parameter count actually changed. We do 288 // not even have to do the default lowering because the the start node, 289 // the only input of a parameter node, only changes if the parameter count 290 // changes. 291 if (GetParameterCountAfterLowering(signature()) != 292 static_cast<int>(signature()->parameter_count())) { 293 int old_index = ParameterIndexOf(node->op()); 294 // TODO(wasm): Make this part not wasm specific. 295 // Prevent special lowering of the instance parameter. 296 if (old_index == wasm::kWasmInstanceParameterIndex) { 297 DefaultLowering(node); 298 break; 299 } 300 // Adjust old_index to be compliant with the signature. 301 --old_index; 302 int new_index = GetParameterIndexAfterLowering(signature(), old_index); 303 // Adjust new_index to consider the instance parameter. 304 ++new_index; 305 NodeProperties::ChangeOp(node, common()->Parameter(new_index)); 306 307 if (signature()->GetParam(old_index) == 308 MachineRepresentation::kWord64) { 309 Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1), 310 graph()->start()); 311 ReplaceNode(node, node, high_node); 312 } 313 } 314 break; 315 } 316 case IrOpcode::kReturn: { 317 int input_count = node->InputCount(); 318 DefaultLowering(node); 319 if (input_count != node->InputCount()) { 320 int new_return_count = GetReturnCountAfterLowering(signature()); 321 if (static_cast<int>(signature()->return_count()) != new_return_count) { 322 NodeProperties::ChangeOp(node, common()->Return(new_return_count)); 323 } 324 } 325 break; 326 } 327 case IrOpcode::kTailCall: { 328 auto call_descriptor = 329 const_cast<CallDescriptor*>(CallDescriptorOf(node->op())); 330 bool returns_require_lowering = 331 GetReturnCountAfterLowering(call_descriptor) != 332 static_cast<int>(call_descriptor->ReturnCount()); 333 if (DefaultLowering(node) || returns_require_lowering) { 334 // Tail calls do not have return values, so adjusting the call 335 // descriptor is enough. 336 auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor); 337 NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor)); 338 } 339 break; 340 } 341 case IrOpcode::kCall: { 342 auto call_descriptor = 343 const_cast<CallDescriptor*>(CallDescriptorOf(node->op())); 344 bool returns_require_lowering = 345 GetReturnCountAfterLowering(call_descriptor) != 346 static_cast<int>(call_descriptor->ReturnCount()); 347 if (DefaultLowering(node) || returns_require_lowering) { 348 // We have to adjust the call descriptor. 349 NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor( 350 zone(), call_descriptor))); 351 } 352 if (returns_require_lowering) { 353 size_t return_arity = call_descriptor->ReturnCount(); 354 if (return_arity == 1) { 355 // We access the additional return values through projections. 356 ReplaceNodeWithProjections(node); 357 } else { 358 ZoneVector<Node*> projections(return_arity, zone()); 359 NodeProperties::CollectValueProjections(node, projections.data(), 360 return_arity); 361 for (size_t old_index = 0, new_index = 0; old_index < return_arity; 362 ++old_index, ++new_index) { 363 Node* use_node = projections[old_index]; 364 DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index); 365 DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor, 366 static_cast<int>(old_index)), 367 static_cast<int>(new_index)); 368 if (new_index != old_index) { 369 NodeProperties::ChangeOp( 370 use_node, common()->Projection(new_index)); 371 } 372 if (call_descriptor->GetReturnType(old_index).representation() == 373 MachineRepresentation::kWord64) { 374 Node* high_node = graph()->NewNode( 375 common()->Projection(new_index + 1), node, 376 graph()->start()); 377 ReplaceNode(use_node, use_node, high_node); 378 ++new_index; 379 } 380 } 381 } 382 } 383 break; 384 } 385 case IrOpcode::kWord64And: { 386 DCHECK_EQ(2, node->InputCount()); 387 Node* left = node->InputAt(0); 388 Node* right = node->InputAt(1); 389 390 Node* low_node = 391 graph()->NewNode(machine()->Word32And(), GetReplacementLow(left), 392 GetReplacementLow(right)); 393 Node* high_node = 394 graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left), 395 GetReplacementHigh(right)); 396 ReplaceNode(node, low_node, high_node); 397 break; 398 } 399 case IrOpcode::kTruncateInt64ToInt32: { 400 DCHECK_EQ(1, node->InputCount()); 401 Node* input = node->InputAt(0); 402 ReplaceNode(node, GetReplacementLow(input), nullptr); 403 node->NullAllInputs(); 404 break; 405 } 406 case IrOpcode::kInt64Add: { 407 DCHECK_EQ(2, node->InputCount()); 408 409 Node* right = node->InputAt(1); 410 node->ReplaceInput(1, GetReplacementLow(right)); 411 node->AppendInput(zone(), GetReplacementHigh(right)); 412 413 Node* left = node->InputAt(0); 414 node->ReplaceInput(0, GetReplacementLow(left)); 415 node->InsertInput(zone(), 1, GetReplacementHigh(left)); 416 417 NodeProperties::ChangeOp(node, machine()->Int32PairAdd()); 418 // We access the additional return values through projections. 419 ReplaceNodeWithProjections(node); 420 break; 421 } 422 case IrOpcode::kInt64Sub: { 423 DCHECK_EQ(2, node->InputCount()); 424 425 Node* right = node->InputAt(1); 426 node->ReplaceInput(1, GetReplacementLow(right)); 427 node->AppendInput(zone(), GetReplacementHigh(right)); 428 429 Node* left = node->InputAt(0); 430 node->ReplaceInput(0, GetReplacementLow(left)); 431 node->InsertInput(zone(), 1, GetReplacementHigh(left)); 432 433 NodeProperties::ChangeOp(node, machine()->Int32PairSub()); 434 // We access the additional return values through projections. 435 ReplaceNodeWithProjections(node); 436 break; 437 } 438 case IrOpcode::kInt64Mul: { 439 DCHECK_EQ(2, node->InputCount()); 440 441 Node* right = node->InputAt(1); 442 node->ReplaceInput(1, GetReplacementLow(right)); 443 node->AppendInput(zone(), GetReplacementHigh(right)); 444 445 Node* left = node->InputAt(0); 446 node->ReplaceInput(0, GetReplacementLow(left)); 447 node->InsertInput(zone(), 1, GetReplacementHigh(left)); 448 449 NodeProperties::ChangeOp(node, machine()->Int32PairMul()); 450 // We access the additional return values through projections. 451 ReplaceNodeWithProjections(node); 452 break; 453 } 454 case IrOpcode::kWord64Or: { 455 DCHECK_EQ(2, node->InputCount()); 456 Node* left = node->InputAt(0); 457 Node* right = node->InputAt(1); 458 459 Node* low_node = 460 graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left), 461 GetReplacementLow(right)); 462 Node* high_node = 463 graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left), 464 GetReplacementHigh(right)); 465 ReplaceNode(node, low_node, high_node); 466 break; 467 } 468 case IrOpcode::kWord64Xor: { 469 DCHECK_EQ(2, node->InputCount()); 470 Node* left = node->InputAt(0); 471 Node* right = node->InputAt(1); 472 473 Node* low_node = 474 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left), 475 GetReplacementLow(right)); 476 Node* high_node = 477 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left), 478 GetReplacementHigh(right)); 479 ReplaceNode(node, low_node, high_node); 480 break; 481 } 482 case IrOpcode::kWord64Shl: { 483 // TODO(turbofan): if the shift count >= 32, then we can set the low word 484 // of the output to 0 and just calculate the high word. 485 DCHECK_EQ(2, node->InputCount()); 486 Node* shift = node->InputAt(1); 487 if (HasReplacementLow(shift)) { 488 // We do not have to care about the high word replacement, because 489 // the shift can only be between 0 and 63 anyways. 490 node->ReplaceInput(1, GetReplacementLow(shift)); 491 } 492 493 Node* value = node->InputAt(0); 494 node->ReplaceInput(0, GetReplacementLow(value)); 495 node->InsertInput(zone(), 1, GetReplacementHigh(value)); 496 497 NodeProperties::ChangeOp(node, machine()->Word32PairShl()); 498 // We access the additional return values through projections. 499 ReplaceNodeWithProjections(node); 500 break; 501 } 502 case IrOpcode::kWord64Shr: { 503 // TODO(turbofan): if the shift count >= 32, then we can set the low word 504 // of the output to 0 and just calculate the high word. 505 DCHECK_EQ(2, node->InputCount()); 506 Node* shift = node->InputAt(1); 507 if (HasReplacementLow(shift)) { 508 // We do not have to care about the high word replacement, because 509 // the shift can only be between 0 and 63 anyways. 510 node->ReplaceInput(1, GetReplacementLow(shift)); 511 } 512 513 Node* value = node->InputAt(0); 514 node->ReplaceInput(0, GetReplacementLow(value)); 515 node->InsertInput(zone(), 1, GetReplacementHigh(value)); 516 517 NodeProperties::ChangeOp(node, machine()->Word32PairShr()); 518 // We access the additional return values through projections. 519 ReplaceNodeWithProjections(node); 520 break; 521 } 522 case IrOpcode::kWord64Sar: { 523 // TODO(turbofan): if the shift count >= 32, then we can set the low word 524 // of the output to 0 and just calculate the high word. 525 DCHECK_EQ(2, node->InputCount()); 526 Node* shift = node->InputAt(1); 527 if (HasReplacementLow(shift)) { 528 // We do not have to care about the high word replacement, because 529 // the shift can only be between 0 and 63 anyways. 530 node->ReplaceInput(1, GetReplacementLow(shift)); 531 } 532 533 Node* value = node->InputAt(0); 534 node->ReplaceInput(0, GetReplacementLow(value)); 535 node->InsertInput(zone(), 1, GetReplacementHigh(value)); 536 537 NodeProperties::ChangeOp(node, machine()->Word32PairSar()); 538 // We access the additional return values through projections. 539 ReplaceNodeWithProjections(node); 540 break; 541 } 542 case IrOpcode::kWord64Equal: { 543 DCHECK_EQ(2, node->InputCount()); 544 Node* left = node->InputAt(0); 545 Node* right = node->InputAt(1); 546 547 // TODO(wasm): Use explicit comparisons and && here? 548 Node* replacement = graph()->NewNode( 549 machine()->Word32Equal(), 550 graph()->NewNode( 551 machine()->Word32Or(), 552 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left), 553 GetReplacementLow(right)), 554 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left), 555 GetReplacementHigh(right))), 556 graph()->NewNode(common()->Int32Constant(0))); 557 558 ReplaceNode(node, replacement, nullptr); 559 break; 560 } 561 case IrOpcode::kInt64LessThan: { 562 LowerComparison(node, machine()->Int32LessThan(), 563 machine()->Uint32LessThan()); 564 break; 565 } 566 case IrOpcode::kInt64LessThanOrEqual: { 567 LowerComparison(node, machine()->Int32LessThan(), 568 machine()->Uint32LessThanOrEqual()); 569 break; 570 } 571 case IrOpcode::kUint64LessThan: { 572 LowerComparison(node, machine()->Uint32LessThan(), 573 machine()->Uint32LessThan()); 574 break; 575 } 576 case IrOpcode::kUint64LessThanOrEqual: { 577 LowerComparison(node, machine()->Uint32LessThan(), 578 machine()->Uint32LessThanOrEqual()); 579 break; 580 } 581 case IrOpcode::kSignExtendWord32ToInt64: 582 case IrOpcode::kChangeInt32ToInt64: { 583 DCHECK_EQ(1, node->InputCount()); 584 Node* input = node->InputAt(0); 585 if (HasReplacementLow(input)) { 586 input = GetReplacementLow(input); 587 } 588 // We use SAR to preserve the sign in the high word. 589 ReplaceNode( 590 node, input, 591 graph()->NewNode(machine()->Word32Sar(), input, 592 graph()->NewNode(common()->Int32Constant(31)))); 593 node->NullAllInputs(); 594 break; 595 } 596 case IrOpcode::kChangeUint32ToUint64: { 597 DCHECK_EQ(1, node->InputCount()); 598 Node* input = node->InputAt(0); 599 if (HasReplacementLow(input)) { 600 input = GetReplacementLow(input); 601 } 602 ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0))); 603 node->NullAllInputs(); 604 break; 605 } 606 case IrOpcode::kBitcastInt64ToFloat64: { 607 DCHECK_EQ(1, node->InputCount()); 608 Node* input = node->InputAt(0); 609 Node* stack_slot = graph()->NewNode( 610 machine()->StackSlot(MachineRepresentation::kWord64)); 611 612 Node* store_high_word = graph()->NewNode( 613 machine()->Store( 614 StoreRepresentation(MachineRepresentation::kWord32, 615 WriteBarrierKind::kNoWriteBarrier)), 616 stack_slot, 617 graph()->NewNode( 618 common()->Int32Constant(kInt64UpperHalfMemoryOffset)), 619 GetReplacementHigh(input), graph()->start(), graph()->start()); 620 621 Node* store_low_word = graph()->NewNode( 622 machine()->Store( 623 StoreRepresentation(MachineRepresentation::kWord32, 624 WriteBarrierKind::kNoWriteBarrier)), 625 stack_slot, 626 graph()->NewNode( 627 common()->Int32Constant(kInt64LowerHalfMemoryOffset)), 628 GetReplacementLow(input), store_high_word, graph()->start()); 629 630 Node* load = 631 graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot, 632 graph()->NewNode(common()->Int32Constant(0)), 633 store_low_word, graph()->start()); 634 635 ReplaceNode(node, load, nullptr); 636 break; 637 } 638 case IrOpcode::kBitcastFloat64ToInt64: { 639 DCHECK_EQ(1, node->InputCount()); 640 Node* input = node->InputAt(0); 641 if (HasReplacementLow(input)) { 642 input = GetReplacementLow(input); 643 } 644 Node* stack_slot = graph()->NewNode( 645 machine()->StackSlot(MachineRepresentation::kWord64)); 646 Node* store = graph()->NewNode( 647 machine()->Store( 648 StoreRepresentation(MachineRepresentation::kFloat64, 649 WriteBarrierKind::kNoWriteBarrier)), 650 stack_slot, graph()->NewNode(common()->Int32Constant(0)), input, 651 graph()->start(), graph()->start()); 652 653 Node* high_node = graph()->NewNode( 654 machine()->Load(MachineType::Int32()), stack_slot, 655 graph()->NewNode( 656 common()->Int32Constant(kInt64UpperHalfMemoryOffset)), 657 store, graph()->start()); 658 659 Node* low_node = graph()->NewNode( 660 machine()->Load(MachineType::Int32()), stack_slot, 661 graph()->NewNode( 662 common()->Int32Constant(kInt64LowerHalfMemoryOffset)), 663 store, graph()->start()); 664 ReplaceNode(node, low_node, high_node); 665 break; 666 } 667 case IrOpcode::kWord64Ror: { 668 DCHECK_EQ(2, node->InputCount()); 669 Node* input = node->InputAt(0); 670 Node* shift = HasReplacementLow(node->InputAt(1)) 671 ? GetReplacementLow(node->InputAt(1)) 672 : node->InputAt(1); 673 Int32Matcher m(shift); 674 if (m.HasValue()) { 675 // Precondition: 0 <= shift < 64. 676 int32_t shift_value = m.Value() & 0x3F; 677 if (shift_value == 0) { 678 ReplaceNode(node, GetReplacementLow(input), 679 GetReplacementHigh(input)); 680 } else if (shift_value == 32) { 681 ReplaceNode(node, GetReplacementHigh(input), 682 GetReplacementLow(input)); 683 } else { 684 Node* low_input; 685 Node* high_input; 686 if (shift_value < 32) { 687 low_input = GetReplacementLow(input); 688 high_input = GetReplacementHigh(input); 689 } else { 690 low_input = GetReplacementHigh(input); 691 high_input = GetReplacementLow(input); 692 } 693 int32_t masked_shift_value = shift_value & 0x1F; 694 Node* masked_shift = 695 graph()->NewNode(common()->Int32Constant(masked_shift_value)); 696 Node* inv_shift = graph()->NewNode( 697 common()->Int32Constant(32 - masked_shift_value)); 698 699 Node* low_node = graph()->NewNode( 700 machine()->Word32Or(), 701 graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift), 702 graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift)); 703 Node* high_node = graph()->NewNode( 704 machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(), 705 high_input, masked_shift), 706 graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift)); 707 ReplaceNode(node, low_node, high_node); 708 } 709 } else { 710 Node* safe_shift = shift; 711 if (!machine()->Word32ShiftIsSafe()) { 712 safe_shift = 713 graph()->NewNode(machine()->Word32And(), shift, 714 graph()->NewNode(common()->Int32Constant(0x1F))); 715 } 716 717 // By creating this bit-mask with SAR and SHL we do not have to deal 718 // with shift == 0 as a special case. 719 Node* inv_mask = graph()->NewNode( 720 machine()->Word32Shl(), 721 graph()->NewNode(machine()->Word32Sar(), 722 graph()->NewNode(common()->Int32Constant( 723 std::numeric_limits<int32_t>::min())), 724 safe_shift), 725 graph()->NewNode(common()->Int32Constant(1))); 726 727 Node* bit_mask = 728 graph()->NewNode(machine()->Word32Xor(), inv_mask, 729 graph()->NewNode(common()->Int32Constant(-1))); 730 731 // We have to mask the shift value for this comparison. If 732 // !machine()->Word32ShiftIsSafe() then the masking should already be 733 // part of the graph. 734 Node* masked_shift6 = shift; 735 if (machine()->Word32ShiftIsSafe()) { 736 masked_shift6 = 737 graph()->NewNode(machine()->Word32And(), shift, 738 graph()->NewNode(common()->Int32Constant(0x3F))); 739 } 740 741 Diamond lt32( 742 graph(), common(), 743 graph()->NewNode(machine()->Int32LessThan(), masked_shift6, 744 graph()->NewNode(common()->Int32Constant(32)))); 745 746 // The low word and the high word can be swapped either at the input or 747 // at the output. We swap the inputs so that shift does not have to be 748 // kept for so long in a register. 749 Node* input_low = 750 lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input), 751 GetReplacementHigh(input)); 752 Node* input_high = 753 lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input), 754 GetReplacementLow(input)); 755 756 Node* rotate_low = 757 graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift); 758 Node* rotate_high = 759 graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift); 760 761 Node* low_node = graph()->NewNode( 762 machine()->Word32Or(), 763 graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask), 764 graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask)); 765 766 Node* high_node = graph()->NewNode( 767 machine()->Word32Or(), 768 graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask), 769 graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask)); 770 771 ReplaceNode(node, low_node, high_node); 772 } 773 break; 774 } 775 case IrOpcode::kWord64Clz: { 776 DCHECK_EQ(1, node->InputCount()); 777 Node* input = node->InputAt(0); 778 Diamond d( 779 graph(), common(), 780 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input), 781 graph()->NewNode(common()->Int32Constant(0)))); 782 783 Node* low_node = d.Phi( 784 MachineRepresentation::kWord32, 785 graph()->NewNode(machine()->Int32Add(), 786 graph()->NewNode(machine()->Word32Clz(), 787 GetReplacementLow(input)), 788 graph()->NewNode(common()->Int32Constant(32))), 789 graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input))); 790 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0))); 791 break; 792 } 793 case IrOpcode::kWord64Ctz: { 794 DCHECK_EQ(1, node->InputCount()); 795 DCHECK(machine()->Word32Ctz().IsSupported()); 796 Node* input = node->InputAt(0); 797 Diamond d( 798 graph(), common(), 799 graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input), 800 graph()->NewNode(common()->Int32Constant(0)))); 801 Node* low_node = 802 d.Phi(MachineRepresentation::kWord32, 803 graph()->NewNode(machine()->Int32Add(), 804 graph()->NewNode(machine()->Word32Ctz().op(), 805 GetReplacementHigh(input)), 806 graph()->NewNode(common()->Int32Constant(32))), 807 graph()->NewNode(machine()->Word32Ctz().op(), 808 GetReplacementLow(input))); 809 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0))); 810 break; 811 } 812 case IrOpcode::kWord64Popcnt: { 813 DCHECK_EQ(1, node->InputCount()); 814 Node* input = node->InputAt(0); 815 // We assume that a Word64Popcnt node only has been created if 816 // Word32Popcnt is actually supported. 817 DCHECK(machine()->Word32Popcnt().IsSupported()); 818 ReplaceNode(node, graph()->NewNode( 819 machine()->Int32Add(), 820 graph()->NewNode(machine()->Word32Popcnt().op(), 821 GetReplacementLow(input)), 822 graph()->NewNode(machine()->Word32Popcnt().op(), 823 GetReplacementHigh(input))), 824 graph()->NewNode(common()->Int32Constant(0))); 825 break; 826 } 827 case IrOpcode::kPhi: { 828 MachineRepresentation rep = PhiRepresentationOf(node->op()); 829 if (rep == MachineRepresentation::kWord64) { 830 // The replacement nodes have already been created, we only have to 831 // replace placeholder nodes. 832 Node* low_node = GetReplacementLow(node); 833 Node* high_node = GetReplacementHigh(node); 834 for (int i = 0; i < node->op()->ValueInputCount(); i++) { 835 low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i))); 836 high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i))); 837 } 838 } else { 839 DefaultLowering(node); 840 } 841 break; 842 } 843 case IrOpcode::kWord64ReverseBytes: { 844 Node* input = node->InputAt(0); 845 ReplaceNode(node, 846 graph()->NewNode(machine()->Word32ReverseBytes(), 847 GetReplacementHigh(input)), 848 graph()->NewNode(machine()->Word32ReverseBytes(), 849 GetReplacementLow(input))); 850 break; 851 } 852 case IrOpcode::kSignExtendWord8ToInt64: { 853 DCHECK_EQ(1, node->InputCount()); 854 Node* input = node->InputAt(0); 855 if (HasReplacementLow(input)) { 856 input = GetReplacementLow(input); 857 } 858 // Sign extend low node to Int32 859 input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input); 860 861 // We use SAR to preserve the sign in the high word. 862 ReplaceNode( 863 node, input, 864 graph()->NewNode(machine()->Word32Sar(), input, 865 graph()->NewNode(common()->Int32Constant(31)))); 866 node->NullAllInputs(); 867 break; 868 } 869 case IrOpcode::kSignExtendWord16ToInt64: { 870 DCHECK_EQ(1, node->InputCount()); 871 Node* input = node->InputAt(0); 872 if (HasReplacementLow(input)) { 873 input = GetReplacementLow(input); 874 } 875 // Sign extend low node to Int32 876 input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input); 877 878 // We use SAR to preserve the sign in the high word. 879 ReplaceNode( 880 node, input, 881 graph()->NewNode(machine()->Word32Sar(), input, 882 graph()->NewNode(common()->Int32Constant(31)))); 883 node->NullAllInputs(); 884 break; 885 } 886 case IrOpcode::kWord64AtomicLoad: { 887 DCHECK_EQ(4, node->InputCount()); 888 MachineType type = AtomicOpType(node->op()); 889 if (type == MachineType::Uint64()) { 890 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad()); 891 ReplaceNodeWithProjections(node); 892 } else { 893 NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type)); 894 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0))); 895 } 896 break; 897 } 898 case IrOpcode::kWord64AtomicStore: { 899 DCHECK_EQ(5, node->InputCount()); 900 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op()); 901 if (rep == MachineRepresentation::kWord64) { 902 Node* value = node->InputAt(2); 903 node->ReplaceInput(2, GetReplacementLow(value)); 904 node->InsertInput(zone(), 3, GetReplacementHigh(value)); 905 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore()); 906 } else { 907 DefaultLowering(node, true); 908 NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep)); 909 } 910 break; 911 } 912 #define ATOMIC_CASE(name) \ 913 case IrOpcode::kWord64Atomic##name: { \ 914 MachineType type = AtomicOpType(node->op()); \ 915 if (type == MachineType::Uint64()) { \ 916 LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \ 917 } else { \ 918 LowerWord64AtomicNarrowOp(node, \ 919 machine()->Word64AtomicNarrow##name(type)); \ 920 } \ 921 break; \ 922 } 923 ATOMIC_CASE(Add) 924 ATOMIC_CASE(Sub) 925 ATOMIC_CASE(And) 926 ATOMIC_CASE(Or) 927 ATOMIC_CASE(Xor) 928 ATOMIC_CASE(Exchange) 929 #undef ATOMIC_CASE 930 case IrOpcode::kWord64AtomicCompareExchange: { 931 MachineType type = AtomicOpType(node->op()); 932 if (type == MachineType::Uint64()) { 933 Node* old_value = node->InputAt(2); 934 Node* new_value = node->InputAt(3); 935 node->ReplaceInput(2, GetReplacementLow(old_value)); 936 node->ReplaceInput(3, GetReplacementHigh(old_value)); 937 node->InsertInput(zone(), 4, GetReplacementLow(new_value)); 938 node->InsertInput(zone(), 5, GetReplacementHigh(new_value)); 939 NodeProperties::ChangeOp(node, 940 machine()->Word32AtomicPairCompareExchange()); 941 ReplaceNodeWithProjections(node); 942 } else { 943 LowerWord64AtomicNarrowOp( 944 node, machine()->Word64AtomicNarrowCompareExchange(type)); 945 } 946 break; 947 } 948 949 default: { DefaultLowering(node); } 950 } 951 } // NOLINT(readability/fn_size) 952 953 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op, 954 const Operator* low_word_op) { 955 DCHECK_EQ(2, node->InputCount()); 956 Node* left = node->InputAt(0); 957 Node* right = node->InputAt(1); 958 Node* replacement = graph()->NewNode( 959 machine()->Word32Or(), 960 graph()->NewNode(high_word_op, GetReplacementHigh(left), 961 GetReplacementHigh(right)), 962 graph()->NewNode( 963 machine()->Word32And(), 964 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left), 965 GetReplacementHigh(right)), 966 graph()->NewNode(low_word_op, GetReplacementLow(left), 967 GetReplacementLow(right)))); 968 969 ReplaceNode(node, replacement, nullptr); 970 } 971 972 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) { 973 bool something_changed = false; 974 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) { 975 Node* input = node->InputAt(i); 976 if (HasReplacementLow(input)) { 977 something_changed = true; 978 node->ReplaceInput(i, GetReplacementLow(input)); 979 } 980 if (!low_word_only && HasReplacementHigh(input)) { 981 something_changed = true; 982 node->InsertInput(zone(), i + 1, GetReplacementHigh(input)); 983 } 984 } 985 return something_changed; 986 } 987 988 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) { 989 // if new_low == nullptr, then also new_high == nullptr. 990 DCHECK(new_low != nullptr || new_high == nullptr); 991 replacements_[old->id()].low = new_low; 992 replacements_[old->id()].high = new_high; 993 } 994 995 bool Int64Lowering::HasReplacementLow(Node* node) { 996 return replacements_[node->id()].low != nullptr; 997 } 998 999 Node* Int64Lowering::GetReplacementLow(Node* node) { 1000 Node* result = replacements_[node->id()].low; 1001 DCHECK(result); 1002 return result; 1003 } 1004 1005 bool Int64Lowering::HasReplacementHigh(Node* node) { 1006 return replacements_[node->id()].high != nullptr; 1007 } 1008 1009 Node* Int64Lowering::GetReplacementHigh(Node* node) { 1010 Node* result = replacements_[node->id()].high; 1011 DCHECK(result); 1012 return result; 1013 } 1014 1015 void Int64Lowering::PreparePhiReplacement(Node* phi) { 1016 MachineRepresentation rep = PhiRepresentationOf(phi->op()); 1017 if (rep == MachineRepresentation::kWord64) { 1018 // We have to create the replacements for a phi node before we actually 1019 // lower the phi to break potential cycles in the graph. The replacements of 1020 // input nodes do not exist yet, so we use a placeholder node to pass the 1021 // graph verifier. 1022 int value_count = phi->op()->ValueInputCount(); 1023 Node** inputs_low = zone()->NewArray<Node*>(value_count + 1); 1024 Node** inputs_high = zone()->NewArray<Node*>(value_count + 1); 1025 for (int i = 0; i < value_count; i++) { 1026 inputs_low[i] = placeholder_; 1027 inputs_high[i] = placeholder_; 1028 } 1029 inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0); 1030 inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0); 1031 ReplaceNode(phi, 1032 graph()->NewNode( 1033 common()->Phi(MachineRepresentation::kWord32, value_count), 1034 value_count + 1, inputs_low, false), 1035 graph()->NewNode( 1036 common()->Phi(MachineRepresentation::kWord32, value_count), 1037 value_count + 1, inputs_high, false)); 1038 } 1039 } 1040 1041 void Int64Lowering::ReplaceNodeWithProjections(Node* node) { 1042 DCHECK(node != nullptr); 1043 Node* low_node = 1044 graph()->NewNode(common()->Projection(0), node, graph()->start()); 1045 Node* high_node = 1046 graph()->NewNode(common()->Projection(1), node, graph()->start()); 1047 ReplaceNode(node, low_node, high_node); 1048 } 1049 1050 } // namespace compiler 1051 } // namespace internal 1052 } // namespace v8 1053
