1 // Copyright 2015 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/compiler/wasm-compiler.h" 6 7 #include <memory> 8 9 #include "src/assembler-inl.h" 10 #include "src/assembler.h" 11 #include "src/base/optional.h" 12 #include "src/base/platform/elapsed-timer.h" 13 #include "src/base/platform/platform.h" 14 #include "src/base/v8-fallthrough.h" 15 #include "src/builtins/builtins.h" 16 #include "src/code-factory.h" 17 #include "src/compiler.h" 18 #include "src/compiler/access-builder.h" 19 #include "src/compiler/code-generator.h" 20 #include "src/compiler/common-operator.h" 21 #include "src/compiler/compiler-source-position-table.h" 22 #include "src/compiler/diamond.h" 23 #include "src/compiler/graph-visualizer.h" 24 #include "src/compiler/graph.h" 25 #include "src/compiler/instruction-selector.h" 26 #include "src/compiler/int64-lowering.h" 27 #include "src/compiler/js-graph.h" 28 #include "src/compiler/js-operator.h" 29 #include "src/compiler/linkage.h" 30 #include "src/compiler/machine-operator.h" 31 #include "src/compiler/node-matchers.h" 32 #include "src/compiler/node-origin-table.h" 33 #include "src/compiler/pipeline.h" 34 #include "src/compiler/simd-scalar-lowering.h" 35 #include "src/compiler/zone-stats.h" 36 #include "src/heap/factory.h" 37 #include "src/isolate-inl.h" 38 #include "src/log-inl.h" 39 #include "src/optimized-compilation-info.h" 40 #include "src/tracing/trace-event.h" 41 #include "src/trap-handler/trap-handler.h" 42 #include "src/wasm/function-body-decoder.h" 43 #include "src/wasm/function-compiler.h" 44 #include "src/wasm/jump-table-assembler.h" 45 #include "src/wasm/memory-tracing.h" 46 #include "src/wasm/wasm-code-manager.h" 47 #include "src/wasm/wasm-limits.h" 48 #include "src/wasm/wasm-linkage.h" 49 #include "src/wasm/wasm-module.h" 50 #include "src/wasm/wasm-objects-inl.h" 51 #include "src/wasm/wasm-opcodes.h" 52 #include "src/wasm/wasm-text.h" 53 54 namespace v8 { 55 namespace internal { 56 namespace compiler { 57 58 // TODO(titzer): pull WASM_64 up to a common header. 59 #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64 60 #define WASM_64 1 61 #else 62 #define WASM_64 0 63 #endif 64 65 #define FATAL_UNSUPPORTED_OPCODE(opcode) \ 66 FATAL("Unsupported opcode 0x%x:%s", (opcode), \ 67 wasm::WasmOpcodes::OpcodeName(opcode)); 68 69 #define WASM_INSTANCE_OBJECT_OFFSET(name) \ 70 (WasmInstanceObject::k##name##Offset - kHeapObjectTag) 71 72 #define LOAD_INSTANCE_FIELD(name, type) \ 73 SetEffect(graph()->NewNode( \ 74 mcgraph()->machine()->Load(type), instance_node_.get(), \ 75 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(name)), Effect(), \ 76 Control())) 77 78 #define LOAD_FIXED_ARRAY_SLOT(array_node, index) \ 79 SetEffect(graph()->NewNode( \ 80 mcgraph()->machine()->Load(MachineType::TaggedPointer()), array_node, \ 81 mcgraph()->Int32Constant(FixedArrayOffsetMinusTag(index)), Effect(), \ 82 Control())) 83 84 int FixedArrayOffsetMinusTag(uint32_t index) { 85 auto access = AccessBuilder::ForFixedArraySlot(index); 86 return access.offset - access.tag(); 87 } 88 89 namespace { 90 91 constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2; 92 93 void MergeControlToEnd(MachineGraph* mcgraph, Node* node) { 94 Graph* g = mcgraph->graph(); 95 if (g->end()) { 96 NodeProperties::MergeControlToEnd(g, mcgraph->common(), node); 97 } else { 98 g->SetEnd(g->NewNode(mcgraph->common()->End(1), node)); 99 } 100 } 101 102 bool ContainsSimd(wasm::FunctionSig* sig) { 103 for (auto type : sig->all()) { 104 if (type == wasm::kWasmS128) return true; 105 } 106 return false; 107 } 108 109 bool ContainsInt64(wasm::FunctionSig* sig) { 110 for (auto type : sig->all()) { 111 if (type == wasm::kWasmI64) return true; 112 } 113 return false; 114 } 115 } // namespace 116 117 WasmGraphBuilder::WasmGraphBuilder( 118 wasm::ModuleEnv* env, Zone* zone, MachineGraph* mcgraph, 119 wasm::FunctionSig* sig, 120 compiler::SourcePositionTable* source_position_table) 121 : zone_(zone), 122 mcgraph_(mcgraph), 123 env_(env), 124 cur_buffer_(def_buffer_), 125 cur_bufsize_(kDefaultBufferSize), 126 has_simd_(ContainsSimd(sig)), 127 untrusted_code_mitigations_(FLAG_untrusted_code_mitigations), 128 sig_(sig), 129 source_position_table_(source_position_table) { 130 DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled()); 131 DCHECK_NOT_NULL(mcgraph_); 132 } 133 134 Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); } 135 136 Node* WasmGraphBuilder::Start(unsigned params) { 137 Node* start = graph()->NewNode(mcgraph()->common()->Start(params)); 138 graph()->SetStart(start); 139 return start; 140 } 141 142 Node* WasmGraphBuilder::Param(unsigned index) { 143 return graph()->NewNode(mcgraph()->common()->Parameter(index), 144 graph()->start()); 145 } 146 147 Node* WasmGraphBuilder::Loop(Node* entry) { 148 return graph()->NewNode(mcgraph()->common()->Loop(1), entry); 149 } 150 151 Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) { 152 Node* terminate = 153 graph()->NewNode(mcgraph()->common()->Terminate(), effect, control); 154 MergeControlToEnd(mcgraph(), terminate); 155 return terminate; 156 } 157 158 bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) { 159 return phi && IrOpcode::IsPhiOpcode(phi->opcode()) && 160 NodeProperties::GetControlInput(phi) == merge; 161 } 162 163 bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success, 164 Node** if_exception) { 165 if (node->op()->HasProperty(compiler::Operator::kNoThrow)) { 166 return false; 167 } 168 169 *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node); 170 *if_exception = 171 graph()->NewNode(mcgraph()->common()->IfException(), node, node); 172 173 return true; 174 } 175 176 void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) { 177 DCHECK(IrOpcode::IsMergeOpcode(merge->opcode())); 178 merge->AppendInput(mcgraph()->zone(), from); 179 int new_size = merge->InputCount(); 180 NodeProperties::ChangeOp( 181 merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size)); 182 } 183 184 void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) { 185 DCHECK(IrOpcode::IsPhiOpcode(phi->opcode())); 186 int new_size = phi->InputCount(); 187 phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from); 188 NodeProperties::ChangeOp( 189 phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size)); 190 } 191 192 Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) { 193 return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls); 194 } 195 196 Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals, 197 Node* control) { 198 DCHECK(IrOpcode::IsMergeOpcode(control->opcode())); 199 Node** buf = Realloc(vals, count, count + 1); 200 buf[count] = control; 201 return graph()->NewNode( 202 mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type), 203 count), 204 count + 1, buf); 205 } 206 207 Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects, 208 Node* control) { 209 DCHECK(IrOpcode::IsMergeOpcode(control->opcode())); 210 Node** buf = Realloc(effects, count, count + 1); 211 buf[count] = control; 212 return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1, 213 buf); 214 } 215 216 Node* WasmGraphBuilder::RefNull() { 217 return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer()); 218 } 219 220 Node* WasmGraphBuilder::NoContextConstant() { 221 // TODO(titzer): avoiding a dependency on JSGraph here. Refactor. 222 return mcgraph()->IntPtrConstant(0); 223 } 224 225 Node* WasmGraphBuilder::Uint32Constant(uint32_t value) { 226 return mcgraph()->Uint32Constant(value); 227 } 228 229 Node* WasmGraphBuilder::Int32Constant(int32_t value) { 230 return mcgraph()->Int32Constant(value); 231 } 232 233 Node* WasmGraphBuilder::Int64Constant(int64_t value) { 234 return mcgraph()->Int64Constant(value); 235 } 236 237 Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) { 238 return mcgraph()->IntPtrConstant(value); 239 } 240 241 void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position, 242 Node** effect, Node** control) { 243 DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks. 244 if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) { 245 return; 246 } 247 if (effect == nullptr) effect = effect_; 248 if (control == nullptr) control = control_; 249 250 // This instruction sequence is matched in the instruction selector to 251 // load the stack pointer directly on some platforms. Hence, when modifying 252 // please also fix WasmStackCheckMatcher in node-matchers.h 253 254 Node* limit_address = graph()->NewNode( 255 mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(), 256 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)), 257 *effect, *control); 258 Node* limit = graph()->NewNode( 259 mcgraph()->machine()->Load(MachineType::Pointer()), limit_address, 260 mcgraph()->IntPtrConstant(0), limit_address, *control); 261 *effect = limit; 262 Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer()); 263 264 Node* check = 265 graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer); 266 267 Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue); 268 stack_check.Chain(*control); 269 270 if (stack_check_call_operator_ == nullptr) { 271 // Build and cache the stack check call operator and the constant 272 // representing the stack check code. 273 auto call_descriptor = Linkage::GetStubCallDescriptor( 274 mcgraph()->zone(), // zone 275 NoContextDescriptor{}, // descriptor 276 0, // stack parameter count 277 CallDescriptor::kNoFlags, // flags 278 Operator::kNoProperties, // properties 279 StubCallMode::kCallWasmRuntimeStub); // stub call mode 280 // A direct call to a wasm runtime stub defined in this module. 281 // Just encode the stub index. This will be patched at relocation. 282 stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant( 283 wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL)); 284 stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor); 285 } 286 287 Node* call = graph()->NewNode(stack_check_call_operator_.get(), 288 stack_check_code_node_.get(), *effect, 289 stack_check.if_false); 290 291 SetSourcePosition(call, position); 292 293 Node* ephi = stack_check.EffectPhi(*effect, call); 294 295 *control = stack_check.merge; 296 *effect = ephi; 297 } 298 299 void WasmGraphBuilder::PatchInStackCheckIfNeeded() { 300 if (!needs_stack_check_) return; 301 302 Node* start = graph()->start(); 303 // Place a stack check which uses a dummy node as control and effect. 304 Node* dummy = graph()->NewNode(mcgraph()->common()->Dead()); 305 Node* control = dummy; 306 Node* effect = dummy; 307 // The function-prologue stack check is associated with position 0, which 308 // is never a position of any instruction in the function. 309 StackCheck(0, &effect, &control); 310 311 // In testing, no steck checks were emitted. Nothing to rewire then. 312 if (effect == dummy) return; 313 314 // Now patch all control uses of {start} to use {control} and all effect uses 315 // to use {effect} instead. Then rewire the dummy node to use start instead. 316 NodeProperties::ReplaceUses(start, start, effect, control); 317 NodeProperties::ReplaceUses(dummy, nullptr, start, start); 318 } 319 320 Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right, 321 wasm::WasmCodePosition position) { 322 const Operator* op; 323 MachineOperatorBuilder* m = mcgraph()->machine(); 324 switch (opcode) { 325 case wasm::kExprI32Add: 326 op = m->Int32Add(); 327 break; 328 case wasm::kExprI32Sub: 329 op = m->Int32Sub(); 330 break; 331 case wasm::kExprI32Mul: 332 op = m->Int32Mul(); 333 break; 334 case wasm::kExprI32DivS: 335 return BuildI32DivS(left, right, position); 336 case wasm::kExprI32DivU: 337 return BuildI32DivU(left, right, position); 338 case wasm::kExprI32RemS: 339 return BuildI32RemS(left, right, position); 340 case wasm::kExprI32RemU: 341 return BuildI32RemU(left, right, position); 342 case wasm::kExprI32And: 343 op = m->Word32And(); 344 break; 345 case wasm::kExprI32Ior: 346 op = m->Word32Or(); 347 break; 348 case wasm::kExprI32Xor: 349 op = m->Word32Xor(); 350 break; 351 case wasm::kExprI32Shl: 352 op = m->Word32Shl(); 353 right = MaskShiftCount32(right); 354 break; 355 case wasm::kExprI32ShrU: 356 op = m->Word32Shr(); 357 right = MaskShiftCount32(right); 358 break; 359 case wasm::kExprI32ShrS: 360 op = m->Word32Sar(); 361 right = MaskShiftCount32(right); 362 break; 363 case wasm::kExprI32Ror: 364 op = m->Word32Ror(); 365 right = MaskShiftCount32(right); 366 break; 367 case wasm::kExprI32Rol: 368 right = MaskShiftCount32(right); 369 return BuildI32Rol(left, right); 370 case wasm::kExprI32Eq: 371 op = m->Word32Equal(); 372 break; 373 case wasm::kExprI32Ne: 374 return Invert(Binop(wasm::kExprI32Eq, left, right)); 375 case wasm::kExprI32LtS: 376 op = m->Int32LessThan(); 377 break; 378 case wasm::kExprI32LeS: 379 op = m->Int32LessThanOrEqual(); 380 break; 381 case wasm::kExprI32LtU: 382 op = m->Uint32LessThan(); 383 break; 384 case wasm::kExprI32LeU: 385 op = m->Uint32LessThanOrEqual(); 386 break; 387 case wasm::kExprI32GtS: 388 op = m->Int32LessThan(); 389 std::swap(left, right); 390 break; 391 case wasm::kExprI32GeS: 392 op = m->Int32LessThanOrEqual(); 393 std::swap(left, right); 394 break; 395 case wasm::kExprI32GtU: 396 op = m->Uint32LessThan(); 397 std::swap(left, right); 398 break; 399 case wasm::kExprI32GeU: 400 op = m->Uint32LessThanOrEqual(); 401 std::swap(left, right); 402 break; 403 case wasm::kExprI64And: 404 op = m->Word64And(); 405 break; 406 case wasm::kExprI64Add: 407 op = m->Int64Add(); 408 break; 409 case wasm::kExprI64Sub: 410 op = m->Int64Sub(); 411 break; 412 case wasm::kExprI64Mul: 413 op = m->Int64Mul(); 414 break; 415 case wasm::kExprI64DivS: 416 return BuildI64DivS(left, right, position); 417 case wasm::kExprI64DivU: 418 return BuildI64DivU(left, right, position); 419 case wasm::kExprI64RemS: 420 return BuildI64RemS(left, right, position); 421 case wasm::kExprI64RemU: 422 return BuildI64RemU(left, right, position); 423 case wasm::kExprI64Ior: 424 op = m->Word64Or(); 425 break; 426 case wasm::kExprI64Xor: 427 op = m->Word64Xor(); 428 break; 429 case wasm::kExprI64Shl: 430 op = m->Word64Shl(); 431 right = MaskShiftCount64(right); 432 break; 433 case wasm::kExprI64ShrU: 434 op = m->Word64Shr(); 435 right = MaskShiftCount64(right); 436 break; 437 case wasm::kExprI64ShrS: 438 op = m->Word64Sar(); 439 right = MaskShiftCount64(right); 440 break; 441 case wasm::kExprI64Eq: 442 op = m->Word64Equal(); 443 break; 444 case wasm::kExprI64Ne: 445 return Invert(Binop(wasm::kExprI64Eq, left, right)); 446 case wasm::kExprI64LtS: 447 op = m->Int64LessThan(); 448 break; 449 case wasm::kExprI64LeS: 450 op = m->Int64LessThanOrEqual(); 451 break; 452 case wasm::kExprI64LtU: 453 op = m->Uint64LessThan(); 454 break; 455 case wasm::kExprI64LeU: 456 op = m->Uint64LessThanOrEqual(); 457 break; 458 case wasm::kExprI64GtS: 459 op = m->Int64LessThan(); 460 std::swap(left, right); 461 break; 462 case wasm::kExprI64GeS: 463 op = m->Int64LessThanOrEqual(); 464 std::swap(left, right); 465 break; 466 case wasm::kExprI64GtU: 467 op = m->Uint64LessThan(); 468 std::swap(left, right); 469 break; 470 case wasm::kExprI64GeU: 471 op = m->Uint64LessThanOrEqual(); 472 std::swap(left, right); 473 break; 474 case wasm::kExprI64Ror: 475 op = m->Word64Ror(); 476 right = MaskShiftCount64(right); 477 break; 478 case wasm::kExprI64Rol: 479 return BuildI64Rol(left, right); 480 case wasm::kExprF32CopySign: 481 return BuildF32CopySign(left, right); 482 case wasm::kExprF64CopySign: 483 return BuildF64CopySign(left, right); 484 case wasm::kExprF32Add: 485 op = m->Float32Add(); 486 break; 487 case wasm::kExprF32Sub: 488 op = m->Float32Sub(); 489 break; 490 case wasm::kExprF32Mul: 491 op = m->Float32Mul(); 492 break; 493 case wasm::kExprF32Div: 494 op = m->Float32Div(); 495 break; 496 case wasm::kExprF32Eq: 497 op = m->Float32Equal(); 498 break; 499 case wasm::kExprF32Ne: 500 return Invert(Binop(wasm::kExprF32Eq, left, right)); 501 case wasm::kExprF32Lt: 502 op = m->Float32LessThan(); 503 break; 504 case wasm::kExprF32Ge: 505 op = m->Float32LessThanOrEqual(); 506 std::swap(left, right); 507 break; 508 case wasm::kExprF32Gt: 509 op = m->Float32LessThan(); 510 std::swap(left, right); 511 break; 512 case wasm::kExprF32Le: 513 op = m->Float32LessThanOrEqual(); 514 break; 515 case wasm::kExprF64Add: 516 op = m->Float64Add(); 517 break; 518 case wasm::kExprF64Sub: 519 op = m->Float64Sub(); 520 break; 521 case wasm::kExprF64Mul: 522 op = m->Float64Mul(); 523 break; 524 case wasm::kExprF64Div: 525 op = m->Float64Div(); 526 break; 527 case wasm::kExprF64Eq: 528 op = m->Float64Equal(); 529 break; 530 case wasm::kExprF64Ne: 531 return Invert(Binop(wasm::kExprF64Eq, left, right)); 532 case wasm::kExprF64Lt: 533 op = m->Float64LessThan(); 534 break; 535 case wasm::kExprF64Le: 536 op = m->Float64LessThanOrEqual(); 537 break; 538 case wasm::kExprF64Gt: 539 op = m->Float64LessThan(); 540 std::swap(left, right); 541 break; 542 case wasm::kExprF64Ge: 543 op = m->Float64LessThanOrEqual(); 544 std::swap(left, right); 545 break; 546 case wasm::kExprF32Min: 547 op = m->Float32Min(); 548 break; 549 case wasm::kExprF64Min: 550 op = m->Float64Min(); 551 break; 552 case wasm::kExprF32Max: 553 op = m->Float32Max(); 554 break; 555 case wasm::kExprF64Max: 556 op = m->Float64Max(); 557 break; 558 case wasm::kExprF64Pow: 559 return BuildF64Pow(left, right); 560 case wasm::kExprF64Atan2: 561 op = m->Float64Atan2(); 562 break; 563 case wasm::kExprF64Mod: 564 return BuildF64Mod(left, right); 565 case wasm::kExprI32AsmjsDivS: 566 return BuildI32AsmjsDivS(left, right); 567 case wasm::kExprI32AsmjsDivU: 568 return BuildI32AsmjsDivU(left, right); 569 case wasm::kExprI32AsmjsRemS: 570 return BuildI32AsmjsRemS(left, right); 571 case wasm::kExprI32AsmjsRemU: 572 return BuildI32AsmjsRemU(left, right); 573 case wasm::kExprI32AsmjsStoreMem8: 574 return BuildAsmjsStoreMem(MachineType::Int8(), left, right); 575 case wasm::kExprI32AsmjsStoreMem16: 576 return BuildAsmjsStoreMem(MachineType::Int16(), left, right); 577 case wasm::kExprI32AsmjsStoreMem: 578 return BuildAsmjsStoreMem(MachineType::Int32(), left, right); 579 case wasm::kExprF32AsmjsStoreMem: 580 return BuildAsmjsStoreMem(MachineType::Float32(), left, right); 581 case wasm::kExprF64AsmjsStoreMem: 582 return BuildAsmjsStoreMem(MachineType::Float64(), left, right); 583 default: 584 FATAL_UNSUPPORTED_OPCODE(opcode); 585 } 586 return graph()->NewNode(op, left, right); 587 } 588 589 Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input, 590 wasm::WasmCodePosition position) { 591 const Operator* op; 592 MachineOperatorBuilder* m = mcgraph()->machine(); 593 switch (opcode) { 594 case wasm::kExprI32Eqz: 595 op = m->Word32Equal(); 596 return graph()->NewNode(op, input, mcgraph()->Int32Constant(0)); 597 case wasm::kExprF32Abs: 598 op = m->Float32Abs(); 599 break; 600 case wasm::kExprF32Neg: { 601 op = m->Float32Neg(); 602 break; 603 } 604 case wasm::kExprF32Sqrt: 605 op = m->Float32Sqrt(); 606 break; 607 case wasm::kExprF64Abs: 608 op = m->Float64Abs(); 609 break; 610 case wasm::kExprF64Neg: { 611 op = m->Float64Neg(); 612 break; 613 } 614 case wasm::kExprF64Sqrt: 615 op = m->Float64Sqrt(); 616 break; 617 case wasm::kExprI32SConvertF32: 618 case wasm::kExprI32UConvertF32: 619 case wasm::kExprI32SConvertF64: 620 case wasm::kExprI32UConvertF64: 621 case wasm::kExprI32SConvertSatF64: 622 case wasm::kExprI32UConvertSatF64: 623 case wasm::kExprI32SConvertSatF32: 624 case wasm::kExprI32UConvertSatF32: 625 return BuildIntConvertFloat(input, position, opcode); 626 case wasm::kExprI32AsmjsSConvertF64: 627 return BuildI32AsmjsSConvertF64(input); 628 case wasm::kExprI32AsmjsUConvertF64: 629 return BuildI32AsmjsUConvertF64(input); 630 case wasm::kExprF32ConvertF64: 631 op = m->TruncateFloat64ToFloat32(); 632 break; 633 case wasm::kExprF64SConvertI32: 634 op = m->ChangeInt32ToFloat64(); 635 break; 636 case wasm::kExprF64UConvertI32: 637 op = m->ChangeUint32ToFloat64(); 638 break; 639 case wasm::kExprF32SConvertI32: 640 op = m->RoundInt32ToFloat32(); 641 break; 642 case wasm::kExprF32UConvertI32: 643 op = m->RoundUint32ToFloat32(); 644 break; 645 case wasm::kExprI32AsmjsSConvertF32: 646 return BuildI32AsmjsSConvertF32(input); 647 case wasm::kExprI32AsmjsUConvertF32: 648 return BuildI32AsmjsUConvertF32(input); 649 case wasm::kExprF64ConvertF32: 650 op = m->ChangeFloat32ToFloat64(); 651 break; 652 case wasm::kExprF32ReinterpretI32: 653 op = m->BitcastInt32ToFloat32(); 654 break; 655 case wasm::kExprI32ReinterpretF32: 656 op = m->BitcastFloat32ToInt32(); 657 break; 658 case wasm::kExprI32Clz: 659 op = m->Word32Clz(); 660 break; 661 case wasm::kExprI32Ctz: { 662 if (m->Word32Ctz().IsSupported()) { 663 op = m->Word32Ctz().op(); 664 break; 665 } else if (m->Word32ReverseBits().IsSupported()) { 666 Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input); 667 Node* result = graph()->NewNode(m->Word32Clz(), reversed); 668 return result; 669 } else { 670 return BuildI32Ctz(input); 671 } 672 } 673 case wasm::kExprI32Popcnt: { 674 if (m->Word32Popcnt().IsSupported()) { 675 op = m->Word32Popcnt().op(); 676 break; 677 } else { 678 return BuildI32Popcnt(input); 679 } 680 } 681 case wasm::kExprF32Floor: { 682 if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input); 683 op = m->Float32RoundDown().op(); 684 break; 685 } 686 case wasm::kExprF32Ceil: { 687 if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input); 688 op = m->Float32RoundUp().op(); 689 break; 690 } 691 case wasm::kExprF32Trunc: { 692 if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input); 693 op = m->Float32RoundTruncate().op(); 694 break; 695 } 696 case wasm::kExprF32NearestInt: { 697 if (!m->Float32RoundTiesEven().IsSupported()) 698 return BuildF32NearestInt(input); 699 op = m->Float32RoundTiesEven().op(); 700 break; 701 } 702 case wasm::kExprF64Floor: { 703 if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input); 704 op = m->Float64RoundDown().op(); 705 break; 706 } 707 case wasm::kExprF64Ceil: { 708 if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input); 709 op = m->Float64RoundUp().op(); 710 break; 711 } 712 case wasm::kExprF64Trunc: { 713 if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input); 714 op = m->Float64RoundTruncate().op(); 715 break; 716 } 717 case wasm::kExprF64NearestInt: { 718 if (!m->Float64RoundTiesEven().IsSupported()) 719 return BuildF64NearestInt(input); 720 op = m->Float64RoundTiesEven().op(); 721 break; 722 } 723 case wasm::kExprF64Acos: { 724 return BuildF64Acos(input); 725 } 726 case wasm::kExprF64Asin: { 727 return BuildF64Asin(input); 728 } 729 case wasm::kExprF64Atan: 730 op = m->Float64Atan(); 731 break; 732 case wasm::kExprF64Cos: { 733 op = m->Float64Cos(); 734 break; 735 } 736 case wasm::kExprF64Sin: { 737 op = m->Float64Sin(); 738 break; 739 } 740 case wasm::kExprF64Tan: { 741 op = m->Float64Tan(); 742 break; 743 } 744 case wasm::kExprF64Exp: { 745 op = m->Float64Exp(); 746 break; 747 } 748 case wasm::kExprF64Log: 749 op = m->Float64Log(); 750 break; 751 case wasm::kExprI32ConvertI64: 752 op = m->TruncateInt64ToInt32(); 753 break; 754 case wasm::kExprI64SConvertI32: 755 op = m->ChangeInt32ToInt64(); 756 break; 757 case wasm::kExprI64UConvertI32: 758 op = m->ChangeUint32ToUint64(); 759 break; 760 case wasm::kExprF64ReinterpretI64: 761 op = m->BitcastInt64ToFloat64(); 762 break; 763 case wasm::kExprI64ReinterpretF64: 764 op = m->BitcastFloat64ToInt64(); 765 break; 766 case wasm::kExprI64Clz: 767 op = m->Word64Clz(); 768 break; 769 case wasm::kExprI64Ctz: { 770 OptionalOperator ctz64 = m->Word64Ctz(); 771 if (ctz64.IsSupported()) { 772 op = ctz64.op(); 773 break; 774 } else if (m->Is32() && m->Word32Ctz().IsSupported()) { 775 op = ctz64.placeholder(); 776 break; 777 } else if (m->Word64ReverseBits().IsSupported()) { 778 Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input); 779 Node* result = graph()->NewNode(m->Word64Clz(), reversed); 780 return result; 781 } else { 782 return BuildI64Ctz(input); 783 } 784 } 785 case wasm::kExprI64Popcnt: { 786 OptionalOperator popcnt64 = m->Word64Popcnt(); 787 if (popcnt64.IsSupported()) { 788 op = popcnt64.op(); 789 } else if (m->Is32() && m->Word32Popcnt().IsSupported()) { 790 op = popcnt64.placeholder(); 791 } else { 792 return BuildI64Popcnt(input); 793 } 794 break; 795 } 796 case wasm::kExprI64Eqz: 797 op = m->Word64Equal(); 798 return graph()->NewNode(op, input, mcgraph()->Int64Constant(0)); 799 case wasm::kExprF32SConvertI64: 800 if (m->Is32()) { 801 return BuildF32SConvertI64(input); 802 } 803 op = m->RoundInt64ToFloat32(); 804 break; 805 case wasm::kExprF32UConvertI64: 806 if (m->Is32()) { 807 return BuildF32UConvertI64(input); 808 } 809 op = m->RoundUint64ToFloat32(); 810 break; 811 case wasm::kExprF64SConvertI64: 812 if (m->Is32()) { 813 return BuildF64SConvertI64(input); 814 } 815 op = m->RoundInt64ToFloat64(); 816 break; 817 case wasm::kExprF64UConvertI64: 818 if (m->Is32()) { 819 return BuildF64UConvertI64(input); 820 } 821 op = m->RoundUint64ToFloat64(); 822 break; 823 case wasm::kExprI32SExtendI8: 824 op = m->SignExtendWord8ToInt32(); 825 break; 826 case wasm::kExprI32SExtendI16: 827 op = m->SignExtendWord16ToInt32(); 828 break; 829 case wasm::kExprI64SExtendI8: 830 op = m->SignExtendWord8ToInt64(); 831 break; 832 case wasm::kExprI64SExtendI16: 833 op = m->SignExtendWord16ToInt64(); 834 break; 835 case wasm::kExprI64SExtendI32: 836 op = m->SignExtendWord32ToInt64(); 837 break; 838 case wasm::kExprI64SConvertF32: 839 case wasm::kExprI64UConvertF32: 840 case wasm::kExprI64SConvertF64: 841 case wasm::kExprI64UConvertF64: 842 case wasm::kExprI64SConvertSatF32: 843 case wasm::kExprI64UConvertSatF32: 844 case wasm::kExprI64SConvertSatF64: 845 case wasm::kExprI64UConvertSatF64: 846 return mcgraph()->machine()->Is32() 847 ? BuildCcallConvertFloat(input, position, opcode) 848 : BuildIntConvertFloat(input, position, opcode); 849 case wasm::kExprRefIsNull: 850 return graph()->NewNode(m->WordEqual(), input, RefNull()); 851 case wasm::kExprI32AsmjsLoadMem8S: 852 return BuildAsmjsLoadMem(MachineType::Int8(), input); 853 case wasm::kExprI32AsmjsLoadMem8U: 854 return BuildAsmjsLoadMem(MachineType::Uint8(), input); 855 case wasm::kExprI32AsmjsLoadMem16S: 856 return BuildAsmjsLoadMem(MachineType::Int16(), input); 857 case wasm::kExprI32AsmjsLoadMem16U: 858 return BuildAsmjsLoadMem(MachineType::Uint16(), input); 859 case wasm::kExprI32AsmjsLoadMem: 860 return BuildAsmjsLoadMem(MachineType::Int32(), input); 861 case wasm::kExprF32AsmjsLoadMem: 862 return BuildAsmjsLoadMem(MachineType::Float32(), input); 863 case wasm::kExprF64AsmjsLoadMem: 864 return BuildAsmjsLoadMem(MachineType::Float64(), input); 865 default: 866 FATAL_UNSUPPORTED_OPCODE(opcode); 867 } 868 return graph()->NewNode(op, input); 869 } 870 871 Node* WasmGraphBuilder::Float32Constant(float value) { 872 return mcgraph()->Float32Constant(value); 873 } 874 875 Node* WasmGraphBuilder::Float64Constant(double value) { 876 return mcgraph()->Float64Constant(value); 877 } 878 879 namespace { 880 Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node, 881 Node** false_node, Node* control, BranchHint hint) { 882 DCHECK_NOT_NULL(cond); 883 DCHECK_NOT_NULL(control); 884 Node* branch = 885 mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control); 886 *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch); 887 *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch); 888 return branch; 889 } 890 } // namespace 891 892 Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node, 893 Node** false_node) { 894 return Branch(mcgraph(), cond, true_node, false_node, Control(), 895 BranchHint::kNone); 896 } 897 898 Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node, 899 Node** false_node) { 900 return Branch(mcgraph(), cond, true_node, false_node, Control(), 901 BranchHint::kTrue); 902 } 903 904 Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node, 905 Node** false_node) { 906 return Branch(mcgraph(), cond, true_node, false_node, Control(), 907 BranchHint::kFalse); 908 } 909 910 TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) { 911 // TODO(wasm): "!env_" should not happen when compiling an actual wasm 912 // function. 913 if (!env_ || !env_->runtime_exception_support) { 914 // We use TrapId::kInvalid as a marker to tell the code generator 915 // to generate a call to a testing c-function instead of a runtime 916 // stub. This code should only be called from a cctest. 917 return TrapId::kInvalid; 918 } 919 920 switch (reason) { 921 #define TRAPREASON_TO_TRAPID(name) \ 922 case wasm::k##name: \ 923 static_assert( \ 924 static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \ 925 "trap id mismatch"); \ 926 return TrapId::k##name; 927 FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID) 928 #undef TRAPREASON_TO_TRAPID 929 default: 930 UNREACHABLE(); 931 } 932 } 933 934 Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond, 935 wasm::WasmCodePosition position) { 936 TrapId trap_id = GetTrapIdForTrap(reason); 937 Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id), 938 cond, Effect(), Control())); 939 SetSourcePosition(node, position); 940 return node; 941 } 942 943 Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond, 944 wasm::WasmCodePosition position) { 945 TrapId trap_id = GetTrapIdForTrap(reason); 946 Node* node = SetControl(graph()->NewNode( 947 mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control())); 948 SetSourcePosition(node, position); 949 return node; 950 } 951 952 // Add a check that traps if {node} is equal to {val}. 953 Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node, 954 int32_t val, 955 wasm::WasmCodePosition position) { 956 Int32Matcher m(node); 957 if (m.HasValue() && !m.Is(val)) return graph()->start(); 958 if (val == 0) { 959 return TrapIfFalse(reason, node, position); 960 } else { 961 return TrapIfTrue(reason, 962 graph()->NewNode(mcgraph()->machine()->Word32Equal(), 963 node, mcgraph()->Int32Constant(val)), 964 position); 965 } 966 } 967 968 // Add a check that traps if {node} is zero. 969 Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node, 970 wasm::WasmCodePosition position) { 971 return TrapIfEq32(reason, node, 0, position); 972 } 973 974 // Add a check that traps if {node} is equal to {val}. 975 Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node, 976 int64_t val, 977 wasm::WasmCodePosition position) { 978 Int64Matcher m(node); 979 if (m.HasValue() && !m.Is(val)) return graph()->start(); 980 return TrapIfTrue(reason, 981 graph()->NewNode(mcgraph()->machine()->Word64Equal(), node, 982 mcgraph()->Int64Constant(val)), 983 position); 984 } 985 986 // Add a check that traps if {node} is zero. 987 Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node, 988 wasm::WasmCodePosition position) { 989 return TrapIfEq64(reason, node, 0, position); 990 } 991 992 Node* WasmGraphBuilder::Switch(unsigned count, Node* key) { 993 return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control()); 994 } 995 996 Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) { 997 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode()); 998 return graph()->NewNode(mcgraph()->common()->IfValue(value), sw); 999 } 1000 1001 Node* WasmGraphBuilder::IfDefault(Node* sw) { 1002 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode()); 1003 return graph()->NewNode(mcgraph()->common()->IfDefault(), sw); 1004 } 1005 1006 Node* WasmGraphBuilder::Return(unsigned count, Node** vals) { 1007 static const int kStackAllocatedNodeBufferSize = 8; 1008 Node* stack_buffer[kStackAllocatedNodeBufferSize]; 1009 std::vector<Node*> heap_buffer; 1010 1011 Node** buf = stack_buffer; 1012 if (count + 3 > kStackAllocatedNodeBufferSize) { 1013 heap_buffer.resize(count + 3); 1014 buf = heap_buffer.data(); 1015 } 1016 1017 buf[0] = mcgraph()->Int32Constant(0); 1018 memcpy(buf + 1, vals, sizeof(void*) * count); 1019 buf[count + 1] = Effect(); 1020 buf[count + 2] = Control(); 1021 Node* ret = 1022 graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf); 1023 1024 MergeControlToEnd(mcgraph(), ret); 1025 return ret; 1026 } 1027 1028 Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); } 1029 1030 Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) { 1031 TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position); 1032 ReturnVoid(); 1033 return nullptr; 1034 } 1035 1036 Node* WasmGraphBuilder::MaskShiftCount32(Node* node) { 1037 static const int32_t kMask32 = 0x1F; 1038 if (!mcgraph()->machine()->Word32ShiftIsSafe()) { 1039 // Shifts by constants are so common we pattern-match them here. 1040 Int32Matcher match(node); 1041 if (match.HasValue()) { 1042 int32_t masked = (match.Value() & kMask32); 1043 if (match.Value() != masked) node = mcgraph()->Int32Constant(masked); 1044 } else { 1045 node = graph()->NewNode(mcgraph()->machine()->Word32And(), node, 1046 mcgraph()->Int32Constant(kMask32)); 1047 } 1048 } 1049 return node; 1050 } 1051 1052 Node* WasmGraphBuilder::MaskShiftCount64(Node* node) { 1053 static const int64_t kMask64 = 0x3F; 1054 if (!mcgraph()->machine()->Word32ShiftIsSafe()) { 1055 // Shifts by constants are so common we pattern-match them here. 1056 Int64Matcher match(node); 1057 if (match.HasValue()) { 1058 int64_t masked = (match.Value() & kMask64); 1059 if (match.Value() != masked) node = mcgraph()->Int64Constant(masked); 1060 } else { 1061 node = graph()->NewNode(mcgraph()->machine()->Word64And(), node, 1062 mcgraph()->Int64Constant(kMask64)); 1063 } 1064 } 1065 return node; 1066 } 1067 1068 static bool ReverseBytesSupported(MachineOperatorBuilder* m, 1069 size_t size_in_bytes) { 1070 switch (size_in_bytes) { 1071 case 4: 1072 case 16: 1073 return true; 1074 case 8: 1075 return m->Is64(); 1076 default: 1077 break; 1078 } 1079 return false; 1080 } 1081 1082 Node* WasmGraphBuilder::BuildChangeEndiannessStore( 1083 Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) { 1084 Node* result; 1085 Node* value = node; 1086 MachineOperatorBuilder* m = mcgraph()->machine(); 1087 int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype); 1088 int valueSizeInBits = 8 * valueSizeInBytes; 1089 bool isFloat = false; 1090 1091 switch (wasmtype) { 1092 case wasm::kWasmF64: 1093 value = graph()->NewNode(m->BitcastFloat64ToInt64(), node); 1094 isFloat = true; 1095 V8_FALLTHROUGH; 1096 case wasm::kWasmI64: 1097 result = mcgraph()->Int64Constant(0); 1098 break; 1099 case wasm::kWasmF32: 1100 value = graph()->NewNode(m->BitcastFloat32ToInt32(), node); 1101 isFloat = true; 1102 V8_FALLTHROUGH; 1103 case wasm::kWasmI32: 1104 result = mcgraph()->Int32Constant(0); 1105 break; 1106 case wasm::kWasmS128: 1107 DCHECK(ReverseBytesSupported(m, valueSizeInBytes)); 1108 break; 1109 default: 1110 UNREACHABLE(); 1111 break; 1112 } 1113 1114 if (mem_rep == MachineRepresentation::kWord8) { 1115 // No need to change endianness for byte size, return original node 1116 return node; 1117 } 1118 if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) { 1119 // In case we store lower part of WasmI64 expression, we can truncate 1120 // upper 32bits 1121 value = graph()->NewNode(m->TruncateInt64ToInt32(), value); 1122 valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32); 1123 valueSizeInBits = 8 * valueSizeInBytes; 1124 if (mem_rep == MachineRepresentation::kWord16) { 1125 value = 1126 graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16)); 1127 } 1128 } else if (wasmtype == wasm::kWasmI32 && 1129 mem_rep == MachineRepresentation::kWord16) { 1130 value = 1131 graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16)); 1132 } 1133 1134 int i; 1135 uint32_t shiftCount; 1136 1137 if (ReverseBytesSupported(m, valueSizeInBytes)) { 1138 switch (valueSizeInBytes) { 1139 case 4: 1140 result = graph()->NewNode(m->Word32ReverseBytes(), value); 1141 break; 1142 case 8: 1143 result = graph()->NewNode(m->Word64ReverseBytes(), value); 1144 break; 1145 case 16: { 1146 Node* byte_reversed_lanes[4]; 1147 for (int lane = 0; lane < 4; lane++) { 1148 byte_reversed_lanes[lane] = graph()->NewNode( 1149 m->Word32ReverseBytes(), 1150 graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane), 1151 value)); 1152 } 1153 1154 // This is making a copy of the value. 1155 result = 1156 graph()->NewNode(mcgraph()->machine()->S128And(), value, value); 1157 1158 for (int lane = 0; lane < 4; lane++) { 1159 result = 1160 graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane), 1161 result, byte_reversed_lanes[lane]); 1162 } 1163 1164 break; 1165 } 1166 default: 1167 UNREACHABLE(); 1168 break; 1169 } 1170 } else { 1171 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2; 1172 i += 8, shiftCount -= 16) { 1173 Node* shiftLower; 1174 Node* shiftHigher; 1175 Node* lowerByte; 1176 Node* higherByte; 1177 1178 DCHECK_LT(0, shiftCount); 1179 DCHECK_EQ(0, (shiftCount + 8) % 16); 1180 1181 if (valueSizeInBits > 32) { 1182 shiftLower = graph()->NewNode(m->Word64Shl(), value, 1183 mcgraph()->Int64Constant(shiftCount)); 1184 shiftHigher = graph()->NewNode(m->Word64Shr(), value, 1185 mcgraph()->Int64Constant(shiftCount)); 1186 lowerByte = graph()->NewNode( 1187 m->Word64And(), shiftLower, 1188 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) 1189 << (valueSizeInBits - 8 - i))); 1190 higherByte = graph()->NewNode( 1191 m->Word64And(), shiftHigher, 1192 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i)); 1193 result = graph()->NewNode(m->Word64Or(), result, lowerByte); 1194 result = graph()->NewNode(m->Word64Or(), result, higherByte); 1195 } else { 1196 shiftLower = graph()->NewNode(m->Word32Shl(), value, 1197 mcgraph()->Int32Constant(shiftCount)); 1198 shiftHigher = graph()->NewNode(m->Word32Shr(), value, 1199 mcgraph()->Int32Constant(shiftCount)); 1200 lowerByte = graph()->NewNode( 1201 m->Word32And(), shiftLower, 1202 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) 1203 << (valueSizeInBits - 8 - i))); 1204 higherByte = graph()->NewNode( 1205 m->Word32And(), shiftHigher, 1206 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i)); 1207 result = graph()->NewNode(m->Word32Or(), result, lowerByte); 1208 result = graph()->NewNode(m->Word32Or(), result, higherByte); 1209 } 1210 } 1211 } 1212 1213 if (isFloat) { 1214 switch (wasmtype) { 1215 case wasm::kWasmF64: 1216 result = graph()->NewNode(m->BitcastInt64ToFloat64(), result); 1217 break; 1218 case wasm::kWasmF32: 1219 result = graph()->NewNode(m->BitcastInt32ToFloat32(), result); 1220 break; 1221 default: 1222 UNREACHABLE(); 1223 break; 1224 } 1225 } 1226 1227 return result; 1228 } 1229 1230 Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node, 1231 MachineType memtype, 1232 wasm::ValueType wasmtype) { 1233 Node* result; 1234 Node* value = node; 1235 MachineOperatorBuilder* m = mcgraph()->machine(); 1236 int valueSizeInBytes = ElementSizeInBytes(memtype.representation()); 1237 int valueSizeInBits = 8 * valueSizeInBytes; 1238 bool isFloat = false; 1239 1240 switch (memtype.representation()) { 1241 case MachineRepresentation::kFloat64: 1242 value = graph()->NewNode(m->BitcastFloat64ToInt64(), node); 1243 isFloat = true; 1244 V8_FALLTHROUGH; 1245 case MachineRepresentation::kWord64: 1246 result = mcgraph()->Int64Constant(0); 1247 break; 1248 case MachineRepresentation::kFloat32: 1249 value = graph()->NewNode(m->BitcastFloat32ToInt32(), node); 1250 isFloat = true; 1251 V8_FALLTHROUGH; 1252 case MachineRepresentation::kWord32: 1253 case MachineRepresentation::kWord16: 1254 result = mcgraph()->Int32Constant(0); 1255 break; 1256 case MachineRepresentation::kWord8: 1257 // No need to change endianness for byte size, return original node 1258 return node; 1259 break; 1260 case MachineRepresentation::kSimd128: 1261 DCHECK(ReverseBytesSupported(m, valueSizeInBytes)); 1262 break; 1263 default: 1264 UNREACHABLE(); 1265 break; 1266 } 1267 1268 int i; 1269 uint32_t shiftCount; 1270 1271 if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) { 1272 switch (valueSizeInBytes) { 1273 case 2: 1274 result = 1275 graph()->NewNode(m->Word32ReverseBytes(), 1276 graph()->NewNode(m->Word32Shl(), value, 1277 mcgraph()->Int32Constant(16))); 1278 break; 1279 case 4: 1280 result = graph()->NewNode(m->Word32ReverseBytes(), value); 1281 break; 1282 case 8: 1283 result = graph()->NewNode(m->Word64ReverseBytes(), value); 1284 break; 1285 case 16: { 1286 Node* byte_reversed_lanes[4]; 1287 for (int lane = 0; lane < 4; lane++) { 1288 byte_reversed_lanes[lane] = graph()->NewNode( 1289 m->Word32ReverseBytes(), 1290 graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane), 1291 value)); 1292 } 1293 1294 // This is making a copy of the value. 1295 result = 1296 graph()->NewNode(mcgraph()->machine()->S128And(), value, value); 1297 1298 for (int lane = 0; lane < 4; lane++) { 1299 result = 1300 graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane), 1301 result, byte_reversed_lanes[lane]); 1302 } 1303 1304 break; 1305 } 1306 default: 1307 UNREACHABLE(); 1308 } 1309 } else { 1310 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2; 1311 i += 8, shiftCount -= 16) { 1312 Node* shiftLower; 1313 Node* shiftHigher; 1314 Node* lowerByte; 1315 Node* higherByte; 1316 1317 DCHECK_LT(0, shiftCount); 1318 DCHECK_EQ(0, (shiftCount + 8) % 16); 1319 1320 if (valueSizeInBits > 32) { 1321 shiftLower = graph()->NewNode(m->Word64Shl(), value, 1322 mcgraph()->Int64Constant(shiftCount)); 1323 shiftHigher = graph()->NewNode(m->Word64Shr(), value, 1324 mcgraph()->Int64Constant(shiftCount)); 1325 lowerByte = graph()->NewNode( 1326 m->Word64And(), shiftLower, 1327 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) 1328 << (valueSizeInBits - 8 - i))); 1329 higherByte = graph()->NewNode( 1330 m->Word64And(), shiftHigher, 1331 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i)); 1332 result = graph()->NewNode(m->Word64Or(), result, lowerByte); 1333 result = graph()->NewNode(m->Word64Or(), result, higherByte); 1334 } else { 1335 shiftLower = graph()->NewNode(m->Word32Shl(), value, 1336 mcgraph()->Int32Constant(shiftCount)); 1337 shiftHigher = graph()->NewNode(m->Word32Shr(), value, 1338 mcgraph()->Int32Constant(shiftCount)); 1339 lowerByte = graph()->NewNode( 1340 m->Word32And(), shiftLower, 1341 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) 1342 << (valueSizeInBits - 8 - i))); 1343 higherByte = graph()->NewNode( 1344 m->Word32And(), shiftHigher, 1345 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i)); 1346 result = graph()->NewNode(m->Word32Or(), result, lowerByte); 1347 result = graph()->NewNode(m->Word32Or(), result, higherByte); 1348 } 1349 } 1350 } 1351 1352 if (isFloat) { 1353 switch (memtype.representation()) { 1354 case MachineRepresentation::kFloat64: 1355 result = graph()->NewNode(m->BitcastInt64ToFloat64(), result); 1356 break; 1357 case MachineRepresentation::kFloat32: 1358 result = graph()->NewNode(m->BitcastInt32ToFloat32(), result); 1359 break; 1360 default: 1361 UNREACHABLE(); 1362 break; 1363 } 1364 } 1365 1366 // We need to sign extend the value 1367 if (memtype.IsSigned()) { 1368 DCHECK(!isFloat); 1369 if (valueSizeInBits < 32) { 1370 Node* shiftBitCount; 1371 // Perform sign extension using following trick 1372 // result = (x << machine_width - type_width) >> (machine_width - 1373 // type_width) 1374 if (wasmtype == wasm::kWasmI64) { 1375 shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits); 1376 result = graph()->NewNode( 1377 m->Word64Sar(), 1378 graph()->NewNode(m->Word64Shl(), 1379 graph()->NewNode(m->ChangeInt32ToInt64(), result), 1380 shiftBitCount), 1381 shiftBitCount); 1382 } else if (wasmtype == wasm::kWasmI32) { 1383 shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits); 1384 result = graph()->NewNode( 1385 m->Word32Sar(), 1386 graph()->NewNode(m->Word32Shl(), result, shiftBitCount), 1387 shiftBitCount); 1388 } 1389 } 1390 } 1391 1392 return result; 1393 } 1394 1395 Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) { 1396 Node* result = Unop( 1397 wasm::kExprF32ReinterpretI32, 1398 Binop(wasm::kExprI32Ior, 1399 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left), 1400 mcgraph()->Int32Constant(0x7FFFFFFF)), 1401 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right), 1402 mcgraph()->Int32Constant(0x80000000)))); 1403 1404 return result; 1405 } 1406 1407 Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) { 1408 #if WASM_64 1409 Node* result = Unop( 1410 wasm::kExprF64ReinterpretI64, 1411 Binop(wasm::kExprI64Ior, 1412 Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left), 1413 mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)), 1414 Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right), 1415 mcgraph()->Int64Constant(0x8000000000000000)))); 1416 1417 return result; 1418 #else 1419 MachineOperatorBuilder* m = mcgraph()->machine(); 1420 1421 Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left); 1422 Node* high_word_right = 1423 graph()->NewNode(m->Float64ExtractHighWord32(), right); 1424 1425 Node* new_high_word = Binop(wasm::kExprI32Ior, 1426 Binop(wasm::kExprI32And, high_word_left, 1427 mcgraph()->Int32Constant(0x7FFFFFFF)), 1428 Binop(wasm::kExprI32And, high_word_right, 1429 mcgraph()->Int32Constant(0x80000000))); 1430 1431 return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word); 1432 #endif 1433 } 1434 1435 namespace { 1436 1437 MachineType IntConvertType(wasm::WasmOpcode opcode) { 1438 switch (opcode) { 1439 case wasm::kExprI32SConvertF32: 1440 case wasm::kExprI32SConvertF64: 1441 case wasm::kExprI32SConvertSatF32: 1442 case wasm::kExprI32SConvertSatF64: 1443 return MachineType::Int32(); 1444 case wasm::kExprI32UConvertF32: 1445 case wasm::kExprI32UConvertF64: 1446 case wasm::kExprI32UConvertSatF32: 1447 case wasm::kExprI32UConvertSatF64: 1448 return MachineType::Uint32(); 1449 case wasm::kExprI64SConvertF32: 1450 case wasm::kExprI64SConvertF64: 1451 case wasm::kExprI64SConvertSatF32: 1452 case wasm::kExprI64SConvertSatF64: 1453 return MachineType::Int64(); 1454 case wasm::kExprI64UConvertF32: 1455 case wasm::kExprI64UConvertF64: 1456 case wasm::kExprI64UConvertSatF32: 1457 case wasm::kExprI64UConvertSatF64: 1458 return MachineType::Uint64(); 1459 default: 1460 UNREACHABLE(); 1461 } 1462 } 1463 1464 MachineType FloatConvertType(wasm::WasmOpcode opcode) { 1465 switch (opcode) { 1466 case wasm::kExprI32SConvertF32: 1467 case wasm::kExprI32UConvertF32: 1468 case wasm::kExprI32SConvertSatF32: 1469 case wasm::kExprI64SConvertF32: 1470 case wasm::kExprI64UConvertF32: 1471 case wasm::kExprI32UConvertSatF32: 1472 case wasm::kExprI64SConvertSatF32: 1473 case wasm::kExprI64UConvertSatF32: 1474 return MachineType::Float32(); 1475 case wasm::kExprI32SConvertF64: 1476 case wasm::kExprI32UConvertF64: 1477 case wasm::kExprI64SConvertF64: 1478 case wasm::kExprI64UConvertF64: 1479 case wasm::kExprI32SConvertSatF64: 1480 case wasm::kExprI32UConvertSatF64: 1481 case wasm::kExprI64SConvertSatF64: 1482 case wasm::kExprI64UConvertSatF64: 1483 return MachineType::Float64(); 1484 default: 1485 UNREACHABLE(); 1486 } 1487 } 1488 1489 const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) { 1490 switch (opcode) { 1491 case wasm::kExprI32SConvertF32: 1492 case wasm::kExprI32SConvertSatF32: 1493 return builder->mcgraph()->machine()->TruncateFloat32ToInt32(); 1494 case wasm::kExprI32UConvertF32: 1495 case wasm::kExprI32UConvertSatF32: 1496 return builder->mcgraph()->machine()->TruncateFloat32ToUint32(); 1497 case wasm::kExprI32SConvertF64: 1498 case wasm::kExprI32SConvertSatF64: 1499 return builder->mcgraph()->machine()->ChangeFloat64ToInt32(); 1500 case wasm::kExprI32UConvertF64: 1501 case wasm::kExprI32UConvertSatF64: 1502 return builder->mcgraph()->machine()->TruncateFloat64ToUint32(); 1503 case wasm::kExprI64SConvertF32: 1504 case wasm::kExprI64SConvertSatF32: 1505 return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64(); 1506 case wasm::kExprI64UConvertF32: 1507 case wasm::kExprI64UConvertSatF32: 1508 return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64(); 1509 case wasm::kExprI64SConvertF64: 1510 case wasm::kExprI64SConvertSatF64: 1511 return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64(); 1512 case wasm::kExprI64UConvertF64: 1513 case wasm::kExprI64UConvertSatF64: 1514 return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64(); 1515 default: 1516 UNREACHABLE(); 1517 } 1518 } 1519 1520 wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) { 1521 switch (opcode) { 1522 case wasm::kExprI32SConvertF32: 1523 case wasm::kExprI32SConvertSatF32: 1524 return wasm::kExprF32SConvertI32; 1525 case wasm::kExprI32UConvertF32: 1526 case wasm::kExprI32UConvertSatF32: 1527 return wasm::kExprF32UConvertI32; 1528 case wasm::kExprI32SConvertF64: 1529 case wasm::kExprI32SConvertSatF64: 1530 return wasm::kExprF64SConvertI32; 1531 case wasm::kExprI32UConvertF64: 1532 case wasm::kExprI32UConvertSatF64: 1533 return wasm::kExprF64UConvertI32; 1534 default: 1535 UNREACHABLE(); 1536 } 1537 } 1538 1539 bool IsTrappingConvertOp(wasm::WasmOpcode opcode) { 1540 switch (opcode) { 1541 case wasm::kExprI32SConvertF32: 1542 case wasm::kExprI32UConvertF32: 1543 case wasm::kExprI32SConvertF64: 1544 case wasm::kExprI32UConvertF64: 1545 case wasm::kExprI64SConvertF32: 1546 case wasm::kExprI64UConvertF32: 1547 case wasm::kExprI64SConvertF64: 1548 case wasm::kExprI64UConvertF64: 1549 return true; 1550 case wasm::kExprI32SConvertSatF64: 1551 case wasm::kExprI32UConvertSatF64: 1552 case wasm::kExprI32SConvertSatF32: 1553 case wasm::kExprI32UConvertSatF32: 1554 case wasm::kExprI64SConvertSatF32: 1555 case wasm::kExprI64UConvertSatF32: 1556 case wasm::kExprI64SConvertSatF64: 1557 case wasm::kExprI64UConvertSatF64: 1558 return false; 1559 default: 1560 UNREACHABLE(); 1561 } 1562 } 1563 1564 Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) { 1565 switch (ty.representation()) { 1566 case MachineRepresentation::kWord32: 1567 return builder->Int32Constant(0); 1568 case MachineRepresentation::kWord64: 1569 return builder->Int64Constant(0); 1570 case MachineRepresentation::kFloat32: 1571 return builder->Float32Constant(0.0); 1572 case MachineRepresentation::kFloat64: 1573 return builder->Float64Constant(0.0); 1574 default: 1575 UNREACHABLE(); 1576 } 1577 } 1578 1579 Node* Min(WasmGraphBuilder* builder, const MachineType& ty) { 1580 switch (ty.semantic()) { 1581 case MachineSemantic::kInt32: 1582 return builder->Int32Constant(std::numeric_limits<int32_t>::min()); 1583 case MachineSemantic::kUint32: 1584 return builder->Int32Constant(std::numeric_limits<uint32_t>::min()); 1585 case MachineSemantic::kInt64: 1586 return builder->Int64Constant(std::numeric_limits<int64_t>::min()); 1587 case MachineSemantic::kUint64: 1588 return builder->Int64Constant(std::numeric_limits<uint64_t>::min()); 1589 default: 1590 UNREACHABLE(); 1591 } 1592 } 1593 1594 Node* Max(WasmGraphBuilder* builder, const MachineType& ty) { 1595 switch (ty.semantic()) { 1596 case MachineSemantic::kInt32: 1597 return builder->Int32Constant(std::numeric_limits<int32_t>::max()); 1598 case MachineSemantic::kUint32: 1599 return builder->Int32Constant(std::numeric_limits<uint32_t>::max()); 1600 case MachineSemantic::kInt64: 1601 return builder->Int64Constant(std::numeric_limits<int64_t>::max()); 1602 case MachineSemantic::kUint64: 1603 return builder->Int64Constant(std::numeric_limits<uint64_t>::max()); 1604 default: 1605 UNREACHABLE(); 1606 } 1607 } 1608 1609 wasm::WasmOpcode TruncOp(const MachineType& ty) { 1610 switch (ty.representation()) { 1611 case MachineRepresentation::kFloat32: 1612 return wasm::kExprF32Trunc; 1613 case MachineRepresentation::kFloat64: 1614 return wasm::kExprF64Trunc; 1615 default: 1616 UNREACHABLE(); 1617 } 1618 } 1619 1620 wasm::WasmOpcode NeOp(const MachineType& ty) { 1621 switch (ty.representation()) { 1622 case MachineRepresentation::kFloat32: 1623 return wasm::kExprF32Ne; 1624 case MachineRepresentation::kFloat64: 1625 return wasm::kExprF64Ne; 1626 default: 1627 UNREACHABLE(); 1628 } 1629 } 1630 1631 wasm::WasmOpcode LtOp(const MachineType& ty) { 1632 switch (ty.representation()) { 1633 case MachineRepresentation::kFloat32: 1634 return wasm::kExprF32Lt; 1635 case MachineRepresentation::kFloat64: 1636 return wasm::kExprF64Lt; 1637 default: 1638 UNREACHABLE(); 1639 } 1640 } 1641 1642 Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode, 1643 const MachineType& int_ty, const MachineType& float_ty, 1644 Node* trunc, Node* converted_value) { 1645 if (int_ty.representation() == MachineRepresentation::kWord32) { 1646 Node* check = builder->Unop(ConvertBackOp(opcode), converted_value); 1647 return builder->Binop(NeOp(float_ty), trunc, check); 1648 } 1649 return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1), 1650 trunc, builder->graph()->start()); 1651 } 1652 1653 Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode, 1654 const MachineType& int_ty, 1655 const MachineType& float_ty, Node* trunc, 1656 Node* converted_value) { 1657 Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc, 1658 converted_value); 1659 if (int_ty.representation() == MachineRepresentation::kWord64) { 1660 test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0)); 1661 } 1662 return test; 1663 } 1664 1665 } // namespace 1666 1667 Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input, 1668 wasm::WasmCodePosition position, 1669 wasm::WasmOpcode opcode) { 1670 const MachineType int_ty = IntConvertType(opcode); 1671 const MachineType float_ty = FloatConvertType(opcode); 1672 const Operator* conv_op = ConvertOp(this, opcode); 1673 Node* trunc = nullptr; 1674 Node* converted_value = nullptr; 1675 const bool is_int32 = 1676 int_ty.representation() == MachineRepresentation::kWord32; 1677 if (is_int32) { 1678 trunc = Unop(TruncOp(float_ty), input); 1679 converted_value = graph()->NewNode(conv_op, trunc); 1680 } else { 1681 trunc = graph()->NewNode(conv_op, input); 1682 converted_value = graph()->NewNode(mcgraph()->common()->Projection(0), 1683 trunc, graph()->start()); 1684 } 1685 if (IsTrappingConvertOp(opcode)) { 1686 Node* test = 1687 ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value); 1688 if (is_int32) { 1689 TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position); 1690 } else { 1691 ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position); 1692 } 1693 return converted_value; 1694 } 1695 Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc, 1696 converted_value); 1697 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse); 1698 tl_d.Chain(Control()); 1699 Node* nan_test = Binop(NeOp(float_ty), input, input); 1700 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse); 1701 nan_d.Nest(tl_d, true); 1702 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty)); 1703 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone); 1704 sat_d.Nest(nan_d, false); 1705 Node* sat_val = 1706 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty)); 1707 Node* nan_val = 1708 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val); 1709 return tl_d.Phi(int_ty.representation(), nan_val, converted_value); 1710 } 1711 1712 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) { 1713 MachineOperatorBuilder* m = mcgraph()->machine(); 1714 // asm.js must use the wacky JS semantics. 1715 input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input); 1716 return graph()->NewNode(m->TruncateFloat64ToWord32(), input); 1717 } 1718 1719 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) { 1720 MachineOperatorBuilder* m = mcgraph()->machine(); 1721 // asm.js must use the wacky JS semantics. 1722 return graph()->NewNode(m->TruncateFloat64ToWord32(), input); 1723 } 1724 1725 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) { 1726 MachineOperatorBuilder* m = mcgraph()->machine(); 1727 // asm.js must use the wacky JS semantics. 1728 input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input); 1729 return graph()->NewNode(m->TruncateFloat64ToWord32(), input); 1730 } 1731 1732 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) { 1733 MachineOperatorBuilder* m = mcgraph()->machine(); 1734 // asm.js must use the wacky JS semantics. 1735 return graph()->NewNode(m->TruncateFloat64ToWord32(), input); 1736 } 1737 1738 Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref, 1739 MachineRepresentation input_type) { 1740 Node* stack_slot_param = 1741 graph()->NewNode(mcgraph()->machine()->StackSlot(input_type)); 1742 1743 const Operator* store_op = mcgraph()->machine()->Store( 1744 StoreRepresentation(input_type, kNoWriteBarrier)); 1745 SetEffect(graph()->NewNode(store_op, stack_slot_param, 1746 mcgraph()->Int32Constant(0), input, Effect(), 1747 Control())); 1748 1749 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()}; 1750 MachineSignature sig(1, 1, sig_types); 1751 1752 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref)); 1753 1754 return BuildCCall(&sig, function, stack_slot_param); 1755 } 1756 1757 Node* WasmGraphBuilder::BuildI32Ctz(Node* input) { 1758 return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(), 1759 MachineRepresentation::kWord32); 1760 } 1761 1762 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) { 1763 return Unop(wasm::kExprI64UConvertI32, 1764 BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(), 1765 MachineRepresentation::kWord64)); 1766 } 1767 1768 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) { 1769 return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(), 1770 MachineRepresentation::kWord32); 1771 } 1772 1773 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) { 1774 return Unop( 1775 wasm::kExprI64UConvertI32, 1776 BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(), 1777 MachineRepresentation::kWord64)); 1778 } 1779 1780 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) { 1781 MachineType type = MachineType::Float32(); 1782 ExternalReference ref = ExternalReference::wasm_f32_trunc(); 1783 1784 return BuildCFuncInstruction(ref, type, input); 1785 } 1786 1787 Node* WasmGraphBuilder::BuildF32Floor(Node* input) { 1788 MachineType type = MachineType::Float32(); 1789 ExternalReference ref = ExternalReference::wasm_f32_floor(); 1790 return BuildCFuncInstruction(ref, type, input); 1791 } 1792 1793 Node* WasmGraphBuilder::BuildF32Ceil(Node* input) { 1794 MachineType type = MachineType::Float32(); 1795 ExternalReference ref = ExternalReference::wasm_f32_ceil(); 1796 return BuildCFuncInstruction(ref, type, input); 1797 } 1798 1799 Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) { 1800 MachineType type = MachineType::Float32(); 1801 ExternalReference ref = ExternalReference::wasm_f32_nearest_int(); 1802 return BuildCFuncInstruction(ref, type, input); 1803 } 1804 1805 Node* WasmGraphBuilder::BuildF64Trunc(Node* input) { 1806 MachineType type = MachineType::Float64(); 1807 ExternalReference ref = ExternalReference::wasm_f64_trunc(); 1808 return BuildCFuncInstruction(ref, type, input); 1809 } 1810 1811 Node* WasmGraphBuilder::BuildF64Floor(Node* input) { 1812 MachineType type = MachineType::Float64(); 1813 ExternalReference ref = ExternalReference::wasm_f64_floor(); 1814 return BuildCFuncInstruction(ref, type, input); 1815 } 1816 1817 Node* WasmGraphBuilder::BuildF64Ceil(Node* input) { 1818 MachineType type = MachineType::Float64(); 1819 ExternalReference ref = ExternalReference::wasm_f64_ceil(); 1820 return BuildCFuncInstruction(ref, type, input); 1821 } 1822 1823 Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) { 1824 MachineType type = MachineType::Float64(); 1825 ExternalReference ref = ExternalReference::wasm_f64_nearest_int(); 1826 return BuildCFuncInstruction(ref, type, input); 1827 } 1828 1829 Node* WasmGraphBuilder::BuildF64Acos(Node* input) { 1830 MachineType type = MachineType::Float64(); 1831 ExternalReference ref = ExternalReference::f64_acos_wrapper_function(); 1832 return BuildCFuncInstruction(ref, type, input); 1833 } 1834 1835 Node* WasmGraphBuilder::BuildF64Asin(Node* input) { 1836 MachineType type = MachineType::Float64(); 1837 ExternalReference ref = ExternalReference::f64_asin_wrapper_function(); 1838 return BuildCFuncInstruction(ref, type, input); 1839 } 1840 1841 Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) { 1842 MachineType type = MachineType::Float64(); 1843 ExternalReference ref = ExternalReference::wasm_float64_pow(); 1844 return BuildCFuncInstruction(ref, type, left, right); 1845 } 1846 1847 Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) { 1848 MachineType type = MachineType::Float64(); 1849 ExternalReference ref = ExternalReference::f64_mod_wrapper_function(); 1850 return BuildCFuncInstruction(ref, type, left, right); 1851 } 1852 1853 Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref, 1854 MachineType type, Node* input0, 1855 Node* input1) { 1856 // We do truncation by calling a C function which calculates the result. 1857 // The input is passed to the C function as a byte buffer holding the two 1858 // input doubles. We reserve this byte buffer as a stack slot, store the 1859 // parameters in this buffer slots, pass a pointer to the buffer to the C 1860 // function, and after calling the C function we collect the return value from 1861 // the buffer. 1862 1863 const int type_size = ElementSizeInBytes(type.representation()); 1864 const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size; 1865 Node* stack_slot = 1866 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes)); 1867 1868 const Operator* store_op = mcgraph()->machine()->Store( 1869 StoreRepresentation(type.representation(), kNoWriteBarrier)); 1870 SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0), 1871 input0, Effect(), Control())); 1872 1873 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref)); 1874 1875 if (input1 != nullptr) { 1876 SetEffect(graph()->NewNode(store_op, stack_slot, 1877 mcgraph()->Int32Constant(type_size), input1, 1878 Effect(), Control())); 1879 } 1880 1881 MachineType sig_types[] = {MachineType::Pointer()}; 1882 MachineSignature sig(0, 1, sig_types); 1883 BuildCCall(&sig, function, stack_slot); 1884 1885 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), 1886 stack_slot, mcgraph()->Int32Constant(0), 1887 Effect(), Control())); 1888 } 1889 1890 Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) { 1891 // TODO(titzer/bradnelson): Check handlng of asm.js case. 1892 return BuildIntToFloatConversionInstruction( 1893 input, ExternalReference::wasm_int64_to_float32(), 1894 MachineRepresentation::kWord64, MachineType::Float32()); 1895 } 1896 Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) { 1897 // TODO(titzer/bradnelson): Check handlng of asm.js case. 1898 return BuildIntToFloatConversionInstruction( 1899 input, ExternalReference::wasm_uint64_to_float32(), 1900 MachineRepresentation::kWord64, MachineType::Float32()); 1901 } 1902 Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) { 1903 return BuildIntToFloatConversionInstruction( 1904 input, ExternalReference::wasm_int64_to_float64(), 1905 MachineRepresentation::kWord64, MachineType::Float64()); 1906 } 1907 Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) { 1908 return BuildIntToFloatConversionInstruction( 1909 input, ExternalReference::wasm_uint64_to_float64(), 1910 MachineRepresentation::kWord64, MachineType::Float64()); 1911 } 1912 1913 Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction( 1914 Node* input, ExternalReference ref, 1915 MachineRepresentation parameter_representation, 1916 const MachineType result_type) { 1917 int stack_slot_size = 1918 std::max(ElementSizeInBytes(parameter_representation), 1919 ElementSizeInBytes(result_type.representation())); 1920 Node* stack_slot = 1921 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size)); 1922 const Operator* store_op = mcgraph()->machine()->Store( 1923 StoreRepresentation(parameter_representation, kNoWriteBarrier)); 1924 SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0), 1925 input, Effect(), Control())); 1926 MachineType sig_types[] = {MachineType::Pointer()}; 1927 MachineSignature sig(0, 1, sig_types); 1928 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref)); 1929 BuildCCall(&sig, function, stack_slot); 1930 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type), 1931 stack_slot, mcgraph()->Int32Constant(0), 1932 Effect(), Control())); 1933 } 1934 1935 namespace { 1936 1937 ExternalReference convert_ccall_ref(WasmGraphBuilder* builder, 1938 wasm::WasmOpcode opcode) { 1939 switch (opcode) { 1940 case wasm::kExprI64SConvertF32: 1941 case wasm::kExprI64SConvertSatF32: 1942 return ExternalReference::wasm_float32_to_int64(); 1943 case wasm::kExprI64UConvertF32: 1944 case wasm::kExprI64UConvertSatF32: 1945 return ExternalReference::wasm_float32_to_uint64(); 1946 case wasm::kExprI64SConvertF64: 1947 case wasm::kExprI64SConvertSatF64: 1948 return ExternalReference::wasm_float64_to_int64(); 1949 case wasm::kExprI64UConvertF64: 1950 case wasm::kExprI64UConvertSatF64: 1951 return ExternalReference::wasm_float64_to_uint64(); 1952 default: 1953 UNREACHABLE(); 1954 } 1955 } 1956 1957 } // namespace 1958 1959 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input, 1960 wasm::WasmCodePosition position, 1961 wasm::WasmOpcode opcode) { 1962 const MachineType int_ty = IntConvertType(opcode); 1963 const MachineType float_ty = FloatConvertType(opcode); 1964 ExternalReference call_ref = convert_ccall_ref(this, opcode); 1965 int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()), 1966 ElementSizeInBytes(float_ty.representation())); 1967 Node* stack_slot = 1968 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size)); 1969 const Operator* store_op = mcgraph()->machine()->Store( 1970 StoreRepresentation(float_ty.representation(), kNoWriteBarrier)); 1971 SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input, 1972 Effect(), Control())); 1973 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()}; 1974 MachineSignature sig(1, 1, sig_types); 1975 Node* function = 1976 graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref)); 1977 Node* overflow = BuildCCall(&sig, function, stack_slot); 1978 if (IsTrappingConvertOp(opcode)) { 1979 ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position); 1980 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), 1981 stack_slot, Int32Constant(0), Effect(), 1982 Control())); 1983 } 1984 Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position); 1985 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse); 1986 tl_d.Chain(Control()); 1987 Node* nan_test = Binop(NeOp(float_ty), input, input); 1988 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse); 1989 nan_d.Nest(tl_d, true); 1990 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty)); 1991 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone); 1992 sat_d.Nest(nan_d, false); 1993 Node* sat_val = 1994 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty)); 1995 Node* load = 1996 SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot, 1997 Int32Constant(0), Effect(), Control())); 1998 Node* nan_val = 1999 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val); 2000 return tl_d.Phi(int_ty.representation(), nan_val, load); 2001 } 2002 2003 Node* WasmGraphBuilder::GrowMemory(Node* input) { 2004 SetNeedsStackCheck(); 2005 2006 WasmGrowMemoryDescriptor interface_descriptor; 2007 auto call_descriptor = Linkage::GetStubCallDescriptor( 2008 mcgraph()->zone(), // zone 2009 interface_descriptor, // descriptor 2010 interface_descriptor.GetStackParameterCount(), // stack parameter count 2011 CallDescriptor::kNoFlags, // flags 2012 Operator::kNoProperties, // properties 2013 StubCallMode::kCallWasmRuntimeStub); // stub call mode 2014 // A direct call to a wasm runtime stub defined in this module. 2015 // Just encode the stub index. This will be patched at relocation. 2016 Node* call_target = mcgraph()->RelocatableIntPtrConstant( 2017 wasm::WasmCode::kWasmGrowMemory, RelocInfo::WASM_STUB_CALL); 2018 return SetEffect( 2019 SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor), 2020 call_target, input, Effect(), Control()))); 2021 } 2022 2023 uint32_t WasmGraphBuilder::GetExceptionEncodedSize( 2024 const wasm::WasmException* exception) const { 2025 const wasm::WasmExceptionSig* sig = exception->sig; 2026 uint32_t encoded_size = 0; 2027 for (size_t i = 0; i < sig->parameter_count(); ++i) { 2028 size_t byte_size = static_cast<size_t>( 2029 wasm::ValueTypes::ElementSizeInBytes(sig->GetParam(i))); 2030 DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0); 2031 DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement); 2032 encoded_size += byte_size / kBytesPerExceptionValuesArrayElement; 2033 } 2034 return encoded_size; 2035 } 2036 2037 Node* WasmGraphBuilder::Throw(uint32_t tag, 2038 const wasm::WasmException* exception, 2039 const Vector<Node*> values) { 2040 SetNeedsStackCheck(); 2041 uint32_t encoded_size = GetExceptionEncodedSize(exception); 2042 Node* create_parameters[] = { 2043 BuildChangeUint31ToSmi(ConvertExceptionTagToRuntimeId(tag)), 2044 BuildChangeUint31ToSmi(Uint32Constant(encoded_size))}; 2045 BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters, 2046 arraysize(create_parameters)); 2047 uint32_t index = 0; 2048 const wasm::WasmExceptionSig* sig = exception->sig; 2049 MachineOperatorBuilder* m = mcgraph()->machine(); 2050 for (size_t i = 0; i < sig->parameter_count(); ++i) { 2051 Node* value = values[i]; 2052 switch (sig->GetParam(i)) { 2053 case wasm::kWasmF32: 2054 value = graph()->NewNode(m->BitcastFloat32ToInt32(), value); 2055 V8_FALLTHROUGH; 2056 case wasm::kWasmI32: 2057 BuildEncodeException32BitValue(&index, value); 2058 break; 2059 case wasm::kWasmF64: 2060 value = graph()->NewNode(m->BitcastFloat64ToInt64(), value); 2061 V8_FALLTHROUGH; 2062 case wasm::kWasmI64: { 2063 Node* upper32 = graph()->NewNode( 2064 m->TruncateInt64ToInt32(), 2065 Binop(wasm::kExprI64ShrU, value, Int64Constant(32))); 2066 BuildEncodeException32BitValue(&index, upper32); 2067 Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value); 2068 BuildEncodeException32BitValue(&index, lower32); 2069 break; 2070 } 2071 default: 2072 UNREACHABLE(); 2073 } 2074 } 2075 DCHECK_EQ(encoded_size, index); 2076 return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0); 2077 } 2078 2079 void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index, 2080 Node* value) { 2081 MachineOperatorBuilder* machine = mcgraph()->machine(); 2082 Node* upper_parameters[] = { 2083 BuildChangeUint31ToSmi(Int32Constant(*index)), 2084 BuildChangeUint31ToSmi( 2085 graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))), 2086 }; 2087 BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters, 2088 arraysize(upper_parameters)); 2089 ++(*index); 2090 Node* lower_parameters[] = { 2091 BuildChangeUint31ToSmi(Int32Constant(*index)), 2092 BuildChangeUint31ToSmi(graph()->NewNode(machine->Word32And(), value, 2093 Int32Constant(0xFFFFu))), 2094 }; 2095 BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters, 2096 arraysize(lower_parameters)); 2097 ++(*index); 2098 } 2099 2100 Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values, 2101 uint32_t* index) { 2102 MachineOperatorBuilder* machine = mcgraph()->machine(); 2103 Node* upper = BuildChangeSmiToInt32(values[*index]); 2104 (*index)++; 2105 upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16)); 2106 Node* lower = BuildChangeSmiToInt32(values[*index]); 2107 (*index)++; 2108 Node* value = graph()->NewNode(machine->Word32Or(), upper, lower); 2109 return value; 2110 } 2111 2112 Node* WasmGraphBuilder::Rethrow() { 2113 SetNeedsStackCheck(); 2114 Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0); 2115 return result; 2116 } 2117 2118 Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) { 2119 // TODO(kschimpf): Handle exceptions from different modules, when they are 2120 // linked at runtime. 2121 return Uint32Constant(tag); 2122 } 2123 2124 Node* WasmGraphBuilder::GetExceptionRuntimeId() { 2125 SetNeedsStackCheck(); 2126 return BuildChangeSmiToInt32( 2127 BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0)); 2128 } 2129 2130 Node** WasmGraphBuilder::GetExceptionValues( 2131 const wasm::WasmException* except_decl) { 2132 // TODO(kschimpf): We need to move this code to the function-body-decoder.cc 2133 // in order to build landing-pad (exception) edges in case the runtime 2134 // call causes an exception. 2135 2136 // Start by getting the encoded values from the exception. 2137 uint32_t encoded_size = GetExceptionEncodedSize(except_decl); 2138 Node** values = Buffer(encoded_size); 2139 for (uint32_t i = 0; i < encoded_size; ++i) { 2140 Node* parameters[] = {BuildChangeUint31ToSmi(Uint32Constant(i))}; 2141 values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement, 2142 parameters, arraysize(parameters)); 2143 } 2144 2145 // Now convert the leading entries to the corresponding parameter values. 2146 uint32_t index = 0; 2147 const wasm::WasmExceptionSig* sig = except_decl->sig; 2148 for (size_t i = 0; i < sig->parameter_count(); ++i) { 2149 Node* value = BuildDecodeException32BitValue(values, &index); 2150 switch (wasm::ValueType type = sig->GetParam(i)) { 2151 case wasm::kWasmF32: { 2152 value = Unop(wasm::kExprF32ReinterpretI32, value); 2153 break; 2154 } 2155 case wasm::kWasmI32: 2156 break; 2157 case wasm::kWasmF64: 2158 case wasm::kWasmI64: { 2159 Node* upper = 2160 Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value), 2161 Int64Constant(32)); 2162 Node* lower = Unop(wasm::kExprI64UConvertI32, 2163 BuildDecodeException32BitValue(values, &index)); 2164 value = Binop(wasm::kExprI64Ior, upper, lower); 2165 if (type == wasm::kWasmF64) { 2166 value = Unop(wasm::kExprF64ReinterpretI64, value); 2167 } 2168 break; 2169 } 2170 default: 2171 UNREACHABLE(); 2172 } 2173 values[i] = value; 2174 } 2175 DCHECK_EQ(index, encoded_size); 2176 return values; 2177 } 2178 2179 Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right, 2180 wasm::WasmCodePosition position) { 2181 MachineOperatorBuilder* m = mcgraph()->machine(); 2182 ZeroCheck32(wasm::kTrapDivByZero, right, position); 2183 Node* before = Control(); 2184 Node* denom_is_m1; 2185 Node* denom_is_not_m1; 2186 BranchExpectFalse( 2187 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)), 2188 &denom_is_m1, &denom_is_not_m1); 2189 SetControl(denom_is_m1); 2190 TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position); 2191 if (Control() != denom_is_m1) { 2192 SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1, 2193 Control())); 2194 } else { 2195 SetControl(before); 2196 } 2197 return graph()->NewNode(m->Int32Div(), left, right, Control()); 2198 } 2199 2200 Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right, 2201 wasm::WasmCodePosition position) { 2202 MachineOperatorBuilder* m = mcgraph()->machine(); 2203 2204 ZeroCheck32(wasm::kTrapRemByZero, right, position); 2205 2206 Diamond d( 2207 graph(), mcgraph()->common(), 2208 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)), 2209 BranchHint::kFalse); 2210 d.Chain(Control()); 2211 2212 return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), 2213 graph()->NewNode(m->Int32Mod(), left, right, d.if_false)); 2214 } 2215 2216 Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right, 2217 wasm::WasmCodePosition position) { 2218 MachineOperatorBuilder* m = mcgraph()->machine(); 2219 return graph()->NewNode(m->Uint32Div(), left, right, 2220 ZeroCheck32(wasm::kTrapDivByZero, right, position)); 2221 } 2222 2223 Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right, 2224 wasm::WasmCodePosition position) { 2225 MachineOperatorBuilder* m = mcgraph()->machine(); 2226 return graph()->NewNode(m->Uint32Mod(), left, right, 2227 ZeroCheck32(wasm::kTrapRemByZero, right, position)); 2228 } 2229 2230 Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) { 2231 MachineOperatorBuilder* m = mcgraph()->machine(); 2232 2233 Int32Matcher mr(right); 2234 if (mr.HasValue()) { 2235 if (mr.Value() == 0) { 2236 return mcgraph()->Int32Constant(0); 2237 } else if (mr.Value() == -1) { 2238 // The result is the negation of the left input. 2239 return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left); 2240 } 2241 return graph()->NewNode(m->Int32Div(), left, right, Control()); 2242 } 2243 2244 // asm.js semantics return 0 on divide or mod by zero. 2245 if (m->Int32DivIsSafe()) { 2246 // The hardware instruction does the right thing (e.g. arm). 2247 return graph()->NewNode(m->Int32Div(), left, right, graph()->start()); 2248 } 2249 2250 // Check denominator for zero. 2251 Diamond z( 2252 graph(), mcgraph()->common(), 2253 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)), 2254 BranchHint::kFalse); 2255 2256 // Check numerator for -1. (avoid minint / -1 case). 2257 Diamond n( 2258 graph(), mcgraph()->common(), 2259 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)), 2260 BranchHint::kFalse); 2261 2262 Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false); 2263 Node* neg = 2264 graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left); 2265 2266 return n.Phi( 2267 MachineRepresentation::kWord32, neg, 2268 z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div)); 2269 } 2270 2271 Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) { 2272 CommonOperatorBuilder* c = mcgraph()->common(); 2273 MachineOperatorBuilder* m = mcgraph()->machine(); 2274 Node* const zero = mcgraph()->Int32Constant(0); 2275 2276 Int32Matcher mr(right); 2277 if (mr.HasValue()) { 2278 if (mr.Value() == 0 || mr.Value() == -1) { 2279 return zero; 2280 } 2281 return graph()->NewNode(m->Int32Mod(), left, right, Control()); 2282 } 2283 2284 // General case for signed integer modulus, with optimization for (unknown) 2285 // power of 2 right hand side. 2286 // 2287 // if 0 < right then 2288 // msk = right - 1 2289 // if right & msk != 0 then 2290 // left % right 2291 // else 2292 // if left < 0 then 2293 // -(-left & msk) 2294 // else 2295 // left & msk 2296 // else 2297 // if right < -1 then 2298 // left % right 2299 // else 2300 // zero 2301 // 2302 // Note: We do not use the Diamond helper class here, because it really hurts 2303 // readability with nested diamonds. 2304 Node* const minus_one = mcgraph()->Int32Constant(-1); 2305 2306 const Operator* const merge_op = c->Merge(2); 2307 const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2); 2308 2309 Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right); 2310 Node* branch0 = 2311 graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start()); 2312 2313 Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0); 2314 Node* true0; 2315 { 2316 Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one); 2317 2318 Node* check1 = graph()->NewNode(m->Word32And(), right, msk); 2319 Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0); 2320 2321 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1); 2322 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1); 2323 2324 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1); 2325 Node* false1; 2326 { 2327 Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero); 2328 Node* branch2 = 2329 graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1); 2330 2331 Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2); 2332 Node* true2 = graph()->NewNode( 2333 m->Int32Sub(), zero, 2334 graph()->NewNode(m->Word32And(), 2335 graph()->NewNode(m->Int32Sub(), zero, left), msk)); 2336 2337 Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2); 2338 Node* false2 = graph()->NewNode(m->Word32And(), left, msk); 2339 2340 if_false1 = graph()->NewNode(merge_op, if_true2, if_false2); 2341 false1 = graph()->NewNode(phi_op, true2, false2, if_false1); 2342 } 2343 2344 if_true0 = graph()->NewNode(merge_op, if_true1, if_false1); 2345 true0 = graph()->NewNode(phi_op, true1, false1, if_true0); 2346 } 2347 2348 Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0); 2349 Node* false0; 2350 { 2351 Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one); 2352 Node* branch1 = 2353 graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0); 2354 2355 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1); 2356 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1); 2357 2358 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1); 2359 Node* false1 = zero; 2360 2361 if_false0 = graph()->NewNode(merge_op, if_true1, if_false1); 2362 false0 = graph()->NewNode(phi_op, true1, false1, if_false0); 2363 } 2364 2365 Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0); 2366 return graph()->NewNode(phi_op, true0, false0, merge0); 2367 } 2368 2369 Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) { 2370 MachineOperatorBuilder* m = mcgraph()->machine(); 2371 // asm.js semantics return 0 on divide or mod by zero. 2372 if (m->Uint32DivIsSafe()) { 2373 // The hardware instruction does the right thing (e.g. arm). 2374 return graph()->NewNode(m->Uint32Div(), left, right, graph()->start()); 2375 } 2376 2377 // Explicit check for x % 0. 2378 Diamond z( 2379 graph(), mcgraph()->common(), 2380 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)), 2381 BranchHint::kFalse); 2382 2383 return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), 2384 graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right, 2385 z.if_false)); 2386 } 2387 2388 Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) { 2389 MachineOperatorBuilder* m = mcgraph()->machine(); 2390 // asm.js semantics return 0 on divide or mod by zero. 2391 // Explicit check for x % 0. 2392 Diamond z( 2393 graph(), mcgraph()->common(), 2394 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)), 2395 BranchHint::kFalse); 2396 2397 Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right, 2398 z.if_false); 2399 return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), 2400 rem); 2401 } 2402 2403 Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right, 2404 wasm::WasmCodePosition position) { 2405 if (mcgraph()->machine()->Is32()) { 2406 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(), 2407 MachineType::Int64(), wasm::kTrapDivByZero, position); 2408 } 2409 ZeroCheck64(wasm::kTrapDivByZero, right, position); 2410 Node* before = Control(); 2411 Node* denom_is_m1; 2412 Node* denom_is_not_m1; 2413 BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right, 2414 mcgraph()->Int64Constant(-1)), 2415 &denom_is_m1, &denom_is_not_m1); 2416 SetControl(denom_is_m1); 2417 TrapIfEq64(wasm::kTrapDivUnrepresentable, left, 2418 std::numeric_limits<int64_t>::min(), position); 2419 if (Control() != denom_is_m1) { 2420 SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1, 2421 Control())); 2422 } else { 2423 SetControl(before); 2424 } 2425 return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right, 2426 Control()); 2427 } 2428 2429 Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right, 2430 wasm::WasmCodePosition position) { 2431 if (mcgraph()->machine()->Is32()) { 2432 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(), 2433 MachineType::Int64(), wasm::kTrapRemByZero, position); 2434 } 2435 ZeroCheck64(wasm::kTrapRemByZero, right, position); 2436 Diamond d(mcgraph()->graph(), mcgraph()->common(), 2437 graph()->NewNode(mcgraph()->machine()->Word64Equal(), right, 2438 mcgraph()->Int64Constant(-1))); 2439 2440 d.Chain(Control()); 2441 2442 Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right, 2443 d.if_false); 2444 2445 return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0), 2446 rem); 2447 } 2448 2449 Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right, 2450 wasm::WasmCodePosition position) { 2451 if (mcgraph()->machine()->Is32()) { 2452 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(), 2453 MachineType::Int64(), wasm::kTrapDivByZero, position); 2454 } 2455 return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right, 2456 ZeroCheck64(wasm::kTrapDivByZero, right, position)); 2457 } 2458 Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right, 2459 wasm::WasmCodePosition position) { 2460 if (mcgraph()->machine()->Is32()) { 2461 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(), 2462 MachineType::Int64(), wasm::kTrapRemByZero, position); 2463 } 2464 return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right, 2465 ZeroCheck64(wasm::kTrapRemByZero, right, position)); 2466 } 2467 2468 Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right, 2469 ExternalReference ref, 2470 MachineType result_type, 2471 wasm::TrapReason trap_zero, 2472 wasm::WasmCodePosition position) { 2473 Node* stack_slot = 2474 graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double))); 2475 2476 const Operator* store_op = mcgraph()->machine()->Store( 2477 StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier)); 2478 SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0), 2479 left, Effect(), Control())); 2480 SetEffect(graph()->NewNode(store_op, stack_slot, 2481 mcgraph()->Int32Constant(sizeof(double)), right, 2482 Effect(), Control())); 2483 2484 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()}; 2485 MachineSignature sig(1, 1, sig_types); 2486 2487 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref)); 2488 Node* call = BuildCCall(&sig, function, stack_slot); 2489 2490 ZeroCheck32(trap_zero, call, position); 2491 TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position); 2492 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type), 2493 stack_slot, mcgraph()->Int32Constant(0), 2494 Effect(), Control())); 2495 } 2496 2497 template <typename... Args> 2498 Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function, 2499 Args... args) { 2500 DCHECK_LE(sig->return_count(), 1); 2501 DCHECK_EQ(sizeof...(args), sig->parameter_count()); 2502 Node* const call_args[] = {function, args..., Effect(), Control()}; 2503 2504 auto call_descriptor = 2505 Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig); 2506 2507 const Operator* op = mcgraph()->common()->Call(call_descriptor); 2508 return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args)); 2509 } 2510 2511 Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args, 2512 Node*** rets, 2513 wasm::WasmCodePosition position, 2514 Node* instance_node, 2515 UseRetpoline use_retpoline) { 2516 if (instance_node == nullptr) { 2517 DCHECK_NOT_NULL(instance_node_); 2518 instance_node = instance_node_.get(); 2519 } 2520 SetNeedsStackCheck(); 2521 const size_t params = sig->parameter_count(); 2522 const size_t extra = 3; // instance_node, effect, and control. 2523 const size_t count = 1 + params + extra; 2524 2525 // Reallocate the buffer to make space for extra inputs. 2526 args = Realloc(args, 1 + params, count); 2527 2528 // Make room for the instance_node parameter at index 1, just after code. 2529 memmove(&args[2], &args[1], params * sizeof(Node*)); 2530 args[1] = instance_node; 2531 2532 // Add effect and control inputs. 2533 args[params + 2] = Effect(); 2534 args[params + 3] = Control(); 2535 2536 auto call_descriptor = 2537 GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline); 2538 const Operator* op = mcgraph()->common()->Call(call_descriptor); 2539 Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args)); 2540 DCHECK(position == wasm::kNoCodePosition || position > 0); 2541 if (position > 0) SetSourcePosition(call, position); 2542 2543 size_t ret_count = sig->return_count(); 2544 if (ret_count == 0) return call; // No return value. 2545 2546 *rets = Buffer(ret_count); 2547 if (ret_count == 1) { 2548 // Only a single return value. 2549 (*rets)[0] = call; 2550 } else { 2551 // Create projections for all return values. 2552 for (size_t i = 0; i < ret_count; i++) { 2553 (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call, 2554 graph()->start()); 2555 } 2556 } 2557 return call; 2558 } 2559 2560 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args, 2561 Node*** rets, 2562 wasm::WasmCodePosition position, 2563 int func_index) { 2564 // Load the instance from the imported_instances array at a known offset. 2565 Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances, 2566 MachineType::TaggedPointer()); 2567 Node* instance_node = LOAD_FIXED_ARRAY_SLOT(imported_instances, func_index); 2568 2569 // Load the target from the imported_targets array at a known offset. 2570 Node* imported_targets = 2571 LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer()); 2572 Node* target_node = SetEffect(graph()->NewNode( 2573 mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets, 2574 mcgraph()->Int32Constant(func_index * kPointerSize), Effect(), 2575 Control())); 2576 args[0] = target_node; 2577 return BuildWasmCall(sig, args, rets, position, instance_node, 2578 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline); 2579 } 2580 2581 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args, 2582 Node*** rets, 2583 wasm::WasmCodePosition position, 2584 Node* func_index) { 2585 // Load the instance from the imported_instances array. 2586 Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances, 2587 MachineType::TaggedPointer()); 2588 // Access fixed array at {header_size - tag + func_index * kPointerSize}. 2589 Node* imported_instances_data = 2590 graph()->NewNode(mcgraph()->machine()->IntAdd(), imported_instances, 2591 mcgraph()->IntPtrConstant(FixedArrayOffsetMinusTag(0))); 2592 Node* func_index_times_pointersize = graph()->NewNode( 2593 mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index), 2594 mcgraph()->Int32Constant(kPointerSize)); 2595 Node* instance_node = SetEffect( 2596 graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()), 2597 imported_instances_data, func_index_times_pointersize, 2598 Effect(), Control())); 2599 2600 // Load the target from the imported_targets array at the offset of 2601 // {func_index}. 2602 Node* imported_targets = 2603 LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer()); 2604 Node* target_node = SetEffect(graph()->NewNode( 2605 mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets, 2606 func_index_times_pointersize, Effect(), Control())); 2607 args[0] = target_node; 2608 return BuildWasmCall(sig, args, rets, position, instance_node, 2609 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline); 2610 } 2611 2612 Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets, 2613 wasm::WasmCodePosition position) { 2614 DCHECK_NULL(args[0]); 2615 wasm::FunctionSig* sig = env_->module->functions[index].sig; 2616 2617 if (env_ && index < env_->module->num_imported_functions) { 2618 // Call to an imported function. 2619 return BuildImportWasmCall(sig, args, rets, position, index); 2620 } 2621 2622 // A direct call to a wasm function defined in this module. 2623 // Just encode the function index. This will be patched at instantiation. 2624 Address code = static_cast<Address>(index); 2625 args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL); 2626 2627 return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline); 2628 } 2629 2630 Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args, 2631 Node*** rets, 2632 wasm::WasmCodePosition position) { 2633 DCHECK_NOT_NULL(args[0]); 2634 DCHECK_NOT_NULL(env_); 2635 2636 // Assume only one table for now. 2637 wasm::FunctionSig* sig = env_->module->signatures[sig_index]; 2638 2639 Node* ift_size = 2640 LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32()); 2641 2642 MachineOperatorBuilder* machine = mcgraph()->machine(); 2643 Node* key = args[0]; 2644 2645 // Bounds check against the table size. 2646 Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size); 2647 TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position); 2648 2649 // Mask the key to prevent SSCA. 2650 if (untrusted_code_mitigations_) { 2651 // mask = ((key - size) & ~key) >> 31 2652 Node* neg_key = 2653 graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1)); 2654 Node* masked_diff = graph()->NewNode( 2655 machine->Word32And(), 2656 graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key); 2657 Node* mask = 2658 graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31)); 2659 key = graph()->NewNode(machine->Word32And(), key, mask); 2660 } 2661 2662 // Load signature from the table and check. 2663 Node* ift_sig_ids = 2664 LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer()); 2665 2666 int32_t expected_sig_id = env_->module->signature_ids[sig_index]; 2667 Node* scaled_key = Uint32ToUintptr( 2668 graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2))); 2669 2670 Node* loaded_sig = 2671 SetEffect(graph()->NewNode(machine->Load(MachineType::Int32()), 2672 ift_sig_ids, scaled_key, Effect(), Control())); 2673 Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig, 2674 Int32Constant(expected_sig_id)); 2675 2676 TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position); 2677 2678 Node* ift_targets = 2679 LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer()); 2680 Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableInstances, 2681 MachineType::TaggedPointer()); 2682 2683 scaled_key = graph()->NewNode(machine->Word32Shl(), key, 2684 Int32Constant(kPointerSizeLog2)); 2685 2686 Node* target = 2687 SetEffect(graph()->NewNode(machine->Load(MachineType::Pointer()), 2688 ift_targets, scaled_key, Effect(), Control())); 2689 2690 auto access = AccessBuilder::ForFixedArrayElement(); 2691 Node* target_instance = SetEffect(graph()->NewNode( 2692 machine->Load(MachineType::TaggedPointer()), 2693 graph()->NewNode(machine->IntAdd(), ift_instances, scaled_key), 2694 Int32Constant(access.header_size - access.tag()), Effect(), Control())); 2695 2696 args[0] = target; 2697 2698 return BuildWasmCall(sig, args, rets, position, target_instance, 2699 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline); 2700 } 2701 2702 Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) { 2703 // Implement Rol by Ror since TurboFan does not have Rol opcode. 2704 // TODO(weiliang): support Word32Rol opcode in TurboFan. 2705 Int32Matcher m(right); 2706 if (m.HasValue()) { 2707 return Binop(wasm::kExprI32Ror, left, 2708 mcgraph()->Int32Constant(32 - m.Value())); 2709 } else { 2710 return Binop(wasm::kExprI32Ror, left, 2711 Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right)); 2712 } 2713 } 2714 2715 Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) { 2716 // Implement Rol by Ror since TurboFan does not have Rol opcode. 2717 // TODO(weiliang): support Word64Rol opcode in TurboFan. 2718 Int64Matcher m(right); 2719 if (m.HasValue()) { 2720 return Binop(wasm::kExprI64Ror, left, 2721 mcgraph()->Int64Constant(64 - m.Value())); 2722 } else { 2723 return Binop(wasm::kExprI64Ror, left, 2724 Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right)); 2725 } 2726 } 2727 2728 Node* WasmGraphBuilder::Invert(Node* node) { 2729 return Unop(wasm::kExprI32Eqz, node); 2730 } 2731 2732 bool CanCover(Node* value, IrOpcode::Value opcode) { 2733 if (value->opcode() != opcode) return false; 2734 bool first = true; 2735 for (Edge const edge : value->use_edges()) { 2736 if (NodeProperties::IsControlEdge(edge)) continue; 2737 if (NodeProperties::IsEffectEdge(edge)) continue; 2738 DCHECK(NodeProperties::IsValueEdge(edge)); 2739 if (!first) return false; 2740 first = false; 2741 } 2742 return true; 2743 } 2744 2745 Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) { 2746 if (mcgraph()->machine()->Is64()) { 2747 value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value); 2748 } 2749 return value; 2750 } 2751 2752 Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) { 2753 value = BuildChangeInt32ToIntPtr(value); 2754 return graph()->NewNode(mcgraph()->machine()->WordShl(), value, 2755 BuildSmiShiftBitsConstant()); 2756 } 2757 2758 Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) { 2759 return graph()->NewNode(mcgraph()->machine()->WordShl(), 2760 Uint32ToUintptr(value), BuildSmiShiftBitsConstant()); 2761 } 2762 2763 Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() { 2764 return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize); 2765 } 2766 2767 Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) { 2768 value = graph()->NewNode(mcgraph()->machine()->WordSar(), value, 2769 BuildSmiShiftBitsConstant()); 2770 if (mcgraph()->machine()->Is64()) { 2771 value = 2772 graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value); 2773 } 2774 return value; 2775 } 2776 2777 void WasmGraphBuilder::InitInstanceCache( 2778 WasmInstanceCacheNodes* instance_cache) { 2779 DCHECK_NOT_NULL(instance_node_); 2780 2781 // Load the memory start. 2782 instance_cache->mem_start = SetEffect(graph()->NewNode( 2783 mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(), 2784 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)), 2785 Effect(), Control())); 2786 2787 // Load the memory size. 2788 instance_cache->mem_size = SetEffect(graph()->NewNode( 2789 mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(), 2790 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)), 2791 Effect(), Control())); 2792 2793 if (untrusted_code_mitigations_) { 2794 // Load the memory mask. 2795 instance_cache->mem_mask = SetEffect(graph()->NewNode( 2796 mcgraph()->machine()->Load(MachineType::UintPtr()), 2797 instance_node_.get(), 2798 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryMask)), 2799 Effect(), Control())); 2800 } else { 2801 // Explicitly set to nullptr to ensure a SEGV when we try to use it. 2802 instance_cache->mem_mask = nullptr; 2803 } 2804 } 2805 2806 void WasmGraphBuilder::PrepareInstanceCacheForLoop( 2807 WasmInstanceCacheNodes* instance_cache, Node* control) { 2808 #define INTRODUCE_PHI(field, rep) \ 2809 instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \ 2810 instance_cache->field, control); 2811 2812 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); 2813 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); 2814 if (untrusted_code_mitigations_) { 2815 INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); 2816 } 2817 2818 #undef INTRODUCE_PHI 2819 } 2820 2821 void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to, 2822 WasmInstanceCacheNodes* from, 2823 Node* merge) { 2824 #define INTRODUCE_PHI(field, rep) \ 2825 if (to->field != from->field) { \ 2826 Node* vals[] = {to->field, from->field, merge}; \ 2827 to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \ 2828 } 2829 2830 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); 2831 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); 2832 if (untrusted_code_mitigations_) { 2833 INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); 2834 } 2835 2836 #undef INTRODUCE_PHI 2837 } 2838 2839 void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to, 2840 WasmInstanceCacheNodes* from, 2841 Node* merge) { 2842 to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), 2843 merge, to->mem_size, from->mem_size); 2844 to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), 2845 merge, to->mem_start, from->mem_start); 2846 if (untrusted_code_mitigations_) { 2847 to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), 2848 merge, to->mem_mask, from->mem_mask); 2849 } 2850 } 2851 2852 Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep, 2853 Node* merge, Node* tnode, 2854 Node* fnode) { 2855 if (IsPhiWithMerge(tnode, merge)) { 2856 AppendToPhi(tnode, fnode); 2857 } else if (tnode != fnode) { 2858 uint32_t count = merge->InputCount(); 2859 // + 1 for the merge node. 2860 Node** vals = Buffer(count + 1); 2861 for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode; 2862 vals[count - 1] = fnode; 2863 vals[count] = merge; 2864 return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1, 2865 vals); 2866 } 2867 return tnode; 2868 } 2869 2870 Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode, 2871 Node* fnode) { 2872 if (IsPhiWithMerge(tnode, merge)) { 2873 AppendToPhi(tnode, fnode); 2874 } else if (tnode != fnode) { 2875 uint32_t count = merge->InputCount(); 2876 Node** effects = Buffer(count); 2877 for (uint32_t j = 0; j < count - 1; j++) { 2878 effects[j] = tnode; 2879 } 2880 effects[count - 1] = fnode; 2881 tnode = EffectPhi(count, effects, merge); 2882 } 2883 return tnode; 2884 } 2885 2886 void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type, 2887 const wasm::WasmGlobal& global, 2888 Node** base_node, 2889 Node** offset_node) { 2890 DCHECK_NOT_NULL(instance_node_); 2891 if (global.mutability && global.imported) { 2892 if (imported_mutable_globals_ == nullptr) { 2893 // Load imported_mutable_globals_ from the instance object at runtime. 2894 imported_mutable_globals_ = graph()->NewNode( 2895 mcgraph()->machine()->Load(MachineType::UintPtr()), 2896 instance_node_.get(), 2897 mcgraph()->Int32Constant( 2898 WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)), 2899 graph()->start(), graph()->start()); 2900 } 2901 *base_node = SetEffect(graph()->NewNode( 2902 mcgraph()->machine()->Load(MachineType::UintPtr()), 2903 imported_mutable_globals_.get(), 2904 mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(), 2905 Control())); 2906 *offset_node = mcgraph()->Int32Constant(0); 2907 } else { 2908 if (globals_start_ == nullptr) { 2909 // Load globals_start from the instance object at runtime. 2910 // TODO(wasm): we currently generate only one load of the {globals_start} 2911 // start per graph, which means it can be placed anywhere by the 2912 // scheduler. This is legal because the globals_start should never change. 2913 // However, in some cases (e.g. if the instance object is already in a 2914 // register), it is slightly more efficient to reload this value from the 2915 // instance object. Since this depends on register allocation, it is not 2916 // possible to express in the graph, and would essentially constitute a 2917 // "mem2reg" optimization in TurboFan. 2918 globals_start_ = graph()->NewNode( 2919 mcgraph()->machine()->Load(MachineType::UintPtr()), 2920 instance_node_.get(), 2921 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)), 2922 graph()->start(), graph()->start()); 2923 } 2924 *base_node = globals_start_.get(); 2925 *offset_node = mcgraph()->Int32Constant(global.offset); 2926 2927 if (mem_type == MachineType::Simd128() && global.offset != 0) { 2928 // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken. 2929 *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node, 2930 *offset_node); 2931 *offset_node = mcgraph()->Int32Constant(0); 2932 } 2933 } 2934 } 2935 2936 Node* WasmGraphBuilder::MemBuffer(uint32_t offset) { 2937 DCHECK_NOT_NULL(instance_cache_); 2938 Node* mem_start = instance_cache_->mem_start; 2939 DCHECK_NOT_NULL(mem_start); 2940 if (offset == 0) return mem_start; 2941 return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start, 2942 mcgraph()->IntPtrConstant(offset)); 2943 } 2944 2945 Node* WasmGraphBuilder::CurrentMemoryPages() { 2946 // CurrentMemoryPages can not be called from asm.js. 2947 DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin); 2948 DCHECK_NOT_NULL(instance_cache_); 2949 Node* mem_size = instance_cache_->mem_size; 2950 DCHECK_NOT_NULL(mem_size); 2951 Node* result = 2952 graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size, 2953 mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2)); 2954 if (mcgraph()->machine()->Is64()) { 2955 result = 2956 graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result); 2957 } 2958 return result; 2959 } 2960 2961 // Only call this function for code which is not reused across instantiations, 2962 // as we do not patch the embedded js_context. 2963 Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f, 2964 Node* js_context, 2965 Node** parameters, 2966 int parameter_count) { 2967 const Runtime::Function* fun = Runtime::FunctionForId(f); 2968 auto call_descriptor = Linkage::GetRuntimeCallDescriptor( 2969 mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties, 2970 CallDescriptor::kNoFlags); 2971 // The CEntryStub is loaded from the instance_node so that generated code is 2972 // Isolate independent. At the moment this is only done for CEntryStub(1). 2973 DCHECK_EQ(1, fun->result_size); 2974 Node* centry_stub = 2975 LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer()); 2976 // At the moment we only allow 4 parameters. If more parameters are needed, 2977 // increase this constant accordingly. 2978 static const int kMaxParams = 4; 2979 DCHECK_GE(kMaxParams, parameter_count); 2980 Node* inputs[kMaxParams + 6]; 2981 int count = 0; 2982 inputs[count++] = centry_stub; 2983 for (int i = 0; i < parameter_count; i++) { 2984 inputs[count++] = parameters[i]; 2985 } 2986 inputs[count++] = 2987 mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref 2988 inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity 2989 inputs[count++] = js_context; // js_context 2990 inputs[count++] = Effect(); 2991 inputs[count++] = Control(); 2992 2993 return SetEffect(mcgraph()->graph()->NewNode( 2994 mcgraph()->common()->Call(call_descriptor), count, inputs)); 2995 } 2996 2997 Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f, 2998 Node** parameters, 2999 int parameter_count) { 3000 return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters, 3001 parameter_count); 3002 } 3003 3004 Node* WasmGraphBuilder::GetGlobal(uint32_t index) { 3005 MachineType mem_type = 3006 wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type); 3007 Node* base = nullptr; 3008 Node* offset = nullptr; 3009 GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base, 3010 &offset); 3011 Node* load = SetEffect(graph()->NewNode(mcgraph()->machine()->Load(mem_type), 3012 base, offset, Effect(), Control())); 3013 #if defined(V8_TARGET_BIG_ENDIAN) 3014 load = BuildChangeEndiannessLoad(load, mem_type, 3015 env_->module->globals[index].type); 3016 #endif 3017 return load; 3018 } 3019 3020 Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) { 3021 MachineType mem_type = 3022 wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type); 3023 Node* base = nullptr; 3024 Node* offset = nullptr; 3025 GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base, 3026 &offset); 3027 const Operator* op = mcgraph()->machine()->Store( 3028 StoreRepresentation(mem_type.representation(), kNoWriteBarrier)); 3029 #if defined(V8_TARGET_BIG_ENDIAN) 3030 val = BuildChangeEndiannessStore(val, mem_type.representation(), 3031 env_->module->globals[index].type); 3032 #endif 3033 return SetEffect( 3034 graph()->NewNode(op, base, offset, val, Effect(), Control())); 3035 } 3036 3037 Node* WasmGraphBuilder::CheckBoundsAndAlignment( 3038 uint8_t access_size, Node* index, uint32_t offset, 3039 wasm::WasmCodePosition position) { 3040 // Atomic operations access the memory, need to be bound checked till 3041 // TrapHandlers are enabled on atomic operations 3042 index = 3043 BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck); 3044 Node* effective_address = 3045 graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(offset), 3046 Uint32ToUintptr(index)); 3047 // Unlike regular memory accesses, unaligned memory accesses for atomic 3048 // operations should trap 3049 // Access sizes are in powers of two, calculate mod without using division 3050 Node* cond = 3051 graph()->NewNode(mcgraph()->machine()->WordAnd(), effective_address, 3052 IntPtrConstant(access_size - 1)); 3053 TrapIfFalse(wasm::kTrapUnalignedAccess, 3054 graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond, 3055 mcgraph()->Int32Constant(0)), 3056 position); 3057 return index; 3058 } 3059 3060 Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index, 3061 uint32_t offset, 3062 wasm::WasmCodePosition position, 3063 EnforceBoundsCheck enforce_check) { 3064 DCHECK_LE(1, access_size); 3065 index = Uint32ToUintptr(index); 3066 if (FLAG_wasm_no_bounds_checks) return index; 3067 3068 if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) { 3069 return index; 3070 } 3071 3072 const bool statically_oob = access_size > env_->max_memory_size || 3073 offset > env_->max_memory_size - access_size; 3074 if (statically_oob) { 3075 // The access will be out of bounds, even for the largest memory. 3076 TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position); 3077 return mcgraph()->IntPtrConstant(0); 3078 } 3079 uint64_t end_offset = uint64_t{offset} + access_size - 1u; 3080 Node* end_offset_node = IntPtrConstant(end_offset); 3081 3082 // The accessed memory is [index + offset, index + end_offset]. 3083 // Check that the last read byte (at {index + end_offset}) is in bounds. 3084 // 1) Check that {end_offset < mem_size}. This also ensures that we can safely 3085 // compute {effective_size} as {mem_size - end_offset)}. 3086 // {effective_size} is >= 1 if condition 1) holds. 3087 // 2) Check that {index + end_offset < mem_size} by 3088 // - computing {effective_size} as {mem_size - end_offset} and 3089 // - checking that {index < effective_size}. 3090 3091 auto m = mcgraph()->machine(); 3092 Node* mem_size = instance_cache_->mem_size; 3093 if (end_offset >= env_->min_memory_size) { 3094 // The end offset is larger than the smallest memory. 3095 // Dynamically check the end offset against the dynamic memory size. 3096 Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size); 3097 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); 3098 } else { 3099 // The end offset is smaller than the smallest memory, so only one check is 3100 // required. Check to see if the index is also a constant. 3101 UintPtrMatcher match(index); 3102 if (match.HasValue()) { 3103 uintptr_t index_val = match.Value(); 3104 if (index_val < env_->min_memory_size - end_offset) { 3105 // The input index is a constant and everything is statically within 3106 // bounds of the smallest possible memory. 3107 return index; 3108 } 3109 } 3110 } 3111 3112 // This produces a positive number, since {end_offset < min_size <= mem_size}. 3113 Node* effective_size = 3114 graph()->NewNode(m->IntSub(), mem_size, end_offset_node); 3115 3116 // Introduce the actual bounds check. 3117 Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size); 3118 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); 3119 3120 if (untrusted_code_mitigations_) { 3121 // In the fallthrough case, condition the index with the memory mask. 3122 Node* mem_mask = instance_cache_->mem_mask; 3123 DCHECK_NOT_NULL(mem_mask); 3124 index = graph()->NewNode(m->WordAnd(), index, mem_mask); 3125 } 3126 return index; 3127 } 3128 3129 const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset, 3130 wasm::ValueType type) { 3131 int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type)); 3132 MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type); 3133 if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported( 3134 wasm::ValueTypes::MachineRepresentationFor(type))) { 3135 return mcgraph()->machine()->Load(mach_type); 3136 } 3137 return mcgraph()->machine()->UnalignedLoad(mach_type); 3138 } 3139 3140 const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset, 3141 wasm::ValueType type) { 3142 int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type)); 3143 MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type); 3144 if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) { 3145 StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier); 3146 return mcgraph()->machine()->Store(store_rep); 3147 } 3148 UnalignedStoreRepresentation store_rep(rep); 3149 return mcgraph()->machine()->UnalignedStore(store_rep); 3150 } 3151 3152 Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store, 3153 MachineRepresentation rep, 3154 Node* index, uint32_t offset, 3155 wasm::WasmCodePosition position) { 3156 int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi. 3157 Node* info = graph()->NewNode( 3158 mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign)); 3159 3160 Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(), 3161 Int32Constant(offset), index); 3162 auto store = [&](int offset, MachineRepresentation rep, Node* data) { 3163 SetEffect(graph()->NewNode( 3164 mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)), 3165 info, mcgraph()->Int32Constant(offset), data, Effect(), Control())); 3166 }; 3167 // Store address, is_store, and mem_rep. 3168 store(offsetof(wasm::MemoryTracingInfo, address), 3169 MachineRepresentation::kWord32, address); 3170 store(offsetof(wasm::MemoryTracingInfo, is_store), 3171 MachineRepresentation::kWord8, 3172 mcgraph()->Int32Constant(is_store ? 1 : 0)); 3173 store(offsetof(wasm::MemoryTracingInfo, mem_rep), 3174 MachineRepresentation::kWord8, 3175 mcgraph()->Int32Constant(static_cast<int>(rep))); 3176 3177 Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1); 3178 SetSourcePosition(call, position); 3179 return call; 3180 } 3181 3182 Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype, 3183 Node* index, uint32_t offset, 3184 uint32_t alignment, 3185 wasm::WasmCodePosition position) { 3186 Node* load; 3187 3188 // Wasm semantics throw on OOB. Introduce explicit bounds check and 3189 // conditioning when not using the trap handler. 3190 index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset, 3191 position, kCanOmitBoundsCheck); 3192 3193 if (memtype.representation() == MachineRepresentation::kWord8 || 3194 mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) { 3195 if (use_trap_handler()) { 3196 load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype), 3197 MemBuffer(offset), index, Effect(), Control()); 3198 SetSourcePosition(load, position); 3199 } else { 3200 load = graph()->NewNode(mcgraph()->machine()->Load(memtype), 3201 MemBuffer(offset), index, Effect(), Control()); 3202 } 3203 } else { 3204 // TODO(eholk): Support unaligned loads with trap handlers. 3205 DCHECK(!use_trap_handler()); 3206 load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype), 3207 MemBuffer(offset), index, Effect(), Control()); 3208 } 3209 3210 SetEffect(load); 3211 3212 #if defined(V8_TARGET_BIG_ENDIAN) 3213 load = BuildChangeEndiannessLoad(load, memtype, type); 3214 #endif 3215 3216 if (type == wasm::kWasmI64 && 3217 ElementSizeInBytes(memtype.representation()) < 8) { 3218 // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes. 3219 if (memtype.IsSigned()) { 3220 // sign extend 3221 load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load); 3222 } else { 3223 // zero extend 3224 load = 3225 graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load); 3226 } 3227 } 3228 3229 if (FLAG_wasm_trace_memory) { 3230 TraceMemoryOperation(false, memtype.representation(), index, offset, 3231 position); 3232 } 3233 3234 return load; 3235 } 3236 3237 Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index, 3238 uint32_t offset, uint32_t alignment, Node* val, 3239 wasm::WasmCodePosition position, 3240 wasm::ValueType type) { 3241 Node* store; 3242 3243 index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset, 3244 position, kCanOmitBoundsCheck); 3245 3246 #if defined(V8_TARGET_BIG_ENDIAN) 3247 val = BuildChangeEndiannessStore(val, mem_rep, type); 3248 #endif 3249 3250 if (mem_rep == MachineRepresentation::kWord8 || 3251 mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) { 3252 if (use_trap_handler()) { 3253 store = 3254 graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep), 3255 MemBuffer(offset), index, val, Effect(), Control()); 3256 SetSourcePosition(store, position); 3257 } else { 3258 StoreRepresentation rep(mem_rep, kNoWriteBarrier); 3259 store = 3260 graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset), 3261 index, val, Effect(), Control()); 3262 } 3263 } else { 3264 // TODO(eholk): Support unaligned stores with trap handlers. 3265 DCHECK(!use_trap_handler()); 3266 UnalignedStoreRepresentation rep(mem_rep); 3267 store = 3268 graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep), 3269 MemBuffer(offset), index, val, Effect(), Control()); 3270 } 3271 3272 SetEffect(store); 3273 3274 if (FLAG_wasm_trace_memory) { 3275 TraceMemoryOperation(true, mem_rep, index, offset, position); 3276 } 3277 3278 return store; 3279 } 3280 3281 namespace { 3282 Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) { 3283 switch (rep) { 3284 case MachineRepresentation::kWord8: 3285 case MachineRepresentation::kWord16: 3286 case MachineRepresentation::kWord32: 3287 return mcgraph->Int32Constant(0); 3288 case MachineRepresentation::kWord64: 3289 return mcgraph->Int64Constant(0); 3290 case MachineRepresentation::kFloat32: 3291 return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN()); 3292 case MachineRepresentation::kFloat64: 3293 return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN()); 3294 default: 3295 UNREACHABLE(); 3296 } 3297 } 3298 } // namespace 3299 3300 Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) { 3301 DCHECK_NOT_NULL(instance_cache_); 3302 Node* mem_start = instance_cache_->mem_start; 3303 Node* mem_size = instance_cache_->mem_size; 3304 DCHECK_NOT_NULL(mem_start); 3305 DCHECK_NOT_NULL(mem_size); 3306 3307 // Asm.js semantics are defined in terms of typed arrays, hence OOB 3308 // reads return {undefined} coerced to the result type (0 for integers, NaN 3309 // for float and double). 3310 // Note that we check against the memory size ignoring the size of the 3311 // stored value, which is conservative if misaligned. Technically, asm.js 3312 // should never have misaligned accesses. 3313 index = Uint32ToUintptr(index); 3314 Diamond bounds_check( 3315 graph(), mcgraph()->common(), 3316 graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size), 3317 BranchHint::kTrue); 3318 bounds_check.Chain(Control()); 3319 3320 if (untrusted_code_mitigations_) { 3321 // Condition the index with the memory mask. 3322 Node* mem_mask = instance_cache_->mem_mask; 3323 DCHECK_NOT_NULL(mem_mask); 3324 index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask); 3325 } 3326 3327 Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start, 3328 index, Effect(), bounds_check.if_true); 3329 SetEffect(bounds_check.EffectPhi(load, Effect())); 3330 SetControl(bounds_check.merge); 3331 return bounds_check.Phi(type.representation(), load, 3332 GetAsmJsOOBValue(type.representation(), mcgraph())); 3333 } 3334 3335 Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) { 3336 if (mcgraph()->machine()->Is32()) return node; 3337 // Fold instances of ChangeUint32ToUint64(IntConstant) directly. 3338 Uint32Matcher matcher(node); 3339 if (matcher.HasValue()) { 3340 uintptr_t value = matcher.Value(); 3341 return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value)); 3342 } 3343 return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node); 3344 } 3345 3346 Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index, 3347 Node* val) { 3348 DCHECK_NOT_NULL(instance_cache_); 3349 Node* mem_start = instance_cache_->mem_start; 3350 Node* mem_size = instance_cache_->mem_size; 3351 DCHECK_NOT_NULL(mem_start); 3352 DCHECK_NOT_NULL(mem_size); 3353 3354 // Asm.js semantics are to ignore OOB writes. 3355 // Note that we check against the memory size ignoring the size of the 3356 // stored value, which is conservative if misaligned. Technically, asm.js 3357 // should never have misaligned accesses. 3358 Diamond bounds_check( 3359 graph(), mcgraph()->common(), 3360 graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size), 3361 BranchHint::kTrue); 3362 bounds_check.Chain(Control()); 3363 3364 if (untrusted_code_mitigations_) { 3365 // Condition the index with the memory mask. 3366 Node* mem_mask = instance_cache_->mem_mask; 3367 DCHECK_NOT_NULL(mem_mask); 3368 index = 3369 graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask); 3370 } 3371 3372 index = Uint32ToUintptr(index); 3373 const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation( 3374 type.representation(), WriteBarrierKind::kNoWriteBarrier)); 3375 Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(), 3376 bounds_check.if_true); 3377 SetEffect(bounds_check.EffectPhi(store, Effect())); 3378 SetControl(bounds_check.merge); 3379 return val; 3380 } 3381 3382 void WasmGraphBuilder::PrintDebugName(Node* node) { 3383 PrintF("#%d:%s", node->id(), node->op()->mnemonic()); 3384 } 3385 3386 Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); } 3387 3388 namespace { 3389 Signature<MachineRepresentation>* CreateMachineSignature( 3390 Zone* zone, wasm::FunctionSig* sig) { 3391 Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(), 3392 sig->parameter_count()); 3393 for (auto ret : sig->returns()) { 3394 builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret)); 3395 } 3396 3397 for (auto param : sig->parameters()) { 3398 builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param)); 3399 } 3400 return builder.Build(); 3401 } 3402 } // namespace 3403 3404 void WasmGraphBuilder::LowerInt64() { 3405 if (mcgraph()->machine()->Is64()) return; 3406 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(), 3407 mcgraph()->zone(), 3408 CreateMachineSignature(mcgraph()->zone(), sig_)); 3409 r.LowerGraph(); 3410 } 3411 3412 void WasmGraphBuilder::SimdScalarLoweringForTesting() { 3413 SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_)) 3414 .LowerGraph(); 3415 } 3416 3417 void WasmGraphBuilder::SetSourcePosition(Node* node, 3418 wasm::WasmCodePosition position) { 3419 DCHECK_NE(position, wasm::kNoCodePosition); 3420 if (source_position_table_) 3421 source_position_table_->SetSourcePosition(node, SourcePosition(position)); 3422 } 3423 3424 Node* WasmGraphBuilder::S128Zero() { 3425 has_simd_ = true; 3426 return graph()->NewNode(mcgraph()->machine()->S128Zero()); 3427 } 3428 3429 Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) { 3430 has_simd_ = true; 3431 switch (opcode) { 3432 case wasm::kExprF32x4Splat: 3433 return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]); 3434 case wasm::kExprF32x4SConvertI32x4: 3435 return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(), 3436 inputs[0]); 3437 case wasm::kExprF32x4UConvertI32x4: 3438 return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(), 3439 inputs[0]); 3440 case wasm::kExprF32x4Abs: 3441 return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]); 3442 case wasm::kExprF32x4Neg: 3443 return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]); 3444 case wasm::kExprF32x4RecipApprox: 3445 return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(), 3446 inputs[0]); 3447 case wasm::kExprF32x4RecipSqrtApprox: 3448 return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(), 3449 inputs[0]); 3450 case wasm::kExprF32x4Add: 3451 return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0], 3452 inputs[1]); 3453 case wasm::kExprF32x4AddHoriz: 3454 return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0], 3455 inputs[1]); 3456 case wasm::kExprF32x4Sub: 3457 return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0], 3458 inputs[1]); 3459 case wasm::kExprF32x4Mul: 3460 return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0], 3461 inputs[1]); 3462 case wasm::kExprF32x4Min: 3463 return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0], 3464 inputs[1]); 3465 case wasm::kExprF32x4Max: 3466 return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0], 3467 inputs[1]); 3468 case wasm::kExprF32x4Eq: 3469 return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0], 3470 inputs[1]); 3471 case wasm::kExprF32x4Ne: 3472 return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0], 3473 inputs[1]); 3474 case wasm::kExprF32x4Lt: 3475 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0], 3476 inputs[1]); 3477 case wasm::kExprF32x4Le: 3478 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0], 3479 inputs[1]); 3480 case wasm::kExprF32x4Gt: 3481 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1], 3482 inputs[0]); 3483 case wasm::kExprF32x4Ge: 3484 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1], 3485 inputs[0]); 3486 case wasm::kExprI32x4Splat: 3487 return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]); 3488 case wasm::kExprI32x4SConvertF32x4: 3489 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(), 3490 inputs[0]); 3491 case wasm::kExprI32x4UConvertF32x4: 3492 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(), 3493 inputs[0]); 3494 case wasm::kExprI32x4SConvertI16x8Low: 3495 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(), 3496 inputs[0]); 3497 case wasm::kExprI32x4SConvertI16x8High: 3498 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(), 3499 inputs[0]); 3500 case wasm::kExprI32x4Neg: 3501 return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]); 3502 case wasm::kExprI32x4Add: 3503 return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0], 3504 inputs[1]); 3505 case wasm::kExprI32x4AddHoriz: 3506 return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0], 3507 inputs[1]); 3508 case wasm::kExprI32x4Sub: 3509 return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0], 3510 inputs[1]); 3511 case wasm::kExprI32x4Mul: 3512 return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0], 3513 inputs[1]); 3514 case wasm::kExprI32x4MinS: 3515 return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0], 3516 inputs[1]); 3517 case wasm::kExprI32x4MaxS: 3518 return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0], 3519 inputs[1]); 3520 case wasm::kExprI32x4Eq: 3521 return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0], 3522 inputs[1]); 3523 case wasm::kExprI32x4Ne: 3524 return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0], 3525 inputs[1]); 3526 case wasm::kExprI32x4LtS: 3527 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1], 3528 inputs[0]); 3529 case wasm::kExprI32x4LeS: 3530 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1], 3531 inputs[0]); 3532 case wasm::kExprI32x4GtS: 3533 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0], 3534 inputs[1]); 3535 case wasm::kExprI32x4GeS: 3536 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0], 3537 inputs[1]); 3538 case wasm::kExprI32x4UConvertI16x8Low: 3539 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(), 3540 inputs[0]); 3541 case wasm::kExprI32x4UConvertI16x8High: 3542 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(), 3543 inputs[0]); 3544 case wasm::kExprI32x4MinU: 3545 return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0], 3546 inputs[1]); 3547 case wasm::kExprI32x4MaxU: 3548 return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0], 3549 inputs[1]); 3550 case wasm::kExprI32x4LtU: 3551 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1], 3552 inputs[0]); 3553 case wasm::kExprI32x4LeU: 3554 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1], 3555 inputs[0]); 3556 case wasm::kExprI32x4GtU: 3557 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0], 3558 inputs[1]); 3559 case wasm::kExprI32x4GeU: 3560 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0], 3561 inputs[1]); 3562 case wasm::kExprI16x8Splat: 3563 return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]); 3564 case wasm::kExprI16x8SConvertI8x16Low: 3565 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(), 3566 inputs[0]); 3567 case wasm::kExprI16x8SConvertI8x16High: 3568 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(), 3569 inputs[0]); 3570 case wasm::kExprI16x8Neg: 3571 return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]); 3572 case wasm::kExprI16x8SConvertI32x4: 3573 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(), 3574 inputs[0], inputs[1]); 3575 case wasm::kExprI16x8Add: 3576 return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0], 3577 inputs[1]); 3578 case wasm::kExprI16x8AddSaturateS: 3579 return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(), 3580 inputs[0], inputs[1]); 3581 case wasm::kExprI16x8AddHoriz: 3582 return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0], 3583 inputs[1]); 3584 case wasm::kExprI16x8Sub: 3585 return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0], 3586 inputs[1]); 3587 case wasm::kExprI16x8SubSaturateS: 3588 return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(), 3589 inputs[0], inputs[1]); 3590 case wasm::kExprI16x8Mul: 3591 return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0], 3592 inputs[1]); 3593 case wasm::kExprI16x8MinS: 3594 return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0], 3595 inputs[1]); 3596 case wasm::kExprI16x8MaxS: 3597 return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0], 3598 inputs[1]); 3599 case wasm::kExprI16x8Eq: 3600 return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0], 3601 inputs[1]); 3602 case wasm::kExprI16x8Ne: 3603 return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0], 3604 inputs[1]); 3605 case wasm::kExprI16x8LtS: 3606 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1], 3607 inputs[0]); 3608 case wasm::kExprI16x8LeS: 3609 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1], 3610 inputs[0]); 3611 case wasm::kExprI16x8GtS: 3612 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0], 3613 inputs[1]); 3614 case wasm::kExprI16x8GeS: 3615 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0], 3616 inputs[1]); 3617 case wasm::kExprI16x8UConvertI8x16Low: 3618 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(), 3619 inputs[0]); 3620 case wasm::kExprI16x8UConvertI8x16High: 3621 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(), 3622 inputs[0]); 3623 case wasm::kExprI16x8UConvertI32x4: 3624 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(), 3625 inputs[0], inputs[1]); 3626 case wasm::kExprI16x8AddSaturateU: 3627 return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(), 3628 inputs[0], inputs[1]); 3629 case wasm::kExprI16x8SubSaturateU: 3630 return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(), 3631 inputs[0], inputs[1]); 3632 case wasm::kExprI16x8MinU: 3633 return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0], 3634 inputs[1]); 3635 case wasm::kExprI16x8MaxU: 3636 return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0], 3637 inputs[1]); 3638 case wasm::kExprI16x8LtU: 3639 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1], 3640 inputs[0]); 3641 case wasm::kExprI16x8LeU: 3642 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1], 3643 inputs[0]); 3644 case wasm::kExprI16x8GtU: 3645 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0], 3646 inputs[1]); 3647 case wasm::kExprI16x8GeU: 3648 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0], 3649 inputs[1]); 3650 case wasm::kExprI8x16Splat: 3651 return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]); 3652 case wasm::kExprI8x16Neg: 3653 return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]); 3654 case wasm::kExprI8x16SConvertI16x8: 3655 return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(), 3656 inputs[0], inputs[1]); 3657 case wasm::kExprI8x16Add: 3658 return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0], 3659 inputs[1]); 3660 case wasm::kExprI8x16AddSaturateS: 3661 return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(), 3662 inputs[0], inputs[1]); 3663 case wasm::kExprI8x16Sub: 3664 return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0], 3665 inputs[1]); 3666 case wasm::kExprI8x16SubSaturateS: 3667 return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(), 3668 inputs[0], inputs[1]); 3669 case wasm::kExprI8x16Mul: 3670 return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0], 3671 inputs[1]); 3672 case wasm::kExprI8x16MinS: 3673 return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0], 3674 inputs[1]); 3675 case wasm::kExprI8x16MaxS: 3676 return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0], 3677 inputs[1]); 3678 case wasm::kExprI8x16Eq: 3679 return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0], 3680 inputs[1]); 3681 case wasm::kExprI8x16Ne: 3682 return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0], 3683 inputs[1]); 3684 case wasm::kExprI8x16LtS: 3685 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1], 3686 inputs[0]); 3687 case wasm::kExprI8x16LeS: 3688 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1], 3689 inputs[0]); 3690 case wasm::kExprI8x16GtS: 3691 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0], 3692 inputs[1]); 3693 case wasm::kExprI8x16GeS: 3694 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0], 3695 inputs[1]); 3696 case wasm::kExprI8x16UConvertI16x8: 3697 return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(), 3698 inputs[0], inputs[1]); 3699 case wasm::kExprI8x16AddSaturateU: 3700 return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(), 3701 inputs[0], inputs[1]); 3702 case wasm::kExprI8x16SubSaturateU: 3703 return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(), 3704 inputs[0], inputs[1]); 3705 case wasm::kExprI8x16MinU: 3706 return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0], 3707 inputs[1]); 3708 case wasm::kExprI8x16MaxU: 3709 return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0], 3710 inputs[1]); 3711 case wasm::kExprI8x16LtU: 3712 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1], 3713 inputs[0]); 3714 case wasm::kExprI8x16LeU: 3715 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1], 3716 inputs[0]); 3717 case wasm::kExprI8x16GtU: 3718 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0], 3719 inputs[1]); 3720 case wasm::kExprI8x16GeU: 3721 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0], 3722 inputs[1]); 3723 case wasm::kExprS128And: 3724 return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0], 3725 inputs[1]); 3726 case wasm::kExprS128Or: 3727 return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0], 3728 inputs[1]); 3729 case wasm::kExprS128Xor: 3730 return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0], 3731 inputs[1]); 3732 case wasm::kExprS128Not: 3733 return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]); 3734 case wasm::kExprS128Select: 3735 return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[0], 3736 inputs[1], inputs[2]); 3737 case wasm::kExprS1x4AnyTrue: 3738 return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]); 3739 case wasm::kExprS1x4AllTrue: 3740 return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]); 3741 case wasm::kExprS1x8AnyTrue: 3742 return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]); 3743 case wasm::kExprS1x8AllTrue: 3744 return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]); 3745 case wasm::kExprS1x16AnyTrue: 3746 return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]); 3747 case wasm::kExprS1x16AllTrue: 3748 return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]); 3749 default: 3750 FATAL_UNSUPPORTED_OPCODE(opcode); 3751 } 3752 } 3753 3754 Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane, 3755 Node* const* inputs) { 3756 has_simd_ = true; 3757 switch (opcode) { 3758 case wasm::kExprF32x4ExtractLane: 3759 return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane), 3760 inputs[0]); 3761 case wasm::kExprF32x4ReplaceLane: 3762 return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane), 3763 inputs[0], inputs[1]); 3764 case wasm::kExprI32x4ExtractLane: 3765 return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane), 3766 inputs[0]); 3767 case wasm::kExprI32x4ReplaceLane: 3768 return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane), 3769 inputs[0], inputs[1]); 3770 case wasm::kExprI16x8ExtractLane: 3771 return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane), 3772 inputs[0]); 3773 case wasm::kExprI16x8ReplaceLane: 3774 return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane), 3775 inputs[0], inputs[1]); 3776 case wasm::kExprI8x16ExtractLane: 3777 return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane), 3778 inputs[0]); 3779 case wasm::kExprI8x16ReplaceLane: 3780 return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane), 3781 inputs[0], inputs[1]); 3782 default: 3783 FATAL_UNSUPPORTED_OPCODE(opcode); 3784 } 3785 } 3786 3787 Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift, 3788 Node* const* inputs) { 3789 has_simd_ = true; 3790 switch (opcode) { 3791 case wasm::kExprI32x4Shl: 3792 return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]); 3793 case wasm::kExprI32x4ShrS: 3794 return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift), 3795 inputs[0]); 3796 case wasm::kExprI32x4ShrU: 3797 return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift), 3798 inputs[0]); 3799 case wasm::kExprI16x8Shl: 3800 return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]); 3801 case wasm::kExprI16x8ShrS: 3802 return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift), 3803 inputs[0]); 3804 case wasm::kExprI16x8ShrU: 3805 return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift), 3806 inputs[0]); 3807 case wasm::kExprI8x16Shl: 3808 return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]); 3809 case wasm::kExprI8x16ShrS: 3810 return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift), 3811 inputs[0]); 3812 case wasm::kExprI8x16ShrU: 3813 return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift), 3814 inputs[0]); 3815 default: 3816 FATAL_UNSUPPORTED_OPCODE(opcode); 3817 } 3818 } 3819 3820 Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16], 3821 Node* const* inputs) { 3822 has_simd_ = true; 3823 return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle), 3824 inputs[0], inputs[1]); 3825 } 3826 3827 #define ATOMIC_BINOP_LIST(V) \ 3828 V(I32AtomicAdd, Add, Uint32, Word32) \ 3829 V(I64AtomicAdd, Add, Uint64, Word64) \ 3830 V(I32AtomicAdd8U, Add, Uint8, Word32) \ 3831 V(I32AtomicAdd16U, Add, Uint16, Word32) \ 3832 V(I64AtomicAdd8U, Add, Uint8, Word64) \ 3833 V(I64AtomicAdd16U, Add, Uint16, Word64) \ 3834 V(I64AtomicAdd32U, Add, Uint32, Word64) \ 3835 V(I32AtomicSub, Sub, Uint32, Word32) \ 3836 V(I64AtomicSub, Sub, Uint64, Word64) \ 3837 V(I32AtomicSub8U, Sub, Uint8, Word32) \ 3838 V(I32AtomicSub16U, Sub, Uint16, Word32) \ 3839 V(I64AtomicSub8U, Sub, Uint8, Word64) \ 3840 V(I64AtomicSub16U, Sub, Uint16, Word64) \ 3841 V(I64AtomicSub32U, Sub, Uint32, Word64) \ 3842 V(I32AtomicAnd, And, Uint32, Word32) \ 3843 V(I64AtomicAnd, And, Uint64, Word64) \ 3844 V(I32AtomicAnd8U, And, Uint8, Word32) \ 3845 V(I64AtomicAnd16U, And, Uint16, Word64) \ 3846 V(I32AtomicAnd16U, And, Uint16, Word32) \ 3847 V(I64AtomicAnd8U, And, Uint8, Word64) \ 3848 V(I64AtomicAnd32U, And, Uint32, Word64) \ 3849 V(I32AtomicOr, Or, Uint32, Word32) \ 3850 V(I64AtomicOr, Or, Uint64, Word64) \ 3851 V(I32AtomicOr8U, Or, Uint8, Word32) \ 3852 V(I32AtomicOr16U, Or, Uint16, Word32) \ 3853 V(I64AtomicOr8U, Or, Uint8, Word64) \ 3854 V(I64AtomicOr16U, Or, Uint16, Word64) \ 3855 V(I64AtomicOr32U, Or, Uint32, Word64) \ 3856 V(I32AtomicXor, Xor, Uint32, Word32) \ 3857 V(I64AtomicXor, Xor, Uint64, Word64) \ 3858 V(I32AtomicXor8U, Xor, Uint8, Word32) \ 3859 V(I32AtomicXor16U, Xor, Uint16, Word32) \ 3860 V(I64AtomicXor8U, Xor, Uint8, Word64) \ 3861 V(I64AtomicXor16U, Xor, Uint16, Word64) \ 3862 V(I64AtomicXor32U, Xor, Uint32, Word64) \ 3863 V(I32AtomicExchange, Exchange, Uint32, Word32) \ 3864 V(I64AtomicExchange, Exchange, Uint64, Word64) \ 3865 V(I32AtomicExchange8U, Exchange, Uint8, Word32) \ 3866 V(I32AtomicExchange16U, Exchange, Uint16, Word32) \ 3867 V(I64AtomicExchange8U, Exchange, Uint8, Word64) \ 3868 V(I64AtomicExchange16U, Exchange, Uint16, Word64) \ 3869 V(I64AtomicExchange32U, Exchange, Uint32, Word64) 3870 3871 #define ATOMIC_CMP_EXCHG_LIST(V) \ 3872 V(I32AtomicCompareExchange, Uint32, Word32) \ 3873 V(I64AtomicCompareExchange, Uint64, Word64) \ 3874 V(I32AtomicCompareExchange8U, Uint8, Word32) \ 3875 V(I32AtomicCompareExchange16U, Uint16, Word32) \ 3876 V(I64AtomicCompareExchange8U, Uint8, Word64) \ 3877 V(I64AtomicCompareExchange16U, Uint16, Word64) \ 3878 V(I64AtomicCompareExchange32U, Uint32, Word64) 3879 3880 #define ATOMIC_LOAD_LIST(V) \ 3881 V(I32AtomicLoad, Uint32, Word32) \ 3882 V(I64AtomicLoad, Uint64, Word64) \ 3883 V(I32AtomicLoad8U, Uint8, Word32) \ 3884 V(I32AtomicLoad16U, Uint16, Word32) \ 3885 V(I64AtomicLoad8U, Uint8, Word64) \ 3886 V(I64AtomicLoad16U, Uint16, Word64) \ 3887 V(I64AtomicLoad32U, Uint32, Word64) 3888 3889 #define ATOMIC_STORE_LIST(V) \ 3890 V(I32AtomicStore, Uint32, kWord32, Word32) \ 3891 V(I64AtomicStore, Uint64, kWord64, Word64) \ 3892 V(I32AtomicStore8U, Uint8, kWord8, Word32) \ 3893 V(I32AtomicStore16U, Uint16, kWord16, Word32) \ 3894 V(I64AtomicStore8U, Uint8, kWord8, Word64) \ 3895 V(I64AtomicStore16U, Uint16, kWord16, Word64) \ 3896 V(I64AtomicStore32U, Uint32, kWord32, Word64) 3897 3898 Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs, 3899 uint32_t alignment, uint32_t offset, 3900 wasm::WasmCodePosition position) { 3901 Node* node; 3902 switch (opcode) { 3903 #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \ 3904 case wasm::kExpr##Name: { \ 3905 Node* index = CheckBoundsAndAlignment( \ 3906 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \ 3907 position); \ 3908 node = graph()->NewNode( \ 3909 mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \ 3910 MemBuffer(offset), index, inputs[1], Effect(), Control()); \ 3911 break; \ 3912 } 3913 ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP) 3914 #undef BUILD_ATOMIC_BINOP 3915 3916 #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \ 3917 case wasm::kExpr##Name: { \ 3918 Node* index = CheckBoundsAndAlignment( \ 3919 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \ 3920 position); \ 3921 node = graph()->NewNode( \ 3922 mcgraph()->machine()->Prefix##AtomicCompareExchange( \ 3923 MachineType::Type()), \ 3924 MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \ 3925 break; \ 3926 } 3927 ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG) 3928 #undef BUILD_ATOMIC_CMP_EXCHG 3929 3930 #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \ 3931 case wasm::kExpr##Name: { \ 3932 Node* index = CheckBoundsAndAlignment( \ 3933 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \ 3934 position); \ 3935 node = graph()->NewNode( \ 3936 mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \ 3937 MemBuffer(offset), index, Effect(), Control()); \ 3938 break; \ 3939 } 3940 ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP) 3941 #undef BUILD_ATOMIC_LOAD_OP 3942 3943 #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \ 3944 case wasm::kExpr##Name: { \ 3945 Node* index = CheckBoundsAndAlignment( \ 3946 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \ 3947 position); \ 3948 node = graph()->NewNode( \ 3949 mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \ 3950 MemBuffer(offset), index, inputs[1], Effect(), Control()); \ 3951 break; \ 3952 } 3953 ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP) 3954 #undef BUILD_ATOMIC_STORE_OP 3955 default: 3956 FATAL_UNSUPPORTED_OPCODE(opcode); 3957 } 3958 return SetEffect(node); 3959 } 3960 3961 #undef ATOMIC_BINOP_LIST 3962 #undef ATOMIC_CMP_EXCHG_LIST 3963 #undef ATOMIC_LOAD_LIST 3964 #undef ATOMIC_STORE_LIST 3965 3966 class WasmDecorator final : public GraphDecorator { 3967 public: 3968 explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder) 3969 : origins_(origins), decoder_(decoder) {} 3970 3971 void Decorate(Node* node) final { 3972 origins_->SetNodeOrigin( 3973 node, NodeOrigin("wasm graph creation", "n/a", 3974 NodeOrigin::kWasmBytecode, decoder_->position())); 3975 } 3976 3977 private: 3978 compiler::NodeOriginTable* origins_; 3979 wasm::Decoder* decoder_; 3980 }; 3981 3982 void WasmGraphBuilder::AddBytecodePositionDecorator( 3983 NodeOriginTable* node_origins, wasm::Decoder* decoder) { 3984 DCHECK_NULL(decorator_); 3985 decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder); 3986 graph()->AddDecorator(decorator_); 3987 } 3988 3989 void WasmGraphBuilder::RemoveBytecodePositionDecorator() { 3990 DCHECK_NOT_NULL(decorator_); 3991 graph()->RemoveDecorator(decorator_); 3992 decorator_ = nullptr; 3993 } 3994 3995 namespace { 3996 bool must_record_function_compilation(Isolate* isolate) { 3997 return isolate->logger()->is_listening_to_code_events() || 3998 isolate->is_profiling(); 3999 } 4000 4001 PRINTF_FORMAT(4, 5) 4002 void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag, 4003 Isolate* isolate, Handle<Code> code, 4004 const char* format, ...) { 4005 DCHECK(must_record_function_compilation(isolate)); 4006 4007 ScopedVector<char> buffer(128); 4008 va_list arguments; 4009 va_start(arguments, format); 4010 int len = VSNPrintF(buffer, format, arguments); 4011 CHECK_LT(0, len); 4012 va_end(arguments); 4013 Handle<String> name_str = 4014 isolate->factory()->NewStringFromAsciiChecked(buffer.start()); 4015 PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str)); 4016 } 4017 4018 class WasmWrapperGraphBuilder : public WasmGraphBuilder { 4019 public: 4020 WasmWrapperGraphBuilder(Zone* zone, wasm::ModuleEnv* env, JSGraph* jsgraph, 4021 wasm::FunctionSig* sig, 4022 compiler::SourcePositionTable* spt, 4023 StubCallMode stub_mode) 4024 : WasmGraphBuilder(env, zone, jsgraph, sig, spt), 4025 isolate_(jsgraph->isolate()), 4026 jsgraph_(jsgraph), 4027 stub_mode_(stub_mode) {} 4028 4029 Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) { 4030 MachineOperatorBuilder* machine = mcgraph()->machine(); 4031 CommonOperatorBuilder* common = mcgraph()->common(); 4032 Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) 4033 ? mcgraph()->RelocatableIntPtrConstant( 4034 wasm::WasmCode::kWasmAllocateHeapNumber, 4035 RelocInfo::WASM_STUB_CALL) 4036 : jsgraph()->HeapConstant( 4037 BUILTIN_CODE(isolate_, AllocateHeapNumber)); 4038 if (!allocate_heap_number_operator_.is_set()) { 4039 auto call_descriptor = Linkage::GetStubCallDescriptor( 4040 mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0, 4041 CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_); 4042 allocate_heap_number_operator_.set(common->Call(call_descriptor)); 4043 } 4044 Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(), 4045 target, Effect(), control); 4046 SetEffect( 4047 graph()->NewNode(machine->Store(StoreRepresentation( 4048 MachineRepresentation::kFloat64, kNoWriteBarrier)), 4049 heap_number, BuildHeapNumberValueIndexConstant(), 4050 value, heap_number, control)); 4051 return heap_number; 4052 } 4053 4054 Node* BuildChangeSmiToFloat64(Node* value) { 4055 return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(), 4056 BuildChangeSmiToInt32(value)); 4057 } 4058 4059 Node* BuildTestHeapObject(Node* value) { 4060 return graph()->NewNode(mcgraph()->machine()->WordAnd(), value, 4061 mcgraph()->IntPtrConstant(kHeapObjectTag)); 4062 } 4063 4064 Node* BuildLoadHeapNumberValue(Node* value) { 4065 return SetEffect(graph()->NewNode( 4066 mcgraph()->machine()->Load(MachineType::Float64()), value, 4067 BuildHeapNumberValueIndexConstant(), Effect(), Control())); 4068 } 4069 4070 Node* BuildHeapNumberValueIndexConstant() { 4071 return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag); 4072 } 4073 4074 Node* BuildChangeInt32ToTagged(Node* value) { 4075 MachineOperatorBuilder* machine = mcgraph()->machine(); 4076 CommonOperatorBuilder* common = mcgraph()->common(); 4077 4078 if (SmiValuesAre32Bits()) { 4079 return BuildChangeInt32ToSmi(value); 4080 } 4081 DCHECK(SmiValuesAre31Bits()); 4082 4083 Node* effect = Effect(); 4084 Node* control = Control(); 4085 Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value, 4086 graph()->start()); 4087 4088 Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start()); 4089 Node* branch = 4090 graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control); 4091 4092 Node* if_true = graph()->NewNode(common->IfTrue(), branch); 4093 Node* vtrue = BuildAllocateHeapNumberWithValue( 4094 graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true); 4095 Node* etrue = Effect(); 4096 4097 Node* if_false = graph()->NewNode(common->IfFalse(), branch); 4098 Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false); 4099 vfalse = BuildChangeInt32ToIntPtr(vfalse); 4100 4101 Node* merge = 4102 SetControl(graph()->NewNode(common->Merge(2), if_true, if_false)); 4103 SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge)); 4104 return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), 4105 vtrue, vfalse, merge); 4106 } 4107 4108 Node* BuildChangeFloat64ToTagged(Node* value) { 4109 MachineOperatorBuilder* machine = mcgraph()->machine(); 4110 CommonOperatorBuilder* common = mcgraph()->common(); 4111 4112 // Check several conditions: 4113 // i32? 4114 // true: zero? 4115 // true: negative? 4116 // true: box 4117 // false: potentially Smi 4118 // false: potentially Smi 4119 // false: box 4120 // For potential Smi values, depending on whether Smis are 31 or 32 bit, we 4121 // still need to check whether the value fits in a Smi. 4122 4123 Node* effect = Effect(); 4124 Node* control = Control(); 4125 Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value); 4126 Node* check_i32 = graph()->NewNode( 4127 machine->Float64Equal(), value, 4128 graph()->NewNode(machine->ChangeInt32ToFloat64(), value32)); 4129 Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control); 4130 4131 Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32); 4132 Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32); 4133 4134 // We only need to check for -0 if the {value} can potentially contain -0. 4135 Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32, 4136 mcgraph()->Int32Constant(0)); 4137 Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse), 4138 check_zero, if_i32); 4139 4140 Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero); 4141 Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero); 4142 4143 // In case of 0, we need to check the high bits for the IEEE -0 pattern. 4144 Node* check_negative = graph()->NewNode( 4145 machine->Int32LessThan(), 4146 graph()->NewNode(machine->Float64ExtractHighWord32(), value), 4147 mcgraph()->Int32Constant(0)); 4148 Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse), 4149 check_negative, if_zero); 4150 4151 Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative); 4152 Node* if_not_negative = 4153 graph()->NewNode(common->IfFalse(), branch_negative); 4154 4155 // We need to create a box for negative 0. 4156 Node* if_smi = 4157 graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative); 4158 Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative); 4159 4160 // On 64-bit machines we can just wrap the 32-bit integer in a smi, for 4161 // 32-bit machines we need to deal with potential overflow and fallback to 4162 // boxing. 4163 Node* vsmi; 4164 if (SmiValuesAre32Bits()) { 4165 vsmi = BuildChangeInt32ToSmi(value32); 4166 } else { 4167 DCHECK(SmiValuesAre31Bits()); 4168 Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32, 4169 value32, if_smi); 4170 4171 Node* check_ovf = 4172 graph()->NewNode(common->Projection(1), smi_tag, if_smi); 4173 Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse), 4174 check_ovf, if_smi); 4175 4176 Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf); 4177 if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box); 4178 4179 if_smi = graph()->NewNode(common->IfFalse(), branch_ovf); 4180 vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi); 4181 vsmi = BuildChangeInt32ToIntPtr(vsmi); 4182 } 4183 4184 // Allocate the box for the {value}. 4185 Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box); 4186 Node* ebox = Effect(); 4187 4188 Node* merge = 4189 SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box)); 4190 SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge)); 4191 return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), 4192 vsmi, vbox, merge); 4193 } 4194 4195 int AddArgumentNodes(Node** args, int pos, int param_count, 4196 wasm::FunctionSig* sig) { 4197 // Convert wasm numbers to JS values. 4198 for (int i = 0; i < param_count; ++i) { 4199 Node* param = 4200 Param(i + 1); // Start from index 1 to drop the instance_node. 4201 args[pos++] = ToJS(param, sig->GetParam(i)); 4202 } 4203 return pos; 4204 } 4205 4206 Node* BuildJavaScriptToNumber(Node* node, Node* js_context) { 4207 auto call_descriptor = Linkage::GetStubCallDescriptor( 4208 mcgraph()->zone(), TypeConversionDescriptor{}, 0, 4209 CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_); 4210 Node* stub_code = 4211 (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) 4212 ? mcgraph()->RelocatableIntPtrConstant( 4213 wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL) 4214 : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber)); 4215 4216 Node* result = SetEffect( 4217 graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code, 4218 node, js_context, Effect(), Control())); 4219 4220 SetSourcePosition(result, 1); 4221 4222 return result; 4223 } 4224 4225 Node* BuildChangeTaggedToFloat64(Node* value) { 4226 MachineOperatorBuilder* machine = mcgraph()->machine(); 4227 CommonOperatorBuilder* common = mcgraph()->common(); 4228 4229 // Implement the following decision tree: 4230 // heap object? 4231 // true: undefined? 4232 // true: f64 const 4233 // false: load heap number value 4234 // false: smi to float64 4235 4236 Node* check_heap_object = BuildTestHeapObject(value); 4237 Diamond is_heap_object(graph(), common, check_heap_object, 4238 BranchHint::kFalse); 4239 is_heap_object.Chain(Control()); 4240 4241 SetControl(is_heap_object.if_true); 4242 Node* orig_effect = Effect(); 4243 4244 Node* undefined_node = 4245 LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer()); 4246 Node* check_undefined = 4247 graph()->NewNode(machine->WordEqual(), value, undefined_node); 4248 Node* effect_tagged = Effect(); 4249 4250 Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse); 4251 is_undefined.Nest(is_heap_object, true); 4252 4253 SetControl(is_undefined.if_false); 4254 Node* vheap_number = BuildLoadHeapNumberValue(value); 4255 Node* effect_undefined = Effect(); 4256 4257 SetControl(is_undefined.merge); 4258 Node* vundefined = 4259 mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN()); 4260 Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64, 4261 vundefined, vheap_number); 4262 4263 effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined); 4264 4265 // If input is Smi: just convert to float64. 4266 Node* vfrom_smi = BuildChangeSmiToFloat64(value); 4267 4268 SetControl(is_heap_object.merge); 4269 SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect)); 4270 return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged, 4271 vfrom_smi); 4272 } 4273 4274 Node* ToJS(Node* node, wasm::ValueType type) { 4275 switch (type) { 4276 case wasm::kWasmI32: 4277 return BuildChangeInt32ToTagged(node); 4278 case wasm::kWasmS128: 4279 case wasm::kWasmI64: 4280 UNREACHABLE(); 4281 case wasm::kWasmF32: 4282 node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(), 4283 node); 4284 return BuildChangeFloat64ToTagged(node); 4285 case wasm::kWasmF64: 4286 return BuildChangeFloat64ToTagged(node); 4287 case wasm::kWasmAnyRef: 4288 return node; 4289 default: 4290 UNREACHABLE(); 4291 } 4292 } 4293 4294 Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) { 4295 DCHECK_NE(wasm::kWasmStmt, type); 4296 4297 // The parameter is of type AnyRef, we take it as is. 4298 if (type == wasm::kWasmAnyRef) { 4299 return node; 4300 } 4301 4302 // Do a JavaScript ToNumber. 4303 Node* num = BuildJavaScriptToNumber(node, js_context); 4304 4305 // Change representation. 4306 num = BuildChangeTaggedToFloat64(num); 4307 4308 switch (type) { 4309 case wasm::kWasmI32: { 4310 num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(), 4311 num); 4312 break; 4313 } 4314 case wasm::kWasmS128: 4315 case wasm::kWasmI64: 4316 UNREACHABLE(); 4317 case wasm::kWasmF32: 4318 num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(), 4319 num); 4320 break; 4321 case wasm::kWasmF64: 4322 break; 4323 default: 4324 UNREACHABLE(); 4325 } 4326 return num; 4327 } 4328 4329 void BuildModifyThreadInWasmFlag(bool new_value) { 4330 if (!trap_handler::IsTrapHandlerEnabled()) return; 4331 Node* thread_in_wasm_flag_address_address = 4332 graph()->NewNode(mcgraph()->common()->ExternalConstant( 4333 ExternalReference::wasm_thread_in_wasm_flag_address_address( 4334 isolate_))); 4335 Node* thread_in_wasm_flag_address = SetEffect(graph()->NewNode( 4336 mcgraph()->machine()->Load(LoadRepresentation(MachineType::Pointer())), 4337 thread_in_wasm_flag_address_address, mcgraph()->Int32Constant(0), 4338 Effect(), Control())); 4339 SetEffect(graph()->NewNode( 4340 mcgraph()->machine()->Store(StoreRepresentation( 4341 MachineRepresentation::kWord32, kNoWriteBarrier)), 4342 thread_in_wasm_flag_address, mcgraph()->Int32Constant(0), 4343 mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control())); 4344 } 4345 4346 Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) { 4347 Node* shared = SetEffect(graph()->NewNode( 4348 jsgraph()->machine()->Load(MachineType::AnyTagged()), closure, 4349 jsgraph()->Int32Constant(JSFunction::kSharedFunctionInfoOffset - 4350 kHeapObjectTag), 4351 Effect(), Control())); 4352 return SetEffect(graph()->NewNode( 4353 jsgraph()->machine()->Load(MachineType::AnyTagged()), shared, 4354 jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset - 4355 kHeapObjectTag), 4356 Effect(), Control())); 4357 } 4358 4359 Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) { 4360 return SetEffect(graph()->NewNode( 4361 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data, 4362 jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset - 4363 kHeapObjectTag), 4364 Effect(), Control())); 4365 } 4366 4367 Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) { 4368 Node* function_index_smi = SetEffect(graph()->NewNode( 4369 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data, 4370 jsgraph()->Int32Constant( 4371 WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag), 4372 Effect(), Control())); 4373 Node* function_index = BuildChangeSmiToInt32(function_index_smi); 4374 return function_index; 4375 } 4376 4377 Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) { 4378 Node* jump_table_offset_smi = SetEffect(graph()->NewNode( 4379 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data, 4380 jsgraph()->Int32Constant( 4381 WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag), 4382 Effect(), Control())); 4383 Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi); 4384 return jump_table_offset; 4385 } 4386 4387 void BuildJSToWasmWrapper(bool is_import) { 4388 const int wasm_count = static_cast<int>(sig_->parameter_count()); 4389 4390 // Build the start and the JS parameter nodes. 4391 SetEffect(SetControl(Start(wasm_count + 5))); 4392 4393 // Create the js_closure and js_context parameters. 4394 Node* js_closure = 4395 graph()->NewNode(jsgraph()->common()->Parameter( 4396 Linkage::kJSCallClosureParamIndex, "%closure"), 4397 graph()->start()); 4398 Node* js_context = graph()->NewNode( 4399 mcgraph()->common()->Parameter( 4400 Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"), 4401 graph()->start()); 4402 4403 // Create the instance_node node to pass as parameter. It is loaded from 4404 // an actual reference to an instance or a placeholder reference, 4405 // called {WasmExportedFunction} via the {WasmExportedFunctionData} 4406 // structure. 4407 Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure); 4408 instance_node_.set( 4409 BuildLoadInstanceFromExportedFunctionData(function_data)); 4410 4411 if (!wasm::IsJSCompatibleSignature(sig_)) { 4412 // Throw a TypeError. Use the js_context of the calling javascript 4413 // function (passed as a parameter), such that the generated code is 4414 // js_context independent. 4415 BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context, 4416 nullptr, 0); 4417 Return(jsgraph()->SmiConstant(0)); 4418 return; 4419 } 4420 4421 const int args_count = wasm_count + 1; // +1 for wasm_code. 4422 Node** args = Buffer(args_count); 4423 Node** rets; 4424 4425 // Convert JS parameters to wasm numbers. 4426 for (int i = 0; i < wasm_count; ++i) { 4427 Node* param = Param(i + 1); 4428 Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i)); 4429 args[i + 1] = wasm_param; 4430 } 4431 4432 // Set the ThreadInWasm flag before we do the actual call. 4433 BuildModifyThreadInWasmFlag(true); 4434 4435 if (is_import) { 4436 // Call to an imported function. 4437 // Load function index from {WasmExportedFunctionData}. 4438 Node* function_index = 4439 BuildLoadFunctionIndexFromExportedFunctionData(function_data); 4440 BuildImportWasmCall(sig_, args, &rets, wasm::kNoCodePosition, 4441 function_index); 4442 } else { 4443 // Call to a wasm function defined in this module. 4444 // The call target is the jump table slot for that function. 4445 Node* jump_table_start = 4446 LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer()); 4447 Node* jump_table_offset = 4448 BuildLoadJumpTableOffsetFromExportedFunctionData(function_data); 4449 Node* jump_table_slot = graph()->NewNode( 4450 mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset); 4451 args[0] = jump_table_slot; 4452 4453 BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr, 4454 kNoRetpoline); 4455 } 4456 4457 // Clear the ThreadInWasm flag. 4458 BuildModifyThreadInWasmFlag(false); 4459 4460 Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant() 4461 : ToJS(rets[0], sig_->GetReturn()); 4462 Return(jsval); 4463 } 4464 4465 bool BuildWasmToJSWrapper(Handle<JSReceiver> target, int index) { 4466 DCHECK(target->IsCallable()); 4467 4468 int wasm_count = static_cast<int>(sig_->parameter_count()); 4469 4470 // Build the start and the parameter nodes. 4471 SetEffect(SetControl(Start(wasm_count + 3))); 4472 4473 // Create the instance_node from the passed parameter. 4474 instance_node_.set(Param(wasm::kWasmInstanceParameterIndex)); 4475 4476 Node* callables_node = LOAD_INSTANCE_FIELD(ImportedFunctionCallables, 4477 MachineType::TaggedPointer()); 4478 Node* callable_node = LOAD_FIXED_ARRAY_SLOT(callables_node, index); 4479 Node* undefined_node = 4480 LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer()); 4481 Node* native_context = 4482 LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer()); 4483 4484 if (!wasm::IsJSCompatibleSignature(sig_)) { 4485 // Throw a TypeError. 4486 BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, 4487 native_context, nullptr, 0); 4488 // We don't need to return a value here, as the runtime call will not 4489 // return anyway (the c entry stub will trigger stack unwinding). 4490 ReturnVoid(); 4491 return false; 4492 } 4493 4494 CallDescriptor* call_descriptor; 4495 Node** args = Buffer(wasm_count + 9); 4496 Node* call = nullptr; 4497 4498 BuildModifyThreadInWasmFlag(false); 4499 4500 if (target->IsJSFunction()) { 4501 Handle<JSFunction> function = Handle<JSFunction>::cast(target); 4502 FieldAccess field_access = AccessBuilder::ForJSFunctionContext(); 4503 Node* function_context = SetEffect(graph()->NewNode( 4504 mcgraph()->machine()->Load(MachineType::TaggedPointer()), 4505 callable_node, 4506 mcgraph()->Int32Constant(field_access.offset - field_access.tag()), 4507 Effect(), Control())); 4508 4509 if (!IsClassConstructor(function->shared()->kind())) { 4510 if (function->shared()->internal_formal_parameter_count() == 4511 wasm_count) { 4512 int pos = 0; 4513 args[pos++] = callable_node; // target callable. 4514 // Receiver. 4515 if (is_sloppy(function->shared()->language_mode()) && 4516 !function->shared()->native()) { 4517 Node* global_proxy = LOAD_FIXED_ARRAY_SLOT( 4518 native_context, Context::GLOBAL_PROXY_INDEX); 4519 args[pos++] = global_proxy; 4520 } else { 4521 args[pos++] = undefined_node; 4522 } 4523 4524 call_descriptor = Linkage::GetJSCallDescriptor( 4525 graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags); 4526 4527 // Convert wasm numbers to JS values. 4528 pos = AddArgumentNodes(args, pos, wasm_count, sig_); 4529 4530 args[pos++] = undefined_node; // new target 4531 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count 4532 args[pos++] = function_context; 4533 args[pos++] = Effect(); 4534 args[pos++] = Control(); 4535 4536 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), 4537 pos, args); 4538 } else if (function->shared()->internal_formal_parameter_count() >= 0) { 4539 int pos = 0; 4540 args[pos++] = mcgraph()->RelocatableIntPtrConstant( 4541 wasm::WasmCode::kWasmArgumentsAdaptor, RelocInfo::WASM_STUB_CALL); 4542 args[pos++] = callable_node; // target callable 4543 args[pos++] = undefined_node; // new target 4544 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count 4545 args[pos++] = mcgraph()->Int32Constant( 4546 function->shared()->internal_formal_parameter_count()); 4547 // Receiver. 4548 if (is_sloppy(function->shared()->language_mode()) && 4549 !function->shared()->native()) { 4550 Node* global_proxy = LOAD_FIXED_ARRAY_SLOT( 4551 native_context, Context::GLOBAL_PROXY_INDEX); 4552 args[pos++] = global_proxy; 4553 } else { 4554 args[pos++] = undefined_node; 4555 } 4556 4557 call_descriptor = Linkage::GetStubCallDescriptor( 4558 mcgraph()->zone(), ArgumentAdaptorDescriptor{}, 1 + wasm_count, 4559 CallDescriptor::kNoFlags, Operator::kNoProperties, 4560 StubCallMode::kCallWasmRuntimeStub); 4561 4562 // Convert wasm numbers to JS values. 4563 pos = AddArgumentNodes(args, pos, wasm_count, sig_); 4564 args[pos++] = function_context; 4565 args[pos++] = Effect(); 4566 args[pos++] = Control(); 4567 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), 4568 pos, args); 4569 } 4570 } 4571 } 4572 4573 // We cannot call the target directly, we have to use the Call builtin. 4574 if (!call) { 4575 int pos = 0; 4576 args[pos++] = mcgraph()->RelocatableIntPtrConstant( 4577 wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL); 4578 args[pos++] = callable_node; 4579 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count 4580 args[pos++] = undefined_node; // receiver 4581 4582 call_descriptor = Linkage::GetStubCallDescriptor( 4583 graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1, 4584 CallDescriptor::kNoFlags, Operator::kNoProperties, 4585 StubCallMode::kCallWasmRuntimeStub); 4586 4587 // Convert wasm numbers to JS values. 4588 pos = AddArgumentNodes(args, pos, wasm_count, sig_); 4589 4590 // The native_context is sufficient here, because all kind of callables 4591 // which depend on the context provide their own context. The context here 4592 // is only needed if the target is a constructor to throw a TypeError, if 4593 // the target is a native function, or if the target is a callable 4594 // JSObject, which can only be constructed by the runtime. 4595 args[pos++] = native_context; 4596 args[pos++] = Effect(); 4597 args[pos++] = Control(); 4598 4599 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos, 4600 args); 4601 } 4602 4603 SetEffect(call); 4604 SetSourcePosition(call, 0); 4605 4606 // Convert the return value back. 4607 Node* val = sig_->return_count() == 0 4608 ? mcgraph()->Int32Constant(0) 4609 : FromJS(call, native_context, sig_->GetReturn()); 4610 4611 BuildModifyThreadInWasmFlag(true); 4612 4613 Return(val); 4614 return true; 4615 } 4616 4617 void BuildWasmInterpreterEntry(uint32_t func_index) { 4618 int param_count = static_cast<int>(sig_->parameter_count()); 4619 4620 // Build the start and the parameter nodes. 4621 SetEffect(SetControl(Start(param_count + 3))); 4622 4623 // Create the instance_node from the passed parameter. 4624 instance_node_.set(Param(wasm::kWasmInstanceParameterIndex)); 4625 4626 // Compute size for the argument buffer. 4627 int args_size_bytes = 0; 4628 for (wasm::ValueType type : sig_->parameters()) { 4629 args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type); 4630 } 4631 4632 // The return value is also passed via this buffer: 4633 DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count()); 4634 // TODO(wasm): Handle multi-value returns. 4635 DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns); 4636 int return_size_bytes = 4637 sig_->return_count() == 0 4638 ? 0 4639 : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn()); 4640 4641 // Get a stack slot for the arguments. 4642 Node* arg_buffer = 4643 args_size_bytes == 0 && return_size_bytes == 0 4644 ? mcgraph()->IntPtrConstant(0) 4645 : graph()->NewNode(mcgraph()->machine()->StackSlot( 4646 std::max(args_size_bytes, return_size_bytes), 8)); 4647 4648 // Now store all our arguments to the buffer. 4649 int offset = 0; 4650 4651 for (int i = 0; i < param_count; ++i) { 4652 wasm::ValueType type = sig_->GetParam(i); 4653 // Start from the parameter with index 1 to drop the instance_node. 4654 SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer, 4655 Int32Constant(offset), Param(i + 1), Effect(), 4656 Control())); 4657 offset += wasm::ValueTypes::ElementSizeInBytes(type); 4658 } 4659 DCHECK_EQ(args_size_bytes, offset); 4660 4661 // We are passing the raw arg_buffer here. To the GC and other parts, it 4662 // looks like a Smi (lowest bit not set). In the runtime function however, 4663 // don't call Smi::value on it, but just cast it to a byte pointer. 4664 Node* parameters[] = { 4665 jsgraph()->SmiConstant(func_index), arg_buffer, 4666 }; 4667 BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters, 4668 arraysize(parameters)); 4669 4670 // Read back the return value. 4671 if (sig_->return_count() == 0) { 4672 Return(Int32Constant(0)); 4673 } else { 4674 // TODO(wasm): Implement multi-return. 4675 DCHECK_EQ(1, sig_->return_count()); 4676 MachineType load_rep = 4677 wasm::ValueTypes::MachineTypeFor(sig_->GetReturn()); 4678 Node* val = SetEffect( 4679 graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer, 4680 Int32Constant(0), Effect(), Control())); 4681 Return(val); 4682 } 4683 4684 if (ContainsInt64(sig_)) LowerInt64(); 4685 } 4686 4687 void BuildCWasmEntry() { 4688 // Build the start and the JS parameter nodes. 4689 SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5))); 4690 4691 // Create parameter nodes (offset by 1 for the receiver parameter). 4692 Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1); 4693 MachineOperatorBuilder* machine = mcgraph()->machine(); 4694 Node* code_obj = graph()->NewNode( 4695 machine->Load(MachineType::Pointer()), foreign_code_obj, 4696 Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), 4697 Effect(), Control()); 4698 Node* instance_node = Param(CWasmEntryParameters::kWasmInstance + 1); 4699 Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1); 4700 4701 int wasm_arg_count = static_cast<int>(sig_->parameter_count()); 4702 int arg_count = wasm_arg_count + 4; // code, instance_node, control, effect 4703 Node** args = Buffer(arg_count); 4704 4705 int pos = 0; 4706 args[pos++] = code_obj; 4707 args[pos++] = instance_node; 4708 4709 int offset = 0; 4710 for (wasm::ValueType type : sig_->parameters()) { 4711 Node* arg_load = SetEffect( 4712 graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer, 4713 Int32Constant(offset), Effect(), Control())); 4714 args[pos++] = arg_load; 4715 offset += wasm::ValueTypes::ElementSizeInBytes(type); 4716 } 4717 4718 args[pos++] = Effect(); 4719 args[pos++] = Control(); 4720 DCHECK_EQ(arg_count, pos); 4721 4722 // Call the wasm code. 4723 auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_); 4724 4725 Node* call = SetEffect(graph()->NewNode( 4726 mcgraph()->common()->Call(call_descriptor), arg_count, args)); 4727 4728 // Store the return value. 4729 DCHECK_GE(1, sig_->return_count()); 4730 if (sig_->return_count() == 1) { 4731 StoreRepresentation store_rep( 4732 wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()), 4733 kNoWriteBarrier); 4734 SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep), 4735 arg_buffer, Int32Constant(0), call, Effect(), 4736 Control())); 4737 } 4738 Return(jsgraph()->SmiConstant(0)); 4739 4740 if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) { 4741 MachineRepresentation sig_reps[] = { 4742 MachineRepresentation::kWord32, // return value 4743 MachineRepresentation::kTagged, // receiver 4744 MachineRepresentation::kTagged, // arg0 (code) 4745 MachineRepresentation::kTagged // arg1 (buffer) 4746 }; 4747 Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps); 4748 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), 4749 mcgraph()->common(), mcgraph()->zone(), &c_entry_sig); 4750 r.LowerGraph(); 4751 } 4752 } 4753 4754 JSGraph* jsgraph() { return jsgraph_; } 4755 4756 private: 4757 Isolate* const isolate_; 4758 JSGraph* jsgraph_; 4759 StubCallMode stub_mode_; 4760 SetOncePointer<const Operator> allocate_heap_number_operator_; 4761 }; 4762 } // namespace 4763 4764 MaybeHandle<Code> CompileJSToWasmWrapper( 4765 Isolate* isolate, const wasm::NativeModule* native_module, 4766 wasm::FunctionSig* sig, bool is_import, 4767 wasm::UseTrapHandler use_trap_handler) { 4768 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"), 4769 "CompileJSToWasmWrapper"); 4770 const wasm::WasmModule* module = native_module->module(); 4771 4772 //---------------------------------------------------------------------------- 4773 // Create the Graph. 4774 //---------------------------------------------------------------------------- 4775 Zone zone(isolate->allocator(), ZONE_NAME); 4776 Graph graph(&zone); 4777 CommonOperatorBuilder common(&zone); 4778 MachineOperatorBuilder machine( 4779 &zone, MachineType::PointerRepresentation(), 4780 InstructionSelector::SupportedMachineOperatorFlags(), 4781 InstructionSelector::AlignmentRequirements()); 4782 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); 4783 4784 Node* control = nullptr; 4785 Node* effect = nullptr; 4786 4787 wasm::ModuleEnv env(module, use_trap_handler, wasm::kRuntimeExceptionSupport); 4788 WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig, nullptr, 4789 StubCallMode::kCallOnHeapBuiltin); 4790 builder.set_control_ptr(&control); 4791 builder.set_effect_ptr(&effect); 4792 builder.BuildJSToWasmWrapper(is_import); 4793 4794 //---------------------------------------------------------------------------- 4795 // Run the compilation pipeline. 4796 //---------------------------------------------------------------------------- 4797 #ifdef DEBUG 4798 EmbeddedVector<char, 32> func_name; 4799 static unsigned id = 0; 4800 func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++)); 4801 #else 4802 Vector<const char> func_name = CStrVector("js-to-wasm"); 4803 #endif 4804 4805 OptimizedCompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION); 4806 4807 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO. 4808 StdoutStream{} << "-- Graph after change lowering -- " << std::endl 4809 << AsRPO(graph); 4810 } 4811 4812 // Schedule and compile to machine code. 4813 int params = static_cast<int>(sig->parameter_count()); 4814 CallDescriptor* incoming = Linkage::GetJSCallDescriptor( 4815 &zone, false, params + 1, CallDescriptor::kNoFlags); 4816 4817 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting( 4818 &info, isolate, incoming, &graph, WasmAssemblerOptions()); 4819 Handle<Code> code; 4820 if (!maybe_code.ToHandle(&code)) { 4821 return maybe_code; 4822 } 4823 #ifdef ENABLE_DISASSEMBLER 4824 if (FLAG_print_opt_code) { 4825 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer()); 4826 OFStream os(tracing_scope.file()); 4827 code->Disassemble(func_name.start(), os); 4828 } 4829 #endif 4830 4831 if (must_record_function_compilation(isolate)) { 4832 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, 4833 "%.*s", func_name.length(), func_name.start()); 4834 } 4835 4836 return code; 4837 } 4838 4839 MaybeHandle<Code> CompileWasmToJSWrapper( 4840 Isolate* isolate, Handle<JSReceiver> target, wasm::FunctionSig* sig, 4841 uint32_t index, wasm::ModuleOrigin origin, 4842 wasm::UseTrapHandler use_trap_handler) { 4843 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"), 4844 "CompileWasmToJSWrapper"); 4845 //---------------------------------------------------------------------------- 4846 // Create the Graph 4847 //---------------------------------------------------------------------------- 4848 Zone zone(isolate->allocator(), ZONE_NAME); 4849 Graph graph(&zone); 4850 CommonOperatorBuilder common(&zone); 4851 MachineOperatorBuilder machine( 4852 &zone, MachineType::PointerRepresentation(), 4853 InstructionSelector::SupportedMachineOperatorFlags(), 4854 InstructionSelector::AlignmentRequirements()); 4855 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); 4856 4857 Node* control = nullptr; 4858 Node* effect = nullptr; 4859 4860 SourcePositionTable* source_position_table = 4861 origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph) 4862 : nullptr; 4863 4864 wasm::ModuleEnv env(nullptr, use_trap_handler, 4865 wasm::kRuntimeExceptionSupport); 4866 4867 WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig, 4868 source_position_table, 4869 StubCallMode::kCallWasmRuntimeStub); 4870 builder.set_control_ptr(&control); 4871 builder.set_effect_ptr(&effect); 4872 builder.BuildWasmToJSWrapper(target, index); 4873 4874 #ifdef DEBUG 4875 EmbeddedVector<char, 32> func_name; 4876 static unsigned id = 0; 4877 func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++)); 4878 #else 4879 Vector<const char> func_name = CStrVector("wasm-to-js"); 4880 #endif 4881 4882 OptimizedCompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION); 4883 4884 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO. 4885 StdoutStream{} << "-- Graph after change lowering -- " << std::endl 4886 << AsRPO(graph); 4887 } 4888 4889 // Schedule and compile to machine code. 4890 CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); 4891 if (machine.Is32()) { 4892 incoming = GetI32WasmCallDescriptor(&zone, incoming); 4893 } 4894 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting( 4895 &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate), 4896 nullptr, source_position_table); 4897 Handle<Code> code; 4898 if (!maybe_code.ToHandle(&code)) { 4899 return maybe_code; 4900 } 4901 #ifdef ENABLE_DISASSEMBLER 4902 if (FLAG_print_opt_code) { 4903 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer()); 4904 OFStream os(tracing_scope.file()); 4905 code->Disassemble(func_name.start(), os); 4906 } 4907 #endif 4908 4909 if (must_record_function_compilation(isolate)) { 4910 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, 4911 "%.*s", func_name.length(), func_name.start()); 4912 } 4913 4914 return code; 4915 } 4916 4917 MaybeHandle<Code> CompileWasmInterpreterEntry(Isolate* isolate, 4918 uint32_t func_index, 4919 wasm::FunctionSig* sig) { 4920 //---------------------------------------------------------------------------- 4921 // Create the Graph 4922 //---------------------------------------------------------------------------- 4923 Zone zone(isolate->allocator(), ZONE_NAME); 4924 Graph graph(&zone); 4925 CommonOperatorBuilder common(&zone); 4926 MachineOperatorBuilder machine( 4927 &zone, MachineType::PointerRepresentation(), 4928 InstructionSelector::SupportedMachineOperatorFlags(), 4929 InstructionSelector::AlignmentRequirements()); 4930 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); 4931 4932 Node* control = nullptr; 4933 Node* effect = nullptr; 4934 4935 WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr, 4936 StubCallMode::kCallWasmRuntimeStub); 4937 builder.set_control_ptr(&control); 4938 builder.set_effect_ptr(&effect); 4939 builder.BuildWasmInterpreterEntry(func_index); 4940 4941 // Schedule and compile to machine code. 4942 CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); 4943 if (machine.Is32()) { 4944 incoming = GetI32WasmCallDescriptor(&zone, incoming); 4945 } 4946 #ifdef DEBUG 4947 EmbeddedVector<char, 32> func_name; 4948 func_name.Truncate( 4949 SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index)); 4950 #else 4951 Vector<const char> func_name = CStrVector("wasm-interpreter-entry"); 4952 #endif 4953 4954 OptimizedCompilationInfo info(func_name, &zone, Code::WASM_INTERPRETER_ENTRY); 4955 4956 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO. 4957 StdoutStream{} << "-- Wasm interpreter entry graph -- " << std::endl 4958 << AsRPO(graph); 4959 } 4960 4961 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting( 4962 &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate), 4963 nullptr); 4964 Handle<Code> code; 4965 if (!maybe_code.ToHandle(&code)) { 4966 return maybe_code; 4967 } 4968 #ifdef ENABLE_DISASSEMBLER 4969 if (FLAG_print_opt_code) { 4970 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer()); 4971 OFStream os(tracing_scope.file()); 4972 code->Disassemble(func_name.start(), os); 4973 } 4974 #endif 4975 4976 if (must_record_function_compilation(isolate)) { 4977 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, 4978 "%.*s", func_name.length(), func_name.start()); 4979 } 4980 4981 return maybe_code; 4982 } 4983 4984 MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) { 4985 Zone zone(isolate->allocator(), ZONE_NAME); 4986 Graph graph(&zone); 4987 CommonOperatorBuilder common(&zone); 4988 MachineOperatorBuilder machine( 4989 &zone, MachineType::PointerRepresentation(), 4990 InstructionSelector::SupportedMachineOperatorFlags(), 4991 InstructionSelector::AlignmentRequirements()); 4992 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); 4993 4994 Node* control = nullptr; 4995 Node* effect = nullptr; 4996 4997 WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr, 4998 StubCallMode::kCallOnHeapBuiltin); 4999 builder.set_control_ptr(&control); 5000 builder.set_effect_ptr(&effect); 5001 builder.BuildCWasmEntry(); 5002 5003 // Schedule and compile to machine code. 5004 CallDescriptor* incoming = Linkage::GetJSCallDescriptor( 5005 &zone, false, CWasmEntryParameters::kNumParameters + 1, 5006 CallDescriptor::kNoFlags); 5007 5008 // Build a name in the form "c-wasm-entry:<params>:<returns>". 5009 static constexpr size_t kMaxNameLen = 128; 5010 char debug_name[kMaxNameLen] = "c-wasm-entry:"; 5011 size_t name_len = strlen(debug_name); 5012 auto append_name_char = [&](char c) { 5013 if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c; 5014 }; 5015 for (wasm::ValueType t : sig->parameters()) { 5016 append_name_char(wasm::ValueTypes::ShortNameOf(t)); 5017 } 5018 append_name_char(':'); 5019 for (wasm::ValueType t : sig->returns()) { 5020 append_name_char(wasm::ValueTypes::ShortNameOf(t)); 5021 } 5022 debug_name[name_len] = '\0'; 5023 Vector<const char> debug_name_vec(debug_name, name_len); 5024 5025 OptimizedCompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY); 5026 5027 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO. 5028 StdoutStream{} << "-- C Wasm entry graph -- " << std::endl << AsRPO(graph); 5029 } 5030 5031 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting( 5032 &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate)); 5033 Handle<Code> code; 5034 if (!maybe_code.ToHandle(&code)) { 5035 return maybe_code; 5036 } 5037 #ifdef ENABLE_DISASSEMBLER 5038 if (FLAG_print_opt_code) { 5039 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer()); 5040 OFStream os(tracing_scope.file()); 5041 code->Disassemble(debug_name, os); 5042 } 5043 #endif 5044 5045 return code; 5046 } 5047 5048 TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit( 5049 wasm::WasmCompilationUnit* wasm_unit) 5050 : wasm_unit_(wasm_unit) {} 5051 5052 // Clears unique_ptrs, but (part of) the type is forward declared in the header. 5053 TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default; 5054 5055 SourcePositionTable* TurbofanWasmCompilationUnit::BuildGraphForWasmFunction( 5056 wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph, 5057 NodeOriginTable* node_origins) { 5058 base::ElapsedTimer decode_timer; 5059 if (FLAG_trace_wasm_decode_time) { 5060 decode_timer.Start(); 5061 } 5062 5063 // Create a TF graph during decoding. 5064 SourcePositionTable* source_position_table = 5065 new (mcgraph->zone()) SourcePositionTable(mcgraph->graph()); 5066 WasmGraphBuilder builder(wasm_unit_->env_, mcgraph->zone(), mcgraph, 5067 wasm_unit_->func_body_.sig, source_position_table); 5068 graph_construction_result_ = wasm::BuildTFGraph( 5069 wasm_unit_->wasm_engine_->allocator(), 5070 wasm_unit_->native_module_->enabled_features(), wasm_unit_->env_->module, 5071 &builder, detected, wasm_unit_->func_body_, node_origins); 5072 if (graph_construction_result_.failed()) { 5073 if (FLAG_trace_wasm_compiler) { 5074 StdoutStream{} << "Compilation failed: " 5075 << graph_construction_result_.error_msg() << std::endl; 5076 } 5077 return nullptr; 5078 } 5079 5080 builder.LowerInt64(); 5081 5082 if (builder.has_simd() && 5083 (!CpuFeatures::SupportsWasmSimd128() || wasm_unit_->env_->lower_simd)) { 5084 SimdScalarLowering( 5085 mcgraph, 5086 CreateMachineSignature(mcgraph->zone(), wasm_unit_->func_body_.sig)) 5087 .LowerGraph(); 5088 } 5089 5090 if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start && 5091 wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) { 5092 PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(), 5093 wasm_unit_->func_body_, wasm_unit_->env_->module, 5094 wasm::kPrintLocals); 5095 } 5096 if (FLAG_trace_wasm_decode_time) { 5097 *decode_ms = decode_timer.Elapsed().InMillisecondsF(); 5098 } 5099 return source_position_table; 5100 } 5101 5102 namespace { 5103 Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) { 5104 if (!name.is_empty()) { 5105 return name; 5106 } 5107 #ifdef DEBUG 5108 constexpr int kBufferLength = 15; 5109 5110 EmbeddedVector<char, kBufferLength> name_vector; 5111 int name_len = SNPrintF(name_vector, "wasm#%d", index); 5112 DCHECK(name_len > 0 && name_len < name_vector.length()); 5113 5114 char* index_name = zone->NewArray<char>(name_len); 5115 memcpy(index_name, name_vector.start(), name_len); 5116 return Vector<const char>(index_name, name_len); 5117 #else 5118 return {}; 5119 #endif 5120 } 5121 5122 } // namespace 5123 5124 void TurbofanWasmCompilationUnit::ExecuteCompilation( 5125 wasm::WasmFeatures* detected) { 5126 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"), 5127 "ExecuteTurbofanCompilation"); 5128 double decode_ms = 0; 5129 size_t node_count = 0; 5130 5131 // Scope for the {graph_zone}. 5132 { 5133 Zone graph_zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME); 5134 MachineGraph* mcgraph = new (&graph_zone) 5135 MachineGraph(new (&graph_zone) Graph(&graph_zone), 5136 new (&graph_zone) CommonOperatorBuilder(&graph_zone), 5137 new (&graph_zone) MachineOperatorBuilder( 5138 &graph_zone, MachineType::PointerRepresentation(), 5139 InstructionSelector::SupportedMachineOperatorFlags(), 5140 InstructionSelector::AlignmentRequirements())); 5141 5142 Zone compilation_zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME); 5143 5144 OptimizedCompilationInfo info( 5145 GetDebugName(&compilation_zone, wasm_unit_->func_name_, 5146 wasm_unit_->func_index_), 5147 &compilation_zone, Code::WASM_FUNCTION); 5148 if (wasm_unit_->env_->runtime_exception_support) { 5149 info.SetWasmRuntimeExceptionSupport(); 5150 } 5151 5152 NodeOriginTable* node_origins = info.trace_turbo_json_enabled() 5153 ? new (&graph_zone) 5154 NodeOriginTable(mcgraph->graph()) 5155 : nullptr; 5156 SourcePositionTable* source_positions = 5157 BuildGraphForWasmFunction(detected, &decode_ms, mcgraph, node_origins); 5158 5159 if (graph_construction_result_.failed()) { 5160 ok_ = false; 5161 return; 5162 } 5163 5164 if (node_origins) { 5165 node_origins->AddDecorator(); 5166 } 5167 5168 base::ElapsedTimer pipeline_timer; 5169 if (FLAG_trace_wasm_decode_time) { 5170 node_count = mcgraph->graph()->NodeCount(); 5171 pipeline_timer.Start(); 5172 } 5173 5174 // Run the compiler pipeline to generate machine code. 5175 auto call_descriptor = 5176 GetWasmCallDescriptor(&compilation_zone, wasm_unit_->func_body_.sig); 5177 if (mcgraph->machine()->Is32()) { 5178 call_descriptor = 5179 GetI32WasmCallDescriptor(&compilation_zone, call_descriptor); 5180 } 5181 5182 std::unique_ptr<OptimizedCompilationJob> job( 5183 Pipeline::NewWasmCompilationJob( 5184 &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor, 5185 source_positions, node_origins, wasm_unit_->func_body_, 5186 const_cast<wasm::WasmModule*>(wasm_unit_->env_->module), 5187 wasm_unit_->native_module_, wasm_unit_->func_index_, 5188 wasm_unit_->env_->module->origin)); 5189 ok_ = job->ExecuteJob() == CompilationJob::SUCCEEDED; 5190 // TODO(bradnelson): Improve histogram handling of size_t. 5191 wasm_unit_->counters_->wasm_compile_function_peak_memory_bytes()->AddSample( 5192 static_cast<int>(mcgraph->graph()->zone()->allocation_size())); 5193 5194 if (FLAG_trace_wasm_decode_time) { 5195 double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF(); 5196 PrintF( 5197 "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, " 5198 "%0.3f ms pipeline\n", 5199 static_cast<unsigned>(wasm_unit_->func_body_.end - 5200 wasm_unit_->func_body_.start), 5201 decode_ms, node_count, pipeline_ms); 5202 } 5203 if (ok_) wasm_code_ = info.wasm_code(); 5204 } 5205 if (ok_) wasm_unit_->native_module()->PublishCode(wasm_code_); 5206 } 5207 5208 wasm::WasmCode* TurbofanWasmCompilationUnit::FinishCompilation( 5209 wasm::ErrorThrower* thrower) { 5210 if (!ok_) { 5211 if (graph_construction_result_.failed()) { 5212 // Add the function as another context for the exception. 5213 EmbeddedVector<char, 128> message; 5214 if (wasm_unit_->func_name_.start() == nullptr) { 5215 SNPrintF(message, "Compiling wasm function #%d failed", 5216 wasm_unit_->func_index_); 5217 } else { 5218 wasm::TruncatedUserString<> trunc_name(wasm_unit_->func_name_); 5219 SNPrintF(message, "Compiling wasm function #%d:%.*s failed", 5220 wasm_unit_->func_index_, trunc_name.length(), 5221 trunc_name.start()); 5222 } 5223 thrower->CompileFailed(message.start(), graph_construction_result_); 5224 } 5225 5226 return nullptr; 5227 } 5228 return wasm_code_; 5229 } 5230 5231 namespace { 5232 // Helper for allocating either an GP or FP reg, or the next stack slot. 5233 class LinkageLocationAllocator { 5234 public: 5235 template <size_t kNumGpRegs, size_t kNumFpRegs> 5236 constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs], 5237 const DoubleRegister (&fp)[kNumFpRegs]) 5238 : allocator_(wasm::LinkageAllocator(gp, fp)) {} 5239 5240 LinkageLocation Next(MachineRepresentation rep) { 5241 MachineType type = MachineType::TypeForRepresentation(rep); 5242 if (IsFloatingPoint(rep)) { 5243 if (allocator_.CanAllocateFP(rep)) { 5244 int reg_code = allocator_.NextFpReg(rep); 5245 return LinkageLocation::ForRegister(reg_code, type); 5246 } 5247 } else if (allocator_.CanAllocateGP()) { 5248 int reg_code = allocator_.NextGpReg(); 5249 return LinkageLocation::ForRegister(reg_code, type); 5250 } 5251 // Cannot use register; use stack slot. 5252 int index = -1 - allocator_.NextStackSlot(rep); 5253 return LinkageLocation::ForCallerFrameSlot(index, type); 5254 } 5255 5256 void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); } 5257 int NumStackSlots() const { return allocator_.NumStackSlots(); } 5258 5259 private: 5260 wasm::LinkageAllocator allocator_; 5261 }; 5262 } // namespace 5263 5264 // General code uses the above configuration data. 5265 CallDescriptor* GetWasmCallDescriptor( 5266 Zone* zone, wasm::FunctionSig* fsig, 5267 WasmGraphBuilder::UseRetpoline use_retpoline) { 5268 // The '+ 1' here is to accomodate the instance object as first parameter. 5269 LocationSignature::Builder locations(zone, fsig->return_count(), 5270 fsig->parameter_count() + 1); 5271 5272 // Add register and/or stack parameter(s). 5273 LinkageLocationAllocator params(wasm::kGpParamRegisters, 5274 wasm::kFpParamRegisters); 5275 5276 // The instance object. 5277 locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer)); 5278 5279 const int parameter_count = static_cast<int>(fsig->parameter_count()); 5280 for (int i = 0; i < parameter_count; i++) { 5281 MachineRepresentation param = 5282 wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i)); 5283 auto l = params.Next(param); 5284 locations.AddParam(l); 5285 } 5286 5287 // Add return location(s). 5288 LinkageLocationAllocator rets(wasm::kGpReturnRegisters, 5289 wasm::kFpReturnRegisters); 5290 5291 int parameter_slots = params.NumStackSlots(); 5292 if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2); 5293 5294 rets.SetStackOffset(parameter_slots); 5295 5296 const int return_count = static_cast<int>(locations.return_count_); 5297 for (int i = 0; i < return_count; i++) { 5298 MachineRepresentation ret = 5299 wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i)); 5300 auto l = rets.Next(ret); 5301 locations.AddReturn(l); 5302 } 5303 5304 const RegList kCalleeSaveRegisters = 0; 5305 const RegList kCalleeSaveFPRegisters = 0; 5306 5307 // The target for wasm calls is always a code object. 5308 MachineType target_type = MachineType::Pointer(); 5309 LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type); 5310 5311 CallDescriptor::Kind kind = CallDescriptor::kCallWasmFunction; 5312 5313 CallDescriptor::Flags flags = 5314 use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags; 5315 return new (zone) CallDescriptor( // -- 5316 kind, // kind 5317 target_type, // target MachineType 5318 target_loc, // target location 5319 locations.Build(), // location_sig 5320 parameter_slots, // stack_parameter_count 5321 compiler::Operator::kNoProperties, // properties 5322 kCalleeSaveRegisters, // callee-saved registers 5323 kCalleeSaveFPRegisters, // callee-saved fp regs 5324 flags, // flags 5325 "wasm-call", // debug name 5326 0, // allocatable registers 5327 rets.NumStackSlots() - parameter_slots); // stack_return_count 5328 } 5329 5330 namespace { 5331 CallDescriptor* ReplaceTypeInCallDescriptorWith( 5332 Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements, 5333 MachineType input_type, MachineRepresentation output_type) { 5334 size_t parameter_count = call_descriptor->ParameterCount(); 5335 size_t return_count = call_descriptor->ReturnCount(); 5336 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) { 5337 if (call_descriptor->GetParameterType(i) == input_type) { 5338 parameter_count += num_replacements - 1; 5339 } 5340 } 5341 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) { 5342 if (call_descriptor->GetReturnType(i) == input_type) { 5343 return_count += num_replacements - 1; 5344 } 5345 } 5346 if (parameter_count == call_descriptor->ParameterCount() && 5347 return_count == call_descriptor->ReturnCount()) { 5348 return call_descriptor; 5349 } 5350 5351 LocationSignature::Builder locations(zone, return_count, parameter_count); 5352 5353 LinkageLocationAllocator params(wasm::kGpParamRegisters, 5354 wasm::kFpParamRegisters); 5355 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) { 5356 if (call_descriptor->GetParameterType(i) == input_type) { 5357 for (size_t j = 0; j < num_replacements; j++) { 5358 locations.AddParam(params.Next(output_type)); 5359 } 5360 } else { 5361 locations.AddParam( 5362 params.Next(call_descriptor->GetParameterType(i).representation())); 5363 } 5364 } 5365 5366 LinkageLocationAllocator rets(wasm::kGpReturnRegisters, 5367 wasm::kFpReturnRegisters); 5368 rets.SetStackOffset(params.NumStackSlots()); 5369 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) { 5370 if (call_descriptor->GetReturnType(i) == input_type) { 5371 for (size_t j = 0; j < num_replacements; j++) { 5372 locations.AddReturn(rets.Next(output_type)); 5373 } 5374 } else { 5375 locations.AddReturn( 5376 rets.Next(call_descriptor->GetReturnType(i).representation())); 5377 } 5378 } 5379 5380 return new (zone) CallDescriptor( // -- 5381 call_descriptor->kind(), // kind 5382 call_descriptor->GetInputType(0), // target MachineType 5383 call_descriptor->GetInputLocation(0), // target location 5384 locations.Build(), // location_sig 5385 params.NumStackSlots(), // stack_parameter_count 5386 call_descriptor->properties(), // properties 5387 call_descriptor->CalleeSavedRegisters(), // callee-saved registers 5388 call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs 5389 call_descriptor->flags(), // flags 5390 call_descriptor->debug_name(), // debug name 5391 call_descriptor->AllocatableRegisters(), // allocatable registers 5392 rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count 5393 } 5394 } // namespace 5395 5396 CallDescriptor* GetI32WasmCallDescriptor(Zone* zone, 5397 CallDescriptor* call_descriptor) { 5398 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2, 5399 MachineType::Int64(), 5400 MachineRepresentation::kWord32); 5401 } 5402 5403 CallDescriptor* GetI32WasmCallDescriptorForSimd( 5404 Zone* zone, CallDescriptor* call_descriptor) { 5405 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4, 5406 MachineType::Simd128(), 5407 MachineRepresentation::kWord32); 5408 } 5409 5410 AssemblerOptions WasmAssemblerOptions() { 5411 AssemblerOptions options; 5412 options.record_reloc_info_for_serialization = true; 5413 options.enable_root_array_delta_access = false; 5414 return options; 5415 } 5416 5417 #undef WASM_64 5418 #undef FATAL_UNSUPPORTED_OPCODE 5419 #undef WASM_INSTANCE_OBJECT_OFFSET 5420 #undef LOAD_INSTANCE_FIELD 5421 #undef LOAD_FIXED_ARRAY_SLOT 5422 5423 } // namespace compiler 5424 } // namespace internal 5425 } // namespace v8 5426
