1 // Copyright 2015 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/interpreter/interpreter-assembler.h" 6 7 #include <limits> 8 #include <ostream> 9 10 #include "src/code-factory.h" 11 #include "src/frames.h" 12 #include "src/interface-descriptors.h" 13 #include "src/interpreter/bytecodes.h" 14 #include "src/interpreter/interpreter.h" 15 #include "src/machine-type.h" 16 #include "src/macro-assembler.h" 17 #include "src/objects-inl.h" 18 #include "src/zone/zone.h" 19 20 namespace v8 { 21 namespace internal { 22 namespace interpreter { 23 24 using compiler::CodeAssemblerState; 25 using compiler::Node; 26 template <class T> 27 using TNode = compiler::TNode<T>; 28 29 InterpreterAssembler::InterpreterAssembler(CodeAssemblerState* state, 30 Bytecode bytecode, 31 OperandScale operand_scale) 32 : CodeStubAssembler(state), 33 bytecode_(bytecode), 34 operand_scale_(operand_scale), 35 VARIABLE_CONSTRUCTOR(interpreted_frame_pointer_, 36 MachineType::PointerRepresentation()), 37 VARIABLE_CONSTRUCTOR( 38 bytecode_array_, MachineRepresentation::kTagged, 39 Parameter(InterpreterDispatchDescriptor::kBytecodeArray)), 40 VARIABLE_CONSTRUCTOR( 41 bytecode_offset_, MachineType::PointerRepresentation(), 42 Parameter(InterpreterDispatchDescriptor::kBytecodeOffset)), 43 VARIABLE_CONSTRUCTOR( 44 dispatch_table_, MachineType::PointerRepresentation(), 45 Parameter(InterpreterDispatchDescriptor::kDispatchTable)), 46 VARIABLE_CONSTRUCTOR( 47 accumulator_, MachineRepresentation::kTagged, 48 Parameter(InterpreterDispatchDescriptor::kAccumulator)), 49 accumulator_use_(AccumulatorUse::kNone), 50 made_call_(false), 51 reloaded_frame_ptr_(false), 52 bytecode_array_valid_(true), 53 disable_stack_check_across_call_(false), 54 stack_pointer_before_call_(nullptr) { 55 #ifdef V8_TRACE_IGNITION 56 TraceBytecode(Runtime::kInterpreterTraceBytecodeEntry); 57 #endif 58 RegisterCallGenerationCallbacks([this] { CallPrologue(); }, 59 [this] { CallEpilogue(); }); 60 61 // Save the bytecode offset immediately if bytecode will make a call along the 62 // critical path, or it is a return bytecode. 63 if (Bytecodes::MakesCallAlongCriticalPath(bytecode) || 64 Bytecodes::Returns(bytecode)) { 65 SaveBytecodeOffset(); 66 } 67 } 68 69 InterpreterAssembler::~InterpreterAssembler() { 70 // If the following check fails the handler does not use the 71 // accumulator in the way described in the bytecode definitions in 72 // bytecodes.h. 73 DCHECK_EQ(accumulator_use_, Bytecodes::GetAccumulatorUse(bytecode_)); 74 UnregisterCallGenerationCallbacks(); 75 } 76 77 Node* InterpreterAssembler::GetInterpretedFramePointer() { 78 if (!interpreted_frame_pointer_.IsBound()) { 79 interpreted_frame_pointer_.Bind(LoadParentFramePointer()); 80 } else if (Bytecodes::MakesCallAlongCriticalPath(bytecode_) && made_call_ && 81 !reloaded_frame_ptr_) { 82 interpreted_frame_pointer_.Bind(LoadParentFramePointer()); 83 reloaded_frame_ptr_ = true; 84 } 85 return interpreted_frame_pointer_.value(); 86 } 87 88 Node* InterpreterAssembler::BytecodeOffset() { 89 if (Bytecodes::MakesCallAlongCriticalPath(bytecode_) && made_call_ && 90 (bytecode_offset_.value() == 91 Parameter(InterpreterDispatchDescriptor::kBytecodeOffset))) { 92 bytecode_offset_.Bind(ReloadBytecodeOffset()); 93 } 94 return bytecode_offset_.value(); 95 } 96 97 Node* InterpreterAssembler::ReloadBytecodeOffset() { 98 Node* offset = LoadAndUntagRegister(Register::bytecode_offset()); 99 if (operand_scale() != OperandScale::kSingle) { 100 // Add one to the offset such that it points to the actual bytecode rather 101 // than the Wide / ExtraWide prefix bytecode. 102 offset = IntPtrAdd(offset, IntPtrConstant(1)); 103 } 104 return offset; 105 } 106 107 void InterpreterAssembler::SaveBytecodeOffset() { 108 Node* offset = BytecodeOffset(); 109 if (operand_scale() != OperandScale::kSingle) { 110 // Subtract one from the offset such that it points to the Wide / ExtraWide 111 // prefix bytecode. 112 offset = IntPtrSub(BytecodeOffset(), IntPtrConstant(1)); 113 } 114 StoreAndTagRegister(offset, Register::bytecode_offset()); 115 } 116 117 Node* InterpreterAssembler::BytecodeArrayTaggedPointer() { 118 // Force a re-load of the bytecode array after every call in case the debugger 119 // has been activated. 120 if (!bytecode_array_valid_) { 121 bytecode_array_.Bind(LoadRegister(Register::bytecode_array())); 122 bytecode_array_valid_ = true; 123 } 124 return bytecode_array_.value(); 125 } 126 127 Node* InterpreterAssembler::DispatchTableRawPointer() { 128 if (Bytecodes::MakesCallAlongCriticalPath(bytecode_) && made_call_ && 129 (dispatch_table_.value() == 130 Parameter(InterpreterDispatchDescriptor::kDispatchTable))) { 131 dispatch_table_.Bind(ExternalConstant( 132 ExternalReference::interpreter_dispatch_table_address(isolate()))); 133 } 134 return dispatch_table_.value(); 135 } 136 137 Node* InterpreterAssembler::GetAccumulatorUnchecked() { 138 return accumulator_.value(); 139 } 140 141 Node* InterpreterAssembler::GetAccumulator() { 142 DCHECK(Bytecodes::ReadsAccumulator(bytecode_)); 143 accumulator_use_ = accumulator_use_ | AccumulatorUse::kRead; 144 return TaggedPoisonOnSpeculation(GetAccumulatorUnchecked()); 145 } 146 147 void InterpreterAssembler::SetAccumulator(Node* value) { 148 DCHECK(Bytecodes::WritesAccumulator(bytecode_)); 149 accumulator_use_ = accumulator_use_ | AccumulatorUse::kWrite; 150 accumulator_.Bind(value); 151 } 152 153 Node* InterpreterAssembler::GetContext() { 154 return LoadRegister(Register::current_context()); 155 } 156 157 void InterpreterAssembler::SetContext(Node* value) { 158 StoreRegister(value, Register::current_context()); 159 } 160 161 Node* InterpreterAssembler::GetContextAtDepth(Node* context, Node* depth) { 162 Variable cur_context(this, MachineRepresentation::kTaggedPointer); 163 cur_context.Bind(context); 164 165 Variable cur_depth(this, MachineRepresentation::kWord32); 166 cur_depth.Bind(depth); 167 168 Label context_found(this); 169 170 Variable* context_search_loop_variables[2] = {&cur_depth, &cur_context}; 171 Label context_search(this, 2, context_search_loop_variables); 172 173 // Fast path if the depth is 0. 174 Branch(Word32Equal(depth, Int32Constant(0)), &context_found, &context_search); 175 176 // Loop until the depth is 0. 177 BIND(&context_search); 178 { 179 cur_depth.Bind(Int32Sub(cur_depth.value(), Int32Constant(1))); 180 cur_context.Bind( 181 LoadContextElement(cur_context.value(), Context::PREVIOUS_INDEX)); 182 183 Branch(Word32Equal(cur_depth.value(), Int32Constant(0)), &context_found, 184 &context_search); 185 } 186 187 BIND(&context_found); 188 return cur_context.value(); 189 } 190 191 void InterpreterAssembler::GotoIfHasContextExtensionUpToDepth(Node* context, 192 Node* depth, 193 Label* target) { 194 Variable cur_context(this, MachineRepresentation::kTaggedPointer); 195 cur_context.Bind(context); 196 197 Variable cur_depth(this, MachineRepresentation::kWord32); 198 cur_depth.Bind(depth); 199 200 Variable* context_search_loop_variables[2] = {&cur_depth, &cur_context}; 201 Label context_search(this, 2, context_search_loop_variables); 202 203 // Loop until the depth is 0. 204 Goto(&context_search); 205 BIND(&context_search); 206 { 207 // TODO(leszeks): We only need to do this check if the context had a sloppy 208 // eval, we could pass in a context chain bitmask to figure out which 209 // contexts actually need to be checked. 210 211 Node* extension_slot = 212 LoadContextElement(cur_context.value(), Context::EXTENSION_INDEX); 213 214 // Jump to the target if the extension slot is not a hole. 215 GotoIf(WordNotEqual(extension_slot, TheHoleConstant()), target); 216 217 cur_depth.Bind(Int32Sub(cur_depth.value(), Int32Constant(1))); 218 cur_context.Bind( 219 LoadContextElement(cur_context.value(), Context::PREVIOUS_INDEX)); 220 221 GotoIf(Word32NotEqual(cur_depth.value(), Int32Constant(0)), 222 &context_search); 223 } 224 } 225 226 Node* InterpreterAssembler::RegisterLocation(Node* reg_index) { 227 return WordPoisonOnSpeculation( 228 IntPtrAdd(GetInterpretedFramePointer(), RegisterFrameOffset(reg_index))); 229 } 230 231 Node* InterpreterAssembler::RegisterLocation(Register reg) { 232 return RegisterLocation(IntPtrConstant(reg.ToOperand())); 233 } 234 235 Node* InterpreterAssembler::RegisterFrameOffset(Node* index) { 236 return TimesPointerSize(index); 237 } 238 239 Node* InterpreterAssembler::LoadRegister(Node* reg_index) { 240 return Load(MachineType::AnyTagged(), GetInterpretedFramePointer(), 241 RegisterFrameOffset(reg_index), LoadSensitivity::kCritical); 242 } 243 244 Node* InterpreterAssembler::LoadRegister(Register reg) { 245 return Load(MachineType::AnyTagged(), GetInterpretedFramePointer(), 246 IntPtrConstant(reg.ToOperand() << kPointerSizeLog2)); 247 } 248 249 Node* InterpreterAssembler::LoadAndUntagRegister(Register reg) { 250 return LoadAndUntagSmi(GetInterpretedFramePointer(), reg.ToOperand() 251 << kPointerSizeLog2); 252 } 253 254 Node* InterpreterAssembler::LoadRegisterAtOperandIndex(int operand_index) { 255 return LoadRegister( 256 BytecodeOperandReg(operand_index, LoadSensitivity::kSafe)); 257 } 258 259 std::pair<Node*, Node*> InterpreterAssembler::LoadRegisterPairAtOperandIndex( 260 int operand_index) { 261 DCHECK_EQ(OperandType::kRegPair, 262 Bytecodes::GetOperandType(bytecode_, operand_index)); 263 Node* first_reg_index = 264 BytecodeOperandReg(operand_index, LoadSensitivity::kSafe); 265 Node* second_reg_index = NextRegister(first_reg_index); 266 return std::make_pair(LoadRegister(first_reg_index), 267 LoadRegister(second_reg_index)); 268 } 269 270 InterpreterAssembler::RegListNodePair 271 InterpreterAssembler::GetRegisterListAtOperandIndex(int operand_index) { 272 DCHECK(Bytecodes::IsRegisterListOperandType( 273 Bytecodes::GetOperandType(bytecode_, operand_index))); 274 DCHECK_EQ(OperandType::kRegCount, 275 Bytecodes::GetOperandType(bytecode_, operand_index + 1)); 276 Node* base_reg = RegisterLocation( 277 BytecodeOperandReg(operand_index, LoadSensitivity::kSafe)); 278 Node* reg_count = BytecodeOperandCount(operand_index + 1); 279 return RegListNodePair(base_reg, reg_count); 280 } 281 282 Node* InterpreterAssembler::LoadRegisterFromRegisterList( 283 const RegListNodePair& reg_list, int index) { 284 Node* location = RegisterLocationInRegisterList(reg_list, index); 285 // Location is already poisoned on speculation, so no need to poison here. 286 return Load(MachineType::AnyTagged(), location); 287 } 288 289 Node* InterpreterAssembler::RegisterLocationInRegisterList( 290 const RegListNodePair& reg_list, int index) { 291 CSA_ASSERT(this, 292 Uint32GreaterThan(reg_list.reg_count(), Int32Constant(index))); 293 Node* offset = RegisterFrameOffset(IntPtrConstant(index)); 294 // Register indexes are negative, so subtract index from base location to get 295 // location. 296 return IntPtrSub(reg_list.base_reg_location(), offset); 297 } 298 299 void InterpreterAssembler::StoreRegister(Node* value, Register reg) { 300 StoreNoWriteBarrier( 301 MachineRepresentation::kTagged, GetInterpretedFramePointer(), 302 IntPtrConstant(reg.ToOperand() << kPointerSizeLog2), value); 303 } 304 305 void InterpreterAssembler::StoreRegister(Node* value, Node* reg_index) { 306 StoreNoWriteBarrier(MachineRepresentation::kTagged, 307 GetInterpretedFramePointer(), 308 RegisterFrameOffset(reg_index), value); 309 } 310 311 void InterpreterAssembler::StoreAndTagRegister(Node* value, Register reg) { 312 int offset = reg.ToOperand() << kPointerSizeLog2; 313 StoreAndTagSmi(GetInterpretedFramePointer(), offset, value); 314 } 315 316 void InterpreterAssembler::StoreRegisterAtOperandIndex(Node* value, 317 int operand_index) { 318 StoreRegister(value, 319 BytecodeOperandReg(operand_index, LoadSensitivity::kSafe)); 320 } 321 322 void InterpreterAssembler::StoreRegisterPairAtOperandIndex(Node* value1, 323 Node* value2, 324 int operand_index) { 325 DCHECK_EQ(OperandType::kRegOutPair, 326 Bytecodes::GetOperandType(bytecode_, operand_index)); 327 Node* first_reg_index = 328 BytecodeOperandReg(operand_index, LoadSensitivity::kSafe); 329 StoreRegister(value1, first_reg_index); 330 Node* second_reg_index = NextRegister(first_reg_index); 331 StoreRegister(value2, second_reg_index); 332 } 333 334 void InterpreterAssembler::StoreRegisterTripleAtOperandIndex( 335 Node* value1, Node* value2, Node* value3, int operand_index) { 336 DCHECK_EQ(OperandType::kRegOutTriple, 337 Bytecodes::GetOperandType(bytecode_, operand_index)); 338 Node* first_reg_index = 339 BytecodeOperandReg(operand_index, LoadSensitivity::kSafe); 340 StoreRegister(value1, first_reg_index); 341 Node* second_reg_index = NextRegister(first_reg_index); 342 StoreRegister(value2, second_reg_index); 343 Node* third_reg_index = NextRegister(second_reg_index); 344 StoreRegister(value3, third_reg_index); 345 } 346 347 Node* InterpreterAssembler::NextRegister(Node* reg_index) { 348 // Register indexes are negative, so the next index is minus one. 349 return IntPtrAdd(reg_index, IntPtrConstant(-1)); 350 } 351 352 Node* InterpreterAssembler::OperandOffset(int operand_index) { 353 return IntPtrConstant( 354 Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale())); 355 } 356 357 Node* InterpreterAssembler::BytecodeOperandUnsignedByte( 358 int operand_index, LoadSensitivity needs_poisoning) { 359 DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_)); 360 DCHECK_EQ(OperandSize::kByte, Bytecodes::GetOperandSize( 361 bytecode_, operand_index, operand_scale())); 362 Node* operand_offset = OperandOffset(operand_index); 363 return Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(), 364 IntPtrAdd(BytecodeOffset(), operand_offset), needs_poisoning); 365 } 366 367 Node* InterpreterAssembler::BytecodeOperandSignedByte( 368 int operand_index, LoadSensitivity needs_poisoning) { 369 DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_)); 370 DCHECK_EQ(OperandSize::kByte, Bytecodes::GetOperandSize( 371 bytecode_, operand_index, operand_scale())); 372 Node* operand_offset = OperandOffset(operand_index); 373 return Load(MachineType::Int8(), BytecodeArrayTaggedPointer(), 374 IntPtrAdd(BytecodeOffset(), operand_offset), needs_poisoning); 375 } 376 377 Node* InterpreterAssembler::BytecodeOperandReadUnaligned( 378 int relative_offset, MachineType result_type, 379 LoadSensitivity needs_poisoning) { 380 static const int kMaxCount = 4; 381 DCHECK(!TargetSupportsUnalignedAccess()); 382 383 int count; 384 switch (result_type.representation()) { 385 case MachineRepresentation::kWord16: 386 count = 2; 387 break; 388 case MachineRepresentation::kWord32: 389 count = 4; 390 break; 391 default: 392 UNREACHABLE(); 393 break; 394 } 395 MachineType msb_type = 396 result_type.IsSigned() ? MachineType::Int8() : MachineType::Uint8(); 397 398 #if V8_TARGET_LITTLE_ENDIAN 399 const int kStep = -1; 400 int msb_offset = count - 1; 401 #elif V8_TARGET_BIG_ENDIAN 402 const int kStep = 1; 403 int msb_offset = 0; 404 #else 405 #error "Unknown Architecture" 406 #endif 407 408 // Read the most signicant bytecode into bytes[0] and then in order 409 // down to least significant in bytes[count - 1]. 410 DCHECK_LE(count, kMaxCount); 411 Node* bytes[kMaxCount]; 412 for (int i = 0; i < count; i++) { 413 MachineType machine_type = (i == 0) ? msb_type : MachineType::Uint8(); 414 Node* offset = IntPtrConstant(relative_offset + msb_offset + i * kStep); 415 Node* array_offset = IntPtrAdd(BytecodeOffset(), offset); 416 bytes[i] = Load(machine_type, BytecodeArrayTaggedPointer(), array_offset, 417 needs_poisoning); 418 } 419 420 // Pack LSB to MSB. 421 Node* result = bytes[--count]; 422 for (int i = 1; --count >= 0; i++) { 423 Node* shift = Int32Constant(i * kBitsPerByte); 424 Node* value = Word32Shl(bytes[count], shift); 425 result = Word32Or(value, result); 426 } 427 return result; 428 } 429 430 Node* InterpreterAssembler::BytecodeOperandUnsignedShort( 431 int operand_index, LoadSensitivity needs_poisoning) { 432 DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_)); 433 DCHECK_EQ( 434 OperandSize::kShort, 435 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale())); 436 int operand_offset = 437 Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale()); 438 if (TargetSupportsUnalignedAccess()) { 439 return Load(MachineType::Uint16(), BytecodeArrayTaggedPointer(), 440 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)), 441 needs_poisoning); 442 } else { 443 return BytecodeOperandReadUnaligned(operand_offset, MachineType::Uint16(), 444 needs_poisoning); 445 } 446 } 447 448 Node* InterpreterAssembler::BytecodeOperandSignedShort( 449 int operand_index, LoadSensitivity needs_poisoning) { 450 DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_)); 451 DCHECK_EQ( 452 OperandSize::kShort, 453 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale())); 454 int operand_offset = 455 Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale()); 456 if (TargetSupportsUnalignedAccess()) { 457 return Load(MachineType::Int16(), BytecodeArrayTaggedPointer(), 458 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)), 459 needs_poisoning); 460 } else { 461 return BytecodeOperandReadUnaligned(operand_offset, MachineType::Int16(), 462 needs_poisoning); 463 } 464 } 465 466 Node* InterpreterAssembler::BytecodeOperandUnsignedQuad( 467 int operand_index, LoadSensitivity needs_poisoning) { 468 DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_)); 469 DCHECK_EQ(OperandSize::kQuad, Bytecodes::GetOperandSize( 470 bytecode_, operand_index, operand_scale())); 471 int operand_offset = 472 Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale()); 473 if (TargetSupportsUnalignedAccess()) { 474 return Load(MachineType::Uint32(), BytecodeArrayTaggedPointer(), 475 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)), 476 needs_poisoning); 477 } else { 478 return BytecodeOperandReadUnaligned(operand_offset, MachineType::Uint32(), 479 needs_poisoning); 480 } 481 } 482 483 Node* InterpreterAssembler::BytecodeOperandSignedQuad( 484 int operand_index, LoadSensitivity needs_poisoning) { 485 DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_)); 486 DCHECK_EQ(OperandSize::kQuad, Bytecodes::GetOperandSize( 487 bytecode_, operand_index, operand_scale())); 488 int operand_offset = 489 Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale()); 490 if (TargetSupportsUnalignedAccess()) { 491 return Load(MachineType::Int32(), BytecodeArrayTaggedPointer(), 492 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)), 493 needs_poisoning); 494 } else { 495 return BytecodeOperandReadUnaligned(operand_offset, MachineType::Int32(), 496 needs_poisoning); 497 } 498 } 499 500 Node* InterpreterAssembler::BytecodeSignedOperand( 501 int operand_index, OperandSize operand_size, 502 LoadSensitivity needs_poisoning) { 503 DCHECK(!Bytecodes::IsUnsignedOperandType( 504 Bytecodes::GetOperandType(bytecode_, operand_index))); 505 switch (operand_size) { 506 case OperandSize::kByte: 507 return BytecodeOperandSignedByte(operand_index, needs_poisoning); 508 case OperandSize::kShort: 509 return BytecodeOperandSignedShort(operand_index, needs_poisoning); 510 case OperandSize::kQuad: 511 return BytecodeOperandSignedQuad(operand_index, needs_poisoning); 512 case OperandSize::kNone: 513 UNREACHABLE(); 514 } 515 return nullptr; 516 } 517 518 Node* InterpreterAssembler::BytecodeUnsignedOperand( 519 int operand_index, OperandSize operand_size, 520 LoadSensitivity needs_poisoning) { 521 DCHECK(Bytecodes::IsUnsignedOperandType( 522 Bytecodes::GetOperandType(bytecode_, operand_index))); 523 switch (operand_size) { 524 case OperandSize::kByte: 525 return BytecodeOperandUnsignedByte(operand_index, needs_poisoning); 526 case OperandSize::kShort: 527 return BytecodeOperandUnsignedShort(operand_index, needs_poisoning); 528 case OperandSize::kQuad: 529 return BytecodeOperandUnsignedQuad(operand_index, needs_poisoning); 530 case OperandSize::kNone: 531 UNREACHABLE(); 532 } 533 return nullptr; 534 } 535 536 Node* InterpreterAssembler::BytecodeOperandCount(int operand_index) { 537 DCHECK_EQ(OperandType::kRegCount, 538 Bytecodes::GetOperandType(bytecode_, operand_index)); 539 OperandSize operand_size = 540 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 541 return BytecodeUnsignedOperand(operand_index, operand_size); 542 } 543 544 Node* InterpreterAssembler::BytecodeOperandFlag(int operand_index) { 545 DCHECK_EQ(OperandType::kFlag8, 546 Bytecodes::GetOperandType(bytecode_, operand_index)); 547 OperandSize operand_size = 548 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 549 DCHECK_EQ(operand_size, OperandSize::kByte); 550 return BytecodeUnsignedOperand(operand_index, operand_size); 551 } 552 553 Node* InterpreterAssembler::BytecodeOperandUImm(int operand_index) { 554 DCHECK_EQ(OperandType::kUImm, 555 Bytecodes::GetOperandType(bytecode_, operand_index)); 556 OperandSize operand_size = 557 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 558 return BytecodeUnsignedOperand(operand_index, operand_size); 559 } 560 561 Node* InterpreterAssembler::BytecodeOperandUImmWord(int operand_index) { 562 return ChangeUint32ToWord(BytecodeOperandUImm(operand_index)); 563 } 564 565 Node* InterpreterAssembler::BytecodeOperandUImmSmi(int operand_index) { 566 return SmiFromInt32(BytecodeOperandUImm(operand_index)); 567 } 568 569 Node* InterpreterAssembler::BytecodeOperandImm(int operand_index) { 570 DCHECK_EQ(OperandType::kImm, 571 Bytecodes::GetOperandType(bytecode_, operand_index)); 572 OperandSize operand_size = 573 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 574 return BytecodeSignedOperand(operand_index, operand_size); 575 } 576 577 Node* InterpreterAssembler::BytecodeOperandImmIntPtr(int operand_index) { 578 return ChangeInt32ToIntPtr(BytecodeOperandImm(operand_index)); 579 } 580 581 Node* InterpreterAssembler::BytecodeOperandImmSmi(int operand_index) { 582 return SmiFromInt32(BytecodeOperandImm(operand_index)); 583 } 584 585 Node* InterpreterAssembler::BytecodeOperandIdxInt32(int operand_index) { 586 DCHECK_EQ(OperandType::kIdx, 587 Bytecodes::GetOperandType(bytecode_, operand_index)); 588 OperandSize operand_size = 589 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 590 return BytecodeUnsignedOperand(operand_index, operand_size); 591 } 592 593 Node* InterpreterAssembler::BytecodeOperandIdx(int operand_index) { 594 return ChangeUint32ToWord(BytecodeOperandIdxInt32(operand_index)); 595 } 596 597 Node* InterpreterAssembler::BytecodeOperandIdxSmi(int operand_index) { 598 return SmiTag(BytecodeOperandIdx(operand_index)); 599 } 600 601 Node* InterpreterAssembler::BytecodeOperandConstantPoolIdx( 602 int operand_index, LoadSensitivity needs_poisoning) { 603 DCHECK_EQ(OperandType::kIdx, 604 Bytecodes::GetOperandType(bytecode_, operand_index)); 605 OperandSize operand_size = 606 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 607 return ChangeUint32ToWord( 608 BytecodeUnsignedOperand(operand_index, operand_size, needs_poisoning)); 609 } 610 611 Node* InterpreterAssembler::BytecodeOperandReg( 612 int operand_index, LoadSensitivity needs_poisoning) { 613 DCHECK(Bytecodes::IsRegisterOperandType( 614 Bytecodes::GetOperandType(bytecode_, operand_index))); 615 OperandSize operand_size = 616 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 617 return ChangeInt32ToIntPtr( 618 BytecodeSignedOperand(operand_index, operand_size, needs_poisoning)); 619 } 620 621 Node* InterpreterAssembler::BytecodeOperandRuntimeId(int operand_index) { 622 DCHECK_EQ(OperandType::kRuntimeId, 623 Bytecodes::GetOperandType(bytecode_, operand_index)); 624 OperandSize operand_size = 625 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 626 DCHECK_EQ(operand_size, OperandSize::kShort); 627 return BytecodeUnsignedOperand(operand_index, operand_size); 628 } 629 630 Node* InterpreterAssembler::BytecodeOperandNativeContextIndex( 631 int operand_index) { 632 DCHECK_EQ(OperandType::kNativeContextIndex, 633 Bytecodes::GetOperandType(bytecode_, operand_index)); 634 OperandSize operand_size = 635 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 636 return ChangeUint32ToWord( 637 BytecodeUnsignedOperand(operand_index, operand_size)); 638 } 639 640 Node* InterpreterAssembler::BytecodeOperandIntrinsicId(int operand_index) { 641 DCHECK_EQ(OperandType::kIntrinsicId, 642 Bytecodes::GetOperandType(bytecode_, operand_index)); 643 OperandSize operand_size = 644 Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()); 645 DCHECK_EQ(operand_size, OperandSize::kByte); 646 return BytecodeUnsignedOperand(operand_index, operand_size); 647 } 648 649 Node* InterpreterAssembler::LoadConstantPoolEntry(Node* index) { 650 TNode<FixedArray> constant_pool = CAST(LoadObjectField( 651 BytecodeArrayTaggedPointer(), BytecodeArray::kConstantPoolOffset)); 652 return LoadFixedArrayElement(constant_pool, UncheckedCast<IntPtrT>(index), 653 LoadSensitivity::kCritical); 654 } 655 656 Node* InterpreterAssembler::LoadAndUntagConstantPoolEntry(Node* index) { 657 return SmiUntag(LoadConstantPoolEntry(index)); 658 } 659 660 Node* InterpreterAssembler::LoadConstantPoolEntryAtOperandIndex( 661 int operand_index) { 662 Node* index = 663 BytecodeOperandConstantPoolIdx(operand_index, LoadSensitivity::kSafe); 664 return LoadConstantPoolEntry(index); 665 } 666 667 Node* InterpreterAssembler::LoadAndUntagConstantPoolEntryAtOperandIndex( 668 int operand_index) { 669 return SmiUntag(LoadConstantPoolEntryAtOperandIndex(operand_index)); 670 } 671 672 TNode<FeedbackVector> InterpreterAssembler::LoadFeedbackVector() { 673 TNode<JSFunction> function = CAST(LoadRegister(Register::function_closure())); 674 return CodeStubAssembler::LoadFeedbackVector(function); 675 } 676 677 void InterpreterAssembler::CallPrologue() { 678 if (!Bytecodes::MakesCallAlongCriticalPath(bytecode_)) { 679 // Bytecodes that make a call along the critical path save the bytecode 680 // offset in the bytecode handler's prologue. For other bytecodes, if 681 // there are multiple calls in the bytecode handler, you need to spill 682 // before each of them, unless SaveBytecodeOffset has explicitly been called 683 // in a path that dominates _all_ of those calls (which we don't track). 684 SaveBytecodeOffset(); 685 } 686 687 if (FLAG_debug_code && !disable_stack_check_across_call_) { 688 DCHECK_NULL(stack_pointer_before_call_); 689 stack_pointer_before_call_ = LoadStackPointer(); 690 } 691 bytecode_array_valid_ = false; 692 made_call_ = true; 693 } 694 695 void InterpreterAssembler::CallEpilogue() { 696 if (FLAG_debug_code && !disable_stack_check_across_call_) { 697 Node* stack_pointer_after_call = LoadStackPointer(); 698 Node* stack_pointer_before_call = stack_pointer_before_call_; 699 stack_pointer_before_call_ = nullptr; 700 AbortIfWordNotEqual(stack_pointer_before_call, stack_pointer_after_call, 701 AbortReason::kUnexpectedStackPointer); 702 } 703 } 704 705 void InterpreterAssembler::IncrementCallCount(Node* feedback_vector, 706 Node* slot_id) { 707 Comment("increment call count"); 708 TNode<Smi> call_count = 709 CAST(LoadFeedbackVectorSlot(feedback_vector, slot_id, kPointerSize)); 710 // The lowest {FeedbackNexus::CallCountField::kShift} bits of the call 711 // count are used as flags. To increment the call count by 1 we hence 712 // have to increment by 1 << {FeedbackNexus::CallCountField::kShift}. 713 Node* new_count = SmiAdd( 714 call_count, SmiConstant(1 << FeedbackNexus::CallCountField::kShift)); 715 // Count is Smi, so we don't need a write barrier. 716 StoreFeedbackVectorSlot(feedback_vector, slot_id, new_count, 717 SKIP_WRITE_BARRIER, kPointerSize); 718 } 719 720 void InterpreterAssembler::CollectCallableFeedback(Node* target, Node* context, 721 Node* feedback_vector, 722 Node* slot_id) { 723 Label extra_checks(this, Label::kDeferred), done(this); 724 725 // Check if we have monomorphic {target} feedback already. 726 TNode<MaybeObject> feedback = 727 LoadFeedbackVectorSlot(feedback_vector, slot_id); 728 Comment("check if monomorphic"); 729 TNode<BoolT> is_monomorphic = IsWeakReferenceTo(feedback, CAST(target)); 730 GotoIf(is_monomorphic, &done); 731 732 // Check if it is a megamorphic {target}. 733 Comment("check if megamorphic"); 734 Node* is_megamorphic = WordEqual( 735 feedback, HeapConstant(FeedbackVector::MegamorphicSentinel(isolate()))); 736 Branch(is_megamorphic, &done, &extra_checks); 737 738 BIND(&extra_checks); 739 { 740 Label initialize(this), mark_megamorphic(this); 741 742 Comment("check if weak reference"); 743 Node* is_uninitialized = WordEqual( 744 feedback, 745 HeapConstant(FeedbackVector::UninitializedSentinel(isolate()))); 746 GotoIf(is_uninitialized, &initialize); 747 CSA_ASSERT(this, IsWeakOrClearedHeapObject(feedback)); 748 749 // If the weak reference is cleared, we have a new chance to become 750 // monomorphic. 751 Comment("check if weak reference is cleared"); 752 Branch(IsClearedWeakHeapObject(feedback), &initialize, &mark_megamorphic); 753 754 BIND(&initialize); 755 { 756 // Check if {target} is a JSFunction in the current native context. 757 Comment("check if function in same native context"); 758 GotoIf(TaggedIsSmi(target), &mark_megamorphic); 759 // Check if the {target} is a JSFunction or JSBoundFunction 760 // in the current native context. 761 VARIABLE(var_current, MachineRepresentation::kTagged, target); 762 Label loop(this, &var_current), done_loop(this); 763 Goto(&loop); 764 BIND(&loop); 765 { 766 Label if_boundfunction(this), if_function(this); 767 Node* current = var_current.value(); 768 CSA_ASSERT(this, TaggedIsNotSmi(current)); 769 Node* current_instance_type = LoadInstanceType(current); 770 GotoIf(InstanceTypeEqual(current_instance_type, JS_BOUND_FUNCTION_TYPE), 771 &if_boundfunction); 772 Branch(InstanceTypeEqual(current_instance_type, JS_FUNCTION_TYPE), 773 &if_function, &mark_megamorphic); 774 775 BIND(&if_function); 776 { 777 // Check that the JSFunction {current} is in the current native 778 // context. 779 Node* current_context = 780 LoadObjectField(current, JSFunction::kContextOffset); 781 Node* current_native_context = LoadNativeContext(current_context); 782 Branch(WordEqual(LoadNativeContext(context), current_native_context), 783 &done_loop, &mark_megamorphic); 784 } 785 786 BIND(&if_boundfunction); 787 { 788 // Continue with the [[BoundTargetFunction]] of {target}. 789 var_current.Bind(LoadObjectField( 790 current, JSBoundFunction::kBoundTargetFunctionOffset)); 791 Goto(&loop); 792 } 793 } 794 BIND(&done_loop); 795 StoreWeakReferenceInFeedbackVector(feedback_vector, slot_id, 796 CAST(target)); 797 ReportFeedbackUpdate(feedback_vector, slot_id, "Call:Initialize"); 798 Goto(&done); 799 } 800 801 BIND(&mark_megamorphic); 802 { 803 // MegamorphicSentinel is an immortal immovable object so 804 // write-barrier is not needed. 805 Comment("transition to megamorphic"); 806 DCHECK(Heap::RootIsImmortalImmovable(Heap::kmegamorphic_symbolRootIndex)); 807 StoreFeedbackVectorSlot( 808 feedback_vector, slot_id, 809 HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())), 810 SKIP_WRITE_BARRIER); 811 ReportFeedbackUpdate(feedback_vector, slot_id, 812 "Call:TransitionMegamorphic"); 813 Goto(&done); 814 } 815 } 816 817 BIND(&done); 818 } 819 820 void InterpreterAssembler::CollectCallFeedback(Node* target, Node* context, 821 Node* feedback_vector, 822 Node* slot_id) { 823 // Increment the call count. 824 IncrementCallCount(feedback_vector, slot_id); 825 826 // Collect the callable {target} feedback. 827 CollectCallableFeedback(target, context, feedback_vector, slot_id); 828 } 829 830 void InterpreterAssembler::CallJSAndDispatch( 831 Node* function, Node* context, const RegListNodePair& args, 832 ConvertReceiverMode receiver_mode) { 833 DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_)); 834 DCHECK(Bytecodes::IsCallOrConstruct(bytecode_) || 835 bytecode_ == Bytecode::kInvokeIntrinsic); 836 DCHECK_EQ(Bytecodes::GetReceiverMode(bytecode_), receiver_mode); 837 838 Node* args_count; 839 if (receiver_mode == ConvertReceiverMode::kNullOrUndefined) { 840 // The receiver is implied, so it is not in the argument list. 841 args_count = args.reg_count(); 842 } else { 843 // Subtract the receiver from the argument count. 844 Node* receiver_count = Int32Constant(1); 845 args_count = Int32Sub(args.reg_count(), receiver_count); 846 } 847 848 Callable callable = CodeFactory::InterpreterPushArgsThenCall( 849 isolate(), receiver_mode, InterpreterPushArgsMode::kOther); 850 Node* code_target = HeapConstant(callable.code()); 851 852 TailCallStubThenBytecodeDispatch(callable.descriptor(), code_target, context, 853 args_count, args.base_reg_location(), 854 function); 855 // TailCallStubThenDispatch updates accumulator with result. 856 accumulator_use_ = accumulator_use_ | AccumulatorUse::kWrite; 857 } 858 859 template <class... TArgs> 860 void InterpreterAssembler::CallJSAndDispatch(Node* function, Node* context, 861 Node* arg_count, 862 ConvertReceiverMode receiver_mode, 863 TArgs... args) { 864 DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_)); 865 DCHECK(Bytecodes::IsCallOrConstruct(bytecode_) || 866 bytecode_ == Bytecode::kInvokeIntrinsic); 867 DCHECK_EQ(Bytecodes::GetReceiverMode(bytecode_), receiver_mode); 868 Callable callable = CodeFactory::Call(isolate()); 869 Node* code_target = HeapConstant(callable.code()); 870 871 if (receiver_mode == ConvertReceiverMode::kNullOrUndefined) { 872 // The first argument parameter (the receiver) is implied to be undefined. 873 TailCallStubThenBytecodeDispatch( 874 callable.descriptor(), code_target, context, function, arg_count, 875 static_cast<Node*>(UndefinedConstant()), args...); 876 } else { 877 TailCallStubThenBytecodeDispatch(callable.descriptor(), code_target, 878 context, function, arg_count, args...); 879 } 880 // TailCallStubThenDispatch updates accumulator with result. 881 accumulator_use_ = accumulator_use_ | AccumulatorUse::kWrite; 882 } 883 884 // Instantiate CallJSAndDispatch() for argument counts used by interpreter 885 // generator. 886 template V8_EXPORT_PRIVATE void InterpreterAssembler::CallJSAndDispatch( 887 Node* function, Node* context, Node* arg_count, 888 ConvertReceiverMode receiver_mode); 889 template V8_EXPORT_PRIVATE void InterpreterAssembler::CallJSAndDispatch( 890 Node* function, Node* context, Node* arg_count, 891 ConvertReceiverMode receiver_mode, Node*); 892 template V8_EXPORT_PRIVATE void InterpreterAssembler::CallJSAndDispatch( 893 Node* function, Node* context, Node* arg_count, 894 ConvertReceiverMode receiver_mode, Node*, Node*); 895 template V8_EXPORT_PRIVATE void InterpreterAssembler::CallJSAndDispatch( 896 Node* function, Node* context, Node* arg_count, 897 ConvertReceiverMode receiver_mode, Node*, Node*, Node*); 898 899 void InterpreterAssembler::CallJSWithSpreadAndDispatch( 900 Node* function, Node* context, const RegListNodePair& args, Node* slot_id, 901 Node* feedback_vector) { 902 DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_)); 903 DCHECK_EQ(Bytecodes::GetReceiverMode(bytecode_), ConvertReceiverMode::kAny); 904 CollectCallFeedback(function, context, feedback_vector, slot_id); 905 Comment("call using CallWithSpread builtin"); 906 Callable callable = CodeFactory::InterpreterPushArgsThenCall( 907 isolate(), ConvertReceiverMode::kAny, 908 InterpreterPushArgsMode::kWithFinalSpread); 909 Node* code_target = HeapConstant(callable.code()); 910 911 Node* receiver_count = Int32Constant(1); 912 Node* args_count = Int32Sub(args.reg_count(), receiver_count); 913 TailCallStubThenBytecodeDispatch(callable.descriptor(), code_target, context, 914 args_count, args.base_reg_location(), 915 function); 916 // TailCallStubThenDispatch updates accumulator with result. 917 accumulator_use_ = accumulator_use_ | AccumulatorUse::kWrite; 918 } 919 920 Node* InterpreterAssembler::Construct(Node* target, Node* context, 921 Node* new_target, 922 const RegListNodePair& args, 923 Node* slot_id, Node* feedback_vector) { 924 DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_)); 925 VARIABLE(var_result, MachineRepresentation::kTagged); 926 VARIABLE(var_site, MachineRepresentation::kTagged); 927 Label extra_checks(this, Label::kDeferred), return_result(this, &var_result), 928 construct(this), construct_array(this, &var_site); 929 930 // Increment the call count. 931 IncrementCallCount(feedback_vector, slot_id); 932 933 // Check if we have monomorphic {new_target} feedback already. 934 TNode<MaybeObject> feedback = 935 LoadFeedbackVectorSlot(feedback_vector, slot_id); 936 Branch(IsWeakReferenceTo(feedback, CAST(new_target)), &construct, 937 &extra_checks); 938 939 BIND(&extra_checks); 940 { 941 Label check_allocation_site(this), check_initialized(this), 942 initialize(this), mark_megamorphic(this); 943 944 // Check if it is a megamorphic {new_target}.. 945 Comment("check if megamorphic"); 946 Node* is_megamorphic = WordEqual( 947 feedback, HeapConstant(FeedbackVector::MegamorphicSentinel(isolate()))); 948 GotoIf(is_megamorphic, &construct); 949 950 Comment("check if weak reference"); 951 GotoIfNot(IsWeakOrClearedHeapObject(feedback), &check_allocation_site); 952 953 // If the weak reference is cleared, we have a new chance to become 954 // monomorphic. 955 Comment("check if weak reference is cleared"); 956 Branch(IsClearedWeakHeapObject(feedback), &initialize, &mark_megamorphic); 957 958 BIND(&check_allocation_site); 959 { 960 // Check if it is an AllocationSite. 961 Comment("check if allocation site"); 962 TNode<HeapObject> strong_feedback = CAST(feedback); 963 GotoIfNot(IsAllocationSite(strong_feedback), &check_initialized); 964 965 // Make sure that {target} and {new_target} are the Array constructor. 966 Node* array_function = LoadContextElement(LoadNativeContext(context), 967 Context::ARRAY_FUNCTION_INDEX); 968 GotoIfNot(WordEqual(target, array_function), &mark_megamorphic); 969 GotoIfNot(WordEqual(new_target, array_function), &mark_megamorphic); 970 var_site.Bind(strong_feedback); 971 Goto(&construct_array); 972 } 973 974 BIND(&check_initialized); 975 { 976 // Check if it is uninitialized. 977 Comment("check if uninitialized"); 978 Node* is_uninitialized = 979 WordEqual(feedback, LoadRoot(Heap::kuninitialized_symbolRootIndex)); 980 Branch(is_uninitialized, &initialize, &mark_megamorphic); 981 } 982 983 BIND(&initialize); 984 { 985 Comment("check if function in same native context"); 986 GotoIf(TaggedIsSmi(new_target), &mark_megamorphic); 987 // Check if the {new_target} is a JSFunction or JSBoundFunction 988 // in the current native context. 989 VARIABLE(var_current, MachineRepresentation::kTagged, new_target); 990 Label loop(this, &var_current), done_loop(this); 991 Goto(&loop); 992 BIND(&loop); 993 { 994 Label if_boundfunction(this), if_function(this); 995 Node* current = var_current.value(); 996 CSA_ASSERT(this, TaggedIsNotSmi(current)); 997 Node* current_instance_type = LoadInstanceType(current); 998 GotoIf(InstanceTypeEqual(current_instance_type, JS_BOUND_FUNCTION_TYPE), 999 &if_boundfunction); 1000 Branch(InstanceTypeEqual(current_instance_type, JS_FUNCTION_TYPE), 1001 &if_function, &mark_megamorphic); 1002 1003 BIND(&if_function); 1004 { 1005 // Check that the JSFunction {current} is in the current native 1006 // context. 1007 Node* current_context = 1008 LoadObjectField(current, JSFunction::kContextOffset); 1009 Node* current_native_context = LoadNativeContext(current_context); 1010 Branch(WordEqual(LoadNativeContext(context), current_native_context), 1011 &done_loop, &mark_megamorphic); 1012 } 1013 1014 BIND(&if_boundfunction); 1015 { 1016 // Continue with the [[BoundTargetFunction]] of {current}. 1017 var_current.Bind(LoadObjectField( 1018 current, JSBoundFunction::kBoundTargetFunctionOffset)); 1019 Goto(&loop); 1020 } 1021 } 1022 BIND(&done_loop); 1023 1024 // Create an AllocationSite if {target} and {new_target} refer 1025 // to the current native context's Array constructor. 1026 Label create_allocation_site(this), store_weak_reference(this); 1027 GotoIfNot(WordEqual(target, new_target), &store_weak_reference); 1028 Node* array_function = LoadContextElement(LoadNativeContext(context), 1029 Context::ARRAY_FUNCTION_INDEX); 1030 Branch(WordEqual(target, array_function), &create_allocation_site, 1031 &store_weak_reference); 1032 1033 BIND(&create_allocation_site); 1034 { 1035 var_site.Bind(CreateAllocationSiteInFeedbackVector(feedback_vector, 1036 SmiTag(slot_id))); 1037 ReportFeedbackUpdate(feedback_vector, slot_id, 1038 "Construct:CreateAllocationSite"); 1039 Goto(&construct_array); 1040 } 1041 1042 BIND(&store_weak_reference); 1043 { 1044 StoreWeakReferenceInFeedbackVector(feedback_vector, slot_id, 1045 CAST(new_target)); 1046 ReportFeedbackUpdate(feedback_vector, slot_id, 1047 "Construct:StoreWeakReference"); 1048 Goto(&construct); 1049 } 1050 } 1051 1052 BIND(&mark_megamorphic); 1053 { 1054 // MegamorphicSentinel is an immortal immovable object so 1055 // write-barrier is not needed. 1056 Comment("transition to megamorphic"); 1057 DCHECK(Heap::RootIsImmortalImmovable(Heap::kmegamorphic_symbolRootIndex)); 1058 StoreFeedbackVectorSlot( 1059 feedback_vector, slot_id, 1060 HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())), 1061 SKIP_WRITE_BARRIER); 1062 ReportFeedbackUpdate(feedback_vector, slot_id, 1063 "Construct:TransitionMegamorphic"); 1064 Goto(&construct); 1065 } 1066 } 1067 1068 BIND(&construct_array); 1069 { 1070 // TODO(bmeurer): Introduce a dedicated builtin to deal with the Array 1071 // constructor feedback collection inside of Ignition. 1072 Comment("call using ConstructArray builtin"); 1073 Callable callable = CodeFactory::InterpreterPushArgsThenConstruct( 1074 isolate(), InterpreterPushArgsMode::kArrayFunction); 1075 Node* code_target = HeapConstant(callable.code()); 1076 var_result.Bind(CallStub(callable.descriptor(), code_target, context, 1077 args.reg_count(), new_target, target, 1078 var_site.value(), args.base_reg_location())); 1079 Goto(&return_result); 1080 } 1081 1082 BIND(&construct); 1083 { 1084 // TODO(bmeurer): Remove the generic type_info parameter from the Construct. 1085 Comment("call using Construct builtin"); 1086 Callable callable = CodeFactory::InterpreterPushArgsThenConstruct( 1087 isolate(), InterpreterPushArgsMode::kOther); 1088 Node* code_target = HeapConstant(callable.code()); 1089 var_result.Bind(CallStub(callable.descriptor(), code_target, context, 1090 args.reg_count(), new_target, target, 1091 UndefinedConstant(), args.base_reg_location())); 1092 Goto(&return_result); 1093 } 1094 1095 BIND(&return_result); 1096 return var_result.value(); 1097 } 1098 1099 Node* InterpreterAssembler::ConstructWithSpread(Node* target, Node* context, 1100 Node* new_target, 1101 const RegListNodePair& args, 1102 Node* slot_id, 1103 Node* feedback_vector) { 1104 // TODO(bmeurer): Unify this with the Construct bytecode feedback 1105 // above once we have a way to pass the AllocationSite to the Array 1106 // constructor _and_ spread the last argument at the same time. 1107 DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_)); 1108 Label extra_checks(this, Label::kDeferred), construct(this); 1109 1110 // Increment the call count. 1111 IncrementCallCount(feedback_vector, slot_id); 1112 1113 // Check if we have monomorphic {new_target} feedback already. 1114 TNode<MaybeObject> feedback = 1115 LoadFeedbackVectorSlot(feedback_vector, slot_id); 1116 Branch(IsWeakReferenceTo(feedback, CAST(new_target)), &construct, 1117 &extra_checks); 1118 1119 BIND(&extra_checks); 1120 { 1121 Label check_initialized(this), initialize(this), mark_megamorphic(this); 1122 1123 // Check if it is a megamorphic {new_target}. 1124 Comment("check if megamorphic"); 1125 Node* is_megamorphic = WordEqual( 1126 feedback, HeapConstant(FeedbackVector::MegamorphicSentinel(isolate()))); 1127 GotoIf(is_megamorphic, &construct); 1128 1129 Comment("check if weak reference"); 1130 GotoIfNot(IsWeakOrClearedHeapObject(feedback), &check_initialized); 1131 1132 // If the weak reference is cleared, we have a new chance to become 1133 // monomorphic. 1134 Comment("check if weak reference is cleared"); 1135 Branch(IsClearedWeakHeapObject(feedback), &initialize, &mark_megamorphic); 1136 1137 BIND(&check_initialized); 1138 { 1139 // Check if it is uninitialized. 1140 Comment("check if uninitialized"); 1141 Node* is_uninitialized = 1142 WordEqual(feedback, LoadRoot(Heap::kuninitialized_symbolRootIndex)); 1143 Branch(is_uninitialized, &initialize, &mark_megamorphic); 1144 } 1145 1146 BIND(&initialize); 1147 { 1148 Comment("check if function in same native context"); 1149 GotoIf(TaggedIsSmi(new_target), &mark_megamorphic); 1150 // Check if the {new_target} is a JSFunction or JSBoundFunction 1151 // in the current native context. 1152 VARIABLE(var_current, MachineRepresentation::kTagged, new_target); 1153 Label loop(this, &var_current), done_loop(this); 1154 Goto(&loop); 1155 BIND(&loop); 1156 { 1157 Label if_boundfunction(this), if_function(this); 1158 Node* current = var_current.value(); 1159 CSA_ASSERT(this, TaggedIsNotSmi(current)); 1160 Node* current_instance_type = LoadInstanceType(current); 1161 GotoIf(InstanceTypeEqual(current_instance_type, JS_BOUND_FUNCTION_TYPE), 1162 &if_boundfunction); 1163 Branch(InstanceTypeEqual(current_instance_type, JS_FUNCTION_TYPE), 1164 &if_function, &mark_megamorphic); 1165 1166 BIND(&if_function); 1167 { 1168 // Check that the JSFunction {current} is in the current native 1169 // context. 1170 Node* current_context = 1171 LoadObjectField(current, JSFunction::kContextOffset); 1172 Node* current_native_context = LoadNativeContext(current_context); 1173 Branch(WordEqual(LoadNativeContext(context), current_native_context), 1174 &done_loop, &mark_megamorphic); 1175 } 1176 1177 BIND(&if_boundfunction); 1178 { 1179 // Continue with the [[BoundTargetFunction]] of {current}. 1180 var_current.Bind(LoadObjectField( 1181 current, JSBoundFunction::kBoundTargetFunctionOffset)); 1182 Goto(&loop); 1183 } 1184 } 1185 BIND(&done_loop); 1186 StoreWeakReferenceInFeedbackVector(feedback_vector, slot_id, 1187 CAST(new_target)); 1188 ReportFeedbackUpdate(feedback_vector, slot_id, 1189 "ConstructWithSpread:Initialize"); 1190 Goto(&construct); 1191 } 1192 1193 BIND(&mark_megamorphic); 1194 { 1195 // MegamorphicSentinel is an immortal immovable object so 1196 // write-barrier is not needed. 1197 Comment("transition to megamorphic"); 1198 DCHECK(Heap::RootIsImmortalImmovable(Heap::kmegamorphic_symbolRootIndex)); 1199 StoreFeedbackVectorSlot( 1200 feedback_vector, slot_id, 1201 HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())), 1202 SKIP_WRITE_BARRIER); 1203 ReportFeedbackUpdate(feedback_vector, slot_id, 1204 "ConstructWithSpread:TransitionMegamorphic"); 1205 Goto(&construct); 1206 } 1207 } 1208 1209 BIND(&construct); 1210 Comment("call using ConstructWithSpread builtin"); 1211 Callable callable = CodeFactory::InterpreterPushArgsThenConstruct( 1212 isolate(), InterpreterPushArgsMode::kWithFinalSpread); 1213 Node* code_target = HeapConstant(callable.code()); 1214 return CallStub(callable.descriptor(), code_target, context, args.reg_count(), 1215 new_target, target, UndefinedConstant(), 1216 args.base_reg_location()); 1217 } 1218 1219 Node* InterpreterAssembler::CallRuntimeN(Node* function_id, Node* context, 1220 const RegListNodePair& args, 1221 int result_size) { 1222 DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_)); 1223 DCHECK(Bytecodes::IsCallRuntime(bytecode_)); 1224 Callable callable = CodeFactory::InterpreterCEntry(isolate(), result_size); 1225 Node* code_target = HeapConstant(callable.code()); 1226 1227 // Get the function entry from the function id. 1228 Node* function_table = ExternalConstant( 1229 ExternalReference::runtime_function_table_address(isolate())); 1230 Node* function_offset = 1231 Int32Mul(function_id, Int32Constant(sizeof(Runtime::Function))); 1232 Node* function = 1233 IntPtrAdd(function_table, ChangeUint32ToWord(function_offset)); 1234 Node* function_entry = 1235 Load(MachineType::Pointer(), function, 1236 IntPtrConstant(offsetof(Runtime::Function, entry))); 1237 1238 return CallStubR(callable.descriptor(), result_size, code_target, context, 1239 args.reg_count(), args.base_reg_location(), function_entry); 1240 } 1241 1242 void InterpreterAssembler::UpdateInterruptBudget(Node* weight, bool backward) { 1243 Comment("[ UpdateInterruptBudget"); 1244 1245 Node* budget_offset = 1246 IntPtrConstant(BytecodeArray::kInterruptBudgetOffset - kHeapObjectTag); 1247 1248 // Assert that the weight is positive (negative weights should be implemented 1249 // as backward updates). 1250 CSA_ASSERT(this, Int32GreaterThanOrEqual(weight, Int32Constant(0))); 1251 1252 // Update budget by |weight| and check if it reaches zero. 1253 Variable new_budget(this, MachineRepresentation::kWord32); 1254 Node* old_budget = 1255 Load(MachineType::Int32(), BytecodeArrayTaggedPointer(), budget_offset); 1256 // Make sure we include the current bytecode in the budget calculation. 1257 Node* budget_after_bytecode = 1258 Int32Sub(old_budget, Int32Constant(CurrentBytecodeSize())); 1259 1260 if (backward) { 1261 new_budget.Bind(Int32Sub(budget_after_bytecode, weight)); 1262 1263 Node* condition = 1264 Int32GreaterThanOrEqual(new_budget.value(), Int32Constant(0)); 1265 Label ok(this), interrupt_check(this, Label::kDeferred); 1266 Branch(condition, &ok, &interrupt_check); 1267 1268 // Perform interrupt and reset budget. 1269 BIND(&interrupt_check); 1270 { 1271 CallRuntime(Runtime::kInterrupt, GetContext()); 1272 new_budget.Bind(Int32Constant(Interpreter::InterruptBudget())); 1273 Goto(&ok); 1274 } 1275 1276 BIND(&ok); 1277 } else { 1278 // For a forward jump, we know we only increase the interrupt budget, so 1279 // no need to check if it's below zero. 1280 new_budget.Bind(Int32Add(budget_after_bytecode, weight)); 1281 } 1282 1283 // Update budget. 1284 StoreNoWriteBarrier(MachineRepresentation::kWord32, 1285 BytecodeArrayTaggedPointer(), budget_offset, 1286 new_budget.value()); 1287 Comment("] UpdateInterruptBudget"); 1288 } 1289 1290 Node* InterpreterAssembler::Advance() { return Advance(CurrentBytecodeSize()); } 1291 1292 Node* InterpreterAssembler::Advance(int delta) { 1293 return Advance(IntPtrConstant(delta)); 1294 } 1295 1296 Node* InterpreterAssembler::Advance(Node* delta, bool backward) { 1297 #ifdef V8_TRACE_IGNITION 1298 TraceBytecode(Runtime::kInterpreterTraceBytecodeExit); 1299 #endif 1300 Node* next_offset = backward ? IntPtrSub(BytecodeOffset(), delta) 1301 : IntPtrAdd(BytecodeOffset(), delta); 1302 bytecode_offset_.Bind(next_offset); 1303 return next_offset; 1304 } 1305 1306 Node* InterpreterAssembler::Jump(Node* delta, bool backward) { 1307 DCHECK(!Bytecodes::IsStarLookahead(bytecode_, operand_scale_)); 1308 1309 UpdateInterruptBudget(TruncateIntPtrToInt32(delta), backward); 1310 Node* new_bytecode_offset = Advance(delta, backward); 1311 Node* target_bytecode = LoadBytecode(new_bytecode_offset); 1312 return DispatchToBytecode(target_bytecode, new_bytecode_offset); 1313 } 1314 1315 Node* InterpreterAssembler::Jump(Node* delta) { return Jump(delta, false); } 1316 1317 Node* InterpreterAssembler::JumpBackward(Node* delta) { 1318 return Jump(delta, true); 1319 } 1320 1321 void InterpreterAssembler::JumpConditional(Node* condition, Node* delta) { 1322 Label match(this), no_match(this); 1323 1324 Branch(condition, &match, &no_match); 1325 BIND(&match); 1326 Jump(delta); 1327 BIND(&no_match); 1328 Dispatch(); 1329 } 1330 1331 void InterpreterAssembler::JumpIfWordEqual(Node* lhs, Node* rhs, Node* delta) { 1332 JumpConditional(WordEqual(lhs, rhs), delta); 1333 } 1334 1335 void InterpreterAssembler::JumpIfWordNotEqual(Node* lhs, Node* rhs, 1336 Node* delta) { 1337 JumpConditional(WordNotEqual(lhs, rhs), delta); 1338 } 1339 1340 Node* InterpreterAssembler::LoadBytecode(Node* bytecode_offset) { 1341 Node* bytecode = 1342 Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(), bytecode_offset); 1343 return ChangeUint32ToWord(bytecode); 1344 } 1345 1346 Node* InterpreterAssembler::StarDispatchLookahead(Node* target_bytecode) { 1347 Label do_inline_star(this), done(this); 1348 1349 Variable var_bytecode(this, MachineType::PointerRepresentation()); 1350 var_bytecode.Bind(target_bytecode); 1351 1352 Node* star_bytecode = IntPtrConstant(static_cast<int>(Bytecode::kStar)); 1353 Node* is_star = WordEqual(target_bytecode, star_bytecode); 1354 Branch(is_star, &do_inline_star, &done); 1355 1356 BIND(&do_inline_star); 1357 { 1358 InlineStar(); 1359 var_bytecode.Bind(LoadBytecode(BytecodeOffset())); 1360 Goto(&done); 1361 } 1362 BIND(&done); 1363 return var_bytecode.value(); 1364 } 1365 1366 void InterpreterAssembler::InlineStar() { 1367 Bytecode previous_bytecode = bytecode_; 1368 AccumulatorUse previous_acc_use = accumulator_use_; 1369 1370 bytecode_ = Bytecode::kStar; 1371 accumulator_use_ = AccumulatorUse::kNone; 1372 1373 #ifdef V8_TRACE_IGNITION 1374 TraceBytecode(Runtime::kInterpreterTraceBytecodeEntry); 1375 #endif 1376 StoreRegister(GetAccumulator(), 1377 BytecodeOperandReg(0, LoadSensitivity::kSafe)); 1378 1379 DCHECK_EQ(accumulator_use_, Bytecodes::GetAccumulatorUse(bytecode_)); 1380 1381 Advance(); 1382 bytecode_ = previous_bytecode; 1383 accumulator_use_ = previous_acc_use; 1384 } 1385 1386 Node* InterpreterAssembler::Dispatch() { 1387 Comment("========= Dispatch"); 1388 DCHECK_IMPLIES(Bytecodes::MakesCallAlongCriticalPath(bytecode_), made_call_); 1389 Node* target_offset = Advance(); 1390 Node* target_bytecode = LoadBytecode(target_offset); 1391 1392 if (Bytecodes::IsStarLookahead(bytecode_, operand_scale_)) { 1393 target_bytecode = StarDispatchLookahead(target_bytecode); 1394 } 1395 return DispatchToBytecode(target_bytecode, BytecodeOffset()); 1396 } 1397 1398 Node* InterpreterAssembler::DispatchToBytecode(Node* target_bytecode, 1399 Node* new_bytecode_offset) { 1400 if (FLAG_trace_ignition_dispatches) { 1401 TraceBytecodeDispatch(target_bytecode); 1402 } 1403 1404 Node* target_code_entry = 1405 Load(MachineType::Pointer(), DispatchTableRawPointer(), 1406 TimesPointerSize(target_bytecode)); 1407 1408 return DispatchToBytecodeHandlerEntry(target_code_entry, new_bytecode_offset, 1409 target_bytecode); 1410 } 1411 1412 Node* InterpreterAssembler::DispatchToBytecodeHandler(Node* handler, 1413 Node* bytecode_offset, 1414 Node* target_bytecode) { 1415 // TODO(ishell): Add CSA::CodeEntryPoint(code). 1416 Node* handler_entry = 1417 IntPtrAdd(BitcastTaggedToWord(handler), 1418 IntPtrConstant(Code::kHeaderSize - kHeapObjectTag)); 1419 return DispatchToBytecodeHandlerEntry(handler_entry, bytecode_offset, 1420 target_bytecode); 1421 } 1422 1423 Node* InterpreterAssembler::DispatchToBytecodeHandlerEntry( 1424 Node* handler_entry, Node* bytecode_offset, Node* target_bytecode) { 1425 // Propagate speculation poisoning. 1426 Node* poisoned_handler_entry = WordPoisonOnSpeculation(handler_entry); 1427 return TailCallBytecodeDispatch( 1428 InterpreterDispatchDescriptor{}, poisoned_handler_entry, 1429 GetAccumulatorUnchecked(), bytecode_offset, BytecodeArrayTaggedPointer(), 1430 DispatchTableRawPointer()); 1431 } 1432 1433 void InterpreterAssembler::DispatchWide(OperandScale operand_scale) { 1434 // Dispatching a wide bytecode requires treating the prefix 1435 // bytecode a base pointer into the dispatch table and dispatching 1436 // the bytecode that follows relative to this base. 1437 // 1438 // Indices 0-255 correspond to bytecodes with operand_scale == 0 1439 // Indices 256-511 correspond to bytecodes with operand_scale == 1 1440 // Indices 512-767 correspond to bytecodes with operand_scale == 2 1441 DCHECK_IMPLIES(Bytecodes::MakesCallAlongCriticalPath(bytecode_), made_call_); 1442 Node* next_bytecode_offset = Advance(1); 1443 Node* next_bytecode = LoadBytecode(next_bytecode_offset); 1444 1445 if (FLAG_trace_ignition_dispatches) { 1446 TraceBytecodeDispatch(next_bytecode); 1447 } 1448 1449 Node* base_index; 1450 switch (operand_scale) { 1451 case OperandScale::kDouble: 1452 base_index = IntPtrConstant(1 << kBitsPerByte); 1453 break; 1454 case OperandScale::kQuadruple: 1455 base_index = IntPtrConstant(2 << kBitsPerByte); 1456 break; 1457 default: 1458 UNREACHABLE(); 1459 } 1460 Node* target_index = IntPtrAdd(base_index, next_bytecode); 1461 Node* target_code_entry = 1462 Load(MachineType::Pointer(), DispatchTableRawPointer(), 1463 TimesPointerSize(target_index)); 1464 1465 DispatchToBytecodeHandlerEntry(target_code_entry, next_bytecode_offset, 1466 next_bytecode); 1467 } 1468 1469 void InterpreterAssembler::UpdateInterruptBudgetOnReturn() { 1470 // TODO(rmcilroy): Investigate whether it is worth supporting self 1471 // optimization of primitive functions like FullCodegen. 1472 1473 // Update profiling count by the number of bytes between the end of the 1474 // current bytecode and the start of the first one, to simulate backedge to 1475 // start of function. 1476 // 1477 // With headers and current offset, the bytecode array layout looks like: 1478 // 1479 // <---------- simulated backedge ---------- 1480 // | header | first bytecode | .... | return bytecode | 1481 // |<------ current offset -------> 1482 // ^ tagged bytecode array pointer 1483 // 1484 // UpdateInterruptBudget already handles adding the bytecode size to the 1485 // length of the back-edge, so we just have to correct for the non-zero offset 1486 // of the first bytecode. 1487 1488 const int kFirstBytecodeOffset = BytecodeArray::kHeaderSize - kHeapObjectTag; 1489 Node* profiling_weight = Int32Sub(TruncateIntPtrToInt32(BytecodeOffset()), 1490 Int32Constant(kFirstBytecodeOffset)); 1491 UpdateInterruptBudget(profiling_weight, true); 1492 } 1493 1494 Node* InterpreterAssembler::LoadOSRNestingLevel() { 1495 return LoadObjectField(BytecodeArrayTaggedPointer(), 1496 BytecodeArray::kOSRNestingLevelOffset, 1497 MachineType::Int8()); 1498 } 1499 1500 void InterpreterAssembler::Abort(AbortReason abort_reason) { 1501 disable_stack_check_across_call_ = true; 1502 Node* abort_id = SmiConstant(abort_reason); 1503 CallRuntime(Runtime::kAbort, GetContext(), abort_id); 1504 disable_stack_check_across_call_ = false; 1505 } 1506 1507 void InterpreterAssembler::AbortIfWordNotEqual(Node* lhs, Node* rhs, 1508 AbortReason abort_reason) { 1509 Label ok(this), abort(this, Label::kDeferred); 1510 Branch(WordEqual(lhs, rhs), &ok, &abort); 1511 1512 BIND(&abort); 1513 Abort(abort_reason); 1514 Goto(&ok); 1515 1516 BIND(&ok); 1517 } 1518 1519 void InterpreterAssembler::MaybeDropFrames(Node* context) { 1520 Node* restart_fp_address = 1521 ExternalConstant(ExternalReference::debug_restart_fp_address(isolate())); 1522 1523 Node* restart_fp = Load(MachineType::Pointer(), restart_fp_address); 1524 Node* null = IntPtrConstant(0); 1525 1526 Label ok(this), drop_frames(this); 1527 Branch(IntPtrEqual(restart_fp, null), &ok, &drop_frames); 1528 1529 BIND(&drop_frames); 1530 // We don't expect this call to return since the frame dropper tears down 1531 // the stack and jumps into the function on the target frame to restart it. 1532 CallStub(CodeFactory::FrameDropperTrampoline(isolate()), context, restart_fp); 1533 Abort(AbortReason::kUnexpectedReturnFromFrameDropper); 1534 Goto(&ok); 1535 1536 BIND(&ok); 1537 } 1538 1539 void InterpreterAssembler::TraceBytecode(Runtime::FunctionId function_id) { 1540 CallRuntime(function_id, GetContext(), BytecodeArrayTaggedPointer(), 1541 SmiTag(BytecodeOffset()), GetAccumulatorUnchecked()); 1542 } 1543 1544 void InterpreterAssembler::TraceBytecodeDispatch(Node* target_bytecode) { 1545 Node* counters_table = ExternalConstant( 1546 ExternalReference::interpreter_dispatch_counters(isolate())); 1547 Node* source_bytecode_table_index = IntPtrConstant( 1548 static_cast<int>(bytecode_) * (static_cast<int>(Bytecode::kLast) + 1)); 1549 1550 Node* counter_offset = 1551 TimesPointerSize(IntPtrAdd(source_bytecode_table_index, target_bytecode)); 1552 Node* old_counter = 1553 Load(MachineType::IntPtr(), counters_table, counter_offset); 1554 1555 Label counter_ok(this), counter_saturated(this, Label::kDeferred); 1556 1557 Node* counter_reached_max = WordEqual( 1558 old_counter, IntPtrConstant(std::numeric_limits<uintptr_t>::max())); 1559 Branch(counter_reached_max, &counter_saturated, &counter_ok); 1560 1561 BIND(&counter_ok); 1562 { 1563 Node* new_counter = IntPtrAdd(old_counter, IntPtrConstant(1)); 1564 StoreNoWriteBarrier(MachineType::PointerRepresentation(), counters_table, 1565 counter_offset, new_counter); 1566 Goto(&counter_saturated); 1567 } 1568 1569 BIND(&counter_saturated); 1570 } 1571 1572 // static 1573 bool InterpreterAssembler::TargetSupportsUnalignedAccess() { 1574 #if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 1575 return false; 1576 #elif V8_TARGET_ARCH_IA32 || V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_S390 || \ 1577 V8_TARGET_ARCH_ARM || V8_TARGET_ARCH_ARM64 || V8_TARGET_ARCH_PPC 1578 return true; 1579 #else 1580 #error "Unknown Architecture" 1581 #endif 1582 } 1583 1584 void InterpreterAssembler::AbortIfRegisterCountInvalid( 1585 Node* parameters_and_registers, Node* formal_parameter_count, 1586 Node* register_count) { 1587 Node* array_size = LoadAndUntagFixedArrayBaseLength(parameters_and_registers); 1588 1589 Label ok(this), abort(this, Label::kDeferred); 1590 Branch(UintPtrLessThanOrEqual( 1591 IntPtrAdd(formal_parameter_count, register_count), array_size), 1592 &ok, &abort); 1593 1594 BIND(&abort); 1595 Abort(AbortReason::kInvalidParametersAndRegistersInGenerator); 1596 Goto(&ok); 1597 1598 BIND(&ok); 1599 } 1600 1601 Node* InterpreterAssembler::ExportParametersAndRegisterFile( 1602 TNode<FixedArray> array, const RegListNodePair& registers, 1603 TNode<Int32T> formal_parameter_count) { 1604 // Store the formal parameters (without receiver) followed by the 1605 // registers into the generator's internal parameters_and_registers field. 1606 TNode<IntPtrT> formal_parameter_count_intptr = 1607 ChangeInt32ToIntPtr(formal_parameter_count); 1608 Node* register_count = ChangeUint32ToWord(registers.reg_count()); 1609 if (FLAG_debug_code) { 1610 CSA_ASSERT(this, IntPtrEqual(registers.base_reg_location(), 1611 RegisterLocation(Register(0)))); 1612 AbortIfRegisterCountInvalid(array, formal_parameter_count_intptr, 1613 register_count); 1614 } 1615 1616 { 1617 Variable var_index(this, MachineType::PointerRepresentation()); 1618 var_index.Bind(IntPtrConstant(0)); 1619 1620 // Iterate over parameters and write them into the array. 1621 Label loop(this, &var_index), done_loop(this); 1622 1623 Node* reg_base = IntPtrAdd( 1624 IntPtrConstant(Register::FromParameterIndex(0, 1).ToOperand() - 1), 1625 formal_parameter_count_intptr); 1626 1627 Goto(&loop); 1628 BIND(&loop); 1629 { 1630 Node* index = var_index.value(); 1631 GotoIfNot(UintPtrLessThan(index, formal_parameter_count_intptr), 1632 &done_loop); 1633 1634 Node* reg_index = IntPtrSub(reg_base, index); 1635 Node* value = LoadRegister(reg_index); 1636 1637 StoreFixedArrayElement(array, index, value); 1638 1639 var_index.Bind(IntPtrAdd(index, IntPtrConstant(1))); 1640 Goto(&loop); 1641 } 1642 BIND(&done_loop); 1643 } 1644 1645 { 1646 // Iterate over register file and write values into array. 1647 // The mapping of register to array index must match that used in 1648 // BytecodeGraphBuilder::VisitResumeGenerator. 1649 Variable var_index(this, MachineType::PointerRepresentation()); 1650 var_index.Bind(IntPtrConstant(0)); 1651 1652 Label loop(this, &var_index), done_loop(this); 1653 Goto(&loop); 1654 BIND(&loop); 1655 { 1656 Node* index = var_index.value(); 1657 GotoIfNot(UintPtrLessThan(index, register_count), &done_loop); 1658 1659 Node* reg_index = 1660 IntPtrSub(IntPtrConstant(Register(0).ToOperand()), index); 1661 Node* value = LoadRegister(reg_index); 1662 1663 Node* array_index = IntPtrAdd(formal_parameter_count_intptr, index); 1664 StoreFixedArrayElement(array, array_index, value); 1665 1666 var_index.Bind(IntPtrAdd(index, IntPtrConstant(1))); 1667 Goto(&loop); 1668 } 1669 BIND(&done_loop); 1670 } 1671 1672 return array; 1673 } 1674 1675 Node* InterpreterAssembler::ImportRegisterFile( 1676 TNode<FixedArray> array, const RegListNodePair& registers, 1677 TNode<Int32T> formal_parameter_count) { 1678 TNode<IntPtrT> formal_parameter_count_intptr = 1679 ChangeInt32ToIntPtr(formal_parameter_count); 1680 TNode<UintPtrT> register_count = ChangeUint32ToWord(registers.reg_count()); 1681 if (FLAG_debug_code) { 1682 CSA_ASSERT(this, IntPtrEqual(registers.base_reg_location(), 1683 RegisterLocation(Register(0)))); 1684 AbortIfRegisterCountInvalid(array, formal_parameter_count_intptr, 1685 register_count); 1686 } 1687 1688 TVARIABLE(IntPtrT, var_index, IntPtrConstant(0)); 1689 1690 // Iterate over array and write values into register file. Also erase the 1691 // array contents to not keep them alive artificially. 1692 Label loop(this, &var_index), done_loop(this); 1693 Goto(&loop); 1694 BIND(&loop); 1695 { 1696 TNode<IntPtrT> index = var_index.value(); 1697 GotoIfNot(UintPtrLessThan(index, register_count), &done_loop); 1698 1699 TNode<IntPtrT> array_index = 1700 IntPtrAdd(formal_parameter_count_intptr, index); 1701 TNode<Object> value = LoadFixedArrayElement(array, array_index); 1702 1703 TNode<IntPtrT> reg_index = 1704 IntPtrSub(IntPtrConstant(Register(0).ToOperand()), index); 1705 StoreRegister(value, reg_index); 1706 1707 StoreFixedArrayElement(array, array_index, 1708 LoadRoot(Heap::kStaleRegisterRootIndex)); 1709 1710 var_index = IntPtrAdd(index, IntPtrConstant(1)); 1711 Goto(&loop); 1712 } 1713 BIND(&done_loop); 1714 1715 return array; 1716 } 1717 1718 int InterpreterAssembler::CurrentBytecodeSize() const { 1719 return Bytecodes::Size(bytecode_, operand_scale_); 1720 } 1721 1722 void InterpreterAssembler::ToNumberOrNumeric(Object::Conversion mode) { 1723 Node* object = GetAccumulator(); 1724 Node* context = GetContext(); 1725 1726 Variable var_type_feedback(this, MachineRepresentation::kTaggedSigned); 1727 Variable var_result(this, MachineRepresentation::kTagged); 1728 Label if_done(this), if_objectissmi(this), if_objectisheapnumber(this), 1729 if_objectisother(this, Label::kDeferred); 1730 1731 GotoIf(TaggedIsSmi(object), &if_objectissmi); 1732 Branch(IsHeapNumber(object), &if_objectisheapnumber, &if_objectisother); 1733 1734 BIND(&if_objectissmi); 1735 { 1736 var_result.Bind(object); 1737 var_type_feedback.Bind(SmiConstant(BinaryOperationFeedback::kSignedSmall)); 1738 Goto(&if_done); 1739 } 1740 1741 BIND(&if_objectisheapnumber); 1742 { 1743 var_result.Bind(object); 1744 var_type_feedback.Bind(SmiConstant(BinaryOperationFeedback::kNumber)); 1745 Goto(&if_done); 1746 } 1747 1748 BIND(&if_objectisother); 1749 { 1750 auto builtin = Builtins::kNonNumberToNumber; 1751 if (mode == Object::Conversion::kToNumeric) { 1752 builtin = Builtins::kNonNumberToNumeric; 1753 // Special case for collecting BigInt feedback. 1754 Label not_bigint(this); 1755 GotoIfNot(IsBigInt(object), ¬_bigint); 1756 { 1757 var_result.Bind(object); 1758 var_type_feedback.Bind(SmiConstant(BinaryOperationFeedback::kBigInt)); 1759 Goto(&if_done); 1760 } 1761 BIND(¬_bigint); 1762 } 1763 1764 // Convert {object} by calling out to the appropriate builtin. 1765 var_result.Bind(CallBuiltin(builtin, context, object)); 1766 var_type_feedback.Bind(SmiConstant(BinaryOperationFeedback::kAny)); 1767 Goto(&if_done); 1768 } 1769 1770 BIND(&if_done); 1771 1772 // Record the type feedback collected for {object}. 1773 Node* slot_index = BytecodeOperandIdx(0); 1774 Node* feedback_vector = LoadFeedbackVector(); 1775 UpdateFeedback(var_type_feedback.value(), feedback_vector, slot_index); 1776 1777 SetAccumulator(var_result.value()); 1778 Dispatch(); 1779 } 1780 1781 void InterpreterAssembler::DeserializeLazyAndDispatch() { 1782 Node* context = GetContext(); 1783 Node* bytecode_offset = BytecodeOffset(); 1784 Node* bytecode = LoadBytecode(bytecode_offset); 1785 1786 Node* target_handler = 1787 CallRuntime(Runtime::kInterpreterDeserializeLazy, context, 1788 SmiTag(bytecode), SmiConstant(operand_scale())); 1789 DispatchToBytecodeHandler(target_handler, bytecode_offset, bytecode); 1790 } 1791 1792 } // namespace interpreter 1793 } // namespace internal 1794 } // namespace v8 1795