Home | History | Annotate | Download | only in interpreter 
      1 // Copyright 2015 the V8 project authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "src/interpreter/interpreter-assembler.h"
      6 
      7 #include <limits>
      8 #include <ostream>
      9 
     10 #include "src/code-factory.h"
     11 #include "src/frames.h"
     12 #include "src/interface-descriptors.h"
     13 #include "src/interpreter/bytecodes.h"
     14 #include "src/interpreter/interpreter.h"
     15 #include "src/machine-type.h"
     16 #include "src/macro-assembler.h"
     17 #include "src/objects-inl.h"
     18 #include "src/zone/zone.h"
     19 
     20 namespace v8 {
     21 namespace internal {
     22 namespace interpreter {
     23 
     24 using compiler::CodeAssemblerState;
     25 using compiler::Node;
     26 
     27 InterpreterAssembler::InterpreterAssembler(CodeAssemblerState* state,
     28                                            Bytecode bytecode,
     29                                            OperandScale operand_scale)
     30     : CodeStubAssembler(state),
     31       bytecode_(bytecode),
     32       operand_scale_(operand_scale),
     33       bytecode_offset_(this, MachineType::PointerRepresentation()),
     34       interpreted_frame_pointer_(this, MachineType::PointerRepresentation()),
     35       bytecode_array_(this, MachineRepresentation::kTagged),
     36       dispatch_table_(this, MachineType::PointerRepresentation()),
     37       accumulator_(this, MachineRepresentation::kTagged),
     38       accumulator_use_(AccumulatorUse::kNone),
     39       made_call_(false),
     40       reloaded_frame_ptr_(false),
     41       saved_bytecode_offset_(false),
     42       disable_stack_check_across_call_(false),
     43       stack_pointer_before_call_(nullptr) {
     44   accumulator_.Bind(Parameter(InterpreterDispatchDescriptor::kAccumulator));
     45   bytecode_offset_.Bind(
     46       Parameter(InterpreterDispatchDescriptor::kBytecodeOffset));
     47   bytecode_array_.Bind(
     48       Parameter(InterpreterDispatchDescriptor::kBytecodeArray));
     49   dispatch_table_.Bind(
     50       Parameter(InterpreterDispatchDescriptor::kDispatchTable));
     51 
     52   if (FLAG_trace_ignition) {
     53     TraceBytecode(Runtime::kInterpreterTraceBytecodeEntry);
     54   }
     55   RegisterCallGenerationCallbacks([this] { CallPrologue(); },
     56                                   [this] { CallEpilogue(); });
     57 }
     58 
     59 InterpreterAssembler::~InterpreterAssembler() {
     60   // If the following check fails the handler does not use the
     61   // accumulator in the way described in the bytecode definitions in
     62   // bytecodes.h.
     63   DCHECK_EQ(accumulator_use_, Bytecodes::GetAccumulatorUse(bytecode_));
     64   UnregisterCallGenerationCallbacks();
     65 }
     66 
     67 Node* InterpreterAssembler::GetInterpretedFramePointer() {
     68   if (!interpreted_frame_pointer_.IsBound()) {
     69     interpreted_frame_pointer_.Bind(LoadParentFramePointer());
     70   } else if (Bytecodes::MakesCallAlongCriticalPath(bytecode_) && made_call_ &&
     71              !reloaded_frame_ptr_) {
     72     interpreted_frame_pointer_.Bind(LoadParentFramePointer());
     73     reloaded_frame_ptr_ = true;
     74   }
     75   return interpreted_frame_pointer_.value();
     76 }
     77 
     78 Node* InterpreterAssembler::GetAccumulatorUnchecked() {
     79   return accumulator_.value();
     80 }
     81 
     82 Node* InterpreterAssembler::GetAccumulator() {
     83   DCHECK(Bytecodes::ReadsAccumulator(bytecode_));
     84   accumulator_use_ = accumulator_use_ | AccumulatorUse::kRead;
     85   return GetAccumulatorUnchecked();
     86 }
     87 
     88 void InterpreterAssembler::SetAccumulator(Node* value) {
     89   DCHECK(Bytecodes::WritesAccumulator(bytecode_));
     90   accumulator_use_ = accumulator_use_ | AccumulatorUse::kWrite;
     91   accumulator_.Bind(value);
     92 }
     93 
     94 Node* InterpreterAssembler::GetContext() {
     95   return LoadRegister(Register::current_context());
     96 }
     97 
     98 void InterpreterAssembler::SetContext(Node* value) {
     99   StoreRegister(value, Register::current_context());
    100 }
    101 
    102 Node* InterpreterAssembler::GetContextAtDepth(Node* context, Node* depth) {
    103   Variable cur_context(this, MachineRepresentation::kTaggedPointer);
    104   cur_context.Bind(context);
    105 
    106   Variable cur_depth(this, MachineRepresentation::kWord32);
    107   cur_depth.Bind(depth);
    108 
    109   Label context_found(this);
    110 
    111   Variable* context_search_loop_variables[2] = {&cur_depth, &cur_context};
    112   Label context_search(this, 2, context_search_loop_variables);
    113 
    114   // Fast path if the depth is 0.
    115   Branch(Word32Equal(depth, Int32Constant(0)), &context_found, &context_search);
    116 
    117   // Loop until the depth is 0.
    118   Bind(&context_search);
    119   {
    120     cur_depth.Bind(Int32Sub(cur_depth.value(), Int32Constant(1)));
    121     cur_context.Bind(
    122         LoadContextElement(cur_context.value(), Context::PREVIOUS_INDEX));
    123 
    124     Branch(Word32Equal(cur_depth.value(), Int32Constant(0)), &context_found,
    125            &context_search);
    126   }
    127 
    128   Bind(&context_found);
    129   return cur_context.value();
    130 }
    131 
    132 void InterpreterAssembler::GotoIfHasContextExtensionUpToDepth(Node* context,
    133                                                               Node* depth,
    134                                                               Label* target) {
    135   Variable cur_context(this, MachineRepresentation::kTaggedPointer);
    136   cur_context.Bind(context);
    137 
    138   Variable cur_depth(this, MachineRepresentation::kWord32);
    139   cur_depth.Bind(depth);
    140 
    141   Variable* context_search_loop_variables[2] = {&cur_depth, &cur_context};
    142   Label context_search(this, 2, context_search_loop_variables);
    143 
    144   // Loop until the depth is 0.
    145   Goto(&context_search);
    146   Bind(&context_search);
    147   {
    148     // TODO(leszeks): We only need to do this check if the context had a sloppy
    149     // eval, we could pass in a context chain bitmask to figure out which
    150     // contexts actually need to be checked.
    151 
    152     Node* extension_slot =
    153         LoadContextElement(cur_context.value(), Context::EXTENSION_INDEX);
    154 
    155     // Jump to the target if the extension slot is not a hole.
    156     GotoIf(WordNotEqual(extension_slot, TheHoleConstant()), target);
    157 
    158     cur_depth.Bind(Int32Sub(cur_depth.value(), Int32Constant(1)));
    159     cur_context.Bind(
    160         LoadContextElement(cur_context.value(), Context::PREVIOUS_INDEX));
    161 
    162     GotoIf(Word32NotEqual(cur_depth.value(), Int32Constant(0)),
    163            &context_search);
    164   }
    165 }
    166 
    167 Node* InterpreterAssembler::BytecodeOffset() {
    168   if (Bytecodes::MakesCallAlongCriticalPath(bytecode_) && made_call_ &&
    169       (bytecode_offset_.value() ==
    170        Parameter(InterpreterDispatchDescriptor::kBytecodeOffset))) {
    171     bytecode_offset_.Bind(LoadAndUntagRegister(Register::bytecode_offset()));
    172   }
    173   return bytecode_offset_.value();
    174 }
    175 
    176 Node* InterpreterAssembler::BytecodeArrayTaggedPointer() {
    177   // Force a re-load of the bytecode array after every call in case the debugger
    178   // has been activated.
    179   if (made_call_ &&
    180       (bytecode_array_.value() ==
    181        Parameter(InterpreterDispatchDescriptor::kBytecodeArray))) {
    182     bytecode_array_.Bind(LoadRegister(Register::bytecode_array()));
    183   }
    184   return bytecode_array_.value();
    185 }
    186 
    187 Node* InterpreterAssembler::DispatchTableRawPointer() {
    188   if (Bytecodes::MakesCallAlongCriticalPath(bytecode_) && made_call_ &&
    189       (dispatch_table_.value() ==
    190        Parameter(InterpreterDispatchDescriptor::kDispatchTable))) {
    191     dispatch_table_.Bind(ExternalConstant(
    192         ExternalReference::interpreter_dispatch_table_address(isolate())));
    193   }
    194   return dispatch_table_.value();
    195 }
    196 
    197 Node* InterpreterAssembler::RegisterLocation(Node* reg_index) {
    198   return IntPtrAdd(GetInterpretedFramePointer(),
    199                    RegisterFrameOffset(reg_index));
    200 }
    201 
    202 Node* InterpreterAssembler::RegisterFrameOffset(Node* index) {
    203   return WordShl(index, kPointerSizeLog2);
    204 }
    205 
    206 Node* InterpreterAssembler::LoadRegister(Register reg) {
    207   return Load(MachineType::AnyTagged(), GetInterpretedFramePointer(),
    208               IntPtrConstant(reg.ToOperand() << kPointerSizeLog2));
    209 }
    210 
    211 Node* InterpreterAssembler::LoadRegister(Node* reg_index) {
    212   return Load(MachineType::AnyTagged(), GetInterpretedFramePointer(),
    213               RegisterFrameOffset(reg_index));
    214 }
    215 
    216 Node* InterpreterAssembler::LoadAndUntagRegister(Register reg) {
    217   return LoadAndUntagSmi(GetInterpretedFramePointer(), reg.ToOperand()
    218                                                            << kPointerSizeLog2);
    219 }
    220 
    221 Node* InterpreterAssembler::StoreRegister(Node* value, Register reg) {
    222   return StoreNoWriteBarrier(
    223       MachineRepresentation::kTagged, GetInterpretedFramePointer(),
    224       IntPtrConstant(reg.ToOperand() << kPointerSizeLog2), value);
    225 }
    226 
    227 Node* InterpreterAssembler::StoreRegister(Node* value, Node* reg_index) {
    228   return StoreNoWriteBarrier(MachineRepresentation::kTagged,
    229                              GetInterpretedFramePointer(),
    230                              RegisterFrameOffset(reg_index), value);
    231 }
    232 
    233 Node* InterpreterAssembler::StoreAndTagRegister(compiler::Node* value,
    234                                                 Register reg) {
    235   int offset = reg.ToOperand() << kPointerSizeLog2;
    236   return StoreAndTagSmi(GetInterpretedFramePointer(), offset, value);
    237 }
    238 
    239 Node* InterpreterAssembler::NextRegister(Node* reg_index) {
    240   // Register indexes are negative, so the next index is minus one.
    241   return IntPtrAdd(reg_index, IntPtrConstant(-1));
    242 }
    243 
    244 Node* InterpreterAssembler::OperandOffset(int operand_index) {
    245   return IntPtrConstant(
    246       Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale()));
    247 }
    248 
    249 Node* InterpreterAssembler::BytecodeOperandUnsignedByte(int operand_index) {
    250   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
    251   DCHECK_EQ(OperandSize::kByte, Bytecodes::GetOperandSize(
    252                                     bytecode_, operand_index, operand_scale()));
    253   Node* operand_offset = OperandOffset(operand_index);
    254   return Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
    255               IntPtrAdd(BytecodeOffset(), operand_offset));
    256 }
    257 
    258 Node* InterpreterAssembler::BytecodeOperandSignedByte(int operand_index) {
    259   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
    260   DCHECK_EQ(OperandSize::kByte, Bytecodes::GetOperandSize(
    261                                     bytecode_, operand_index, operand_scale()));
    262   Node* operand_offset = OperandOffset(operand_index);
    263   return Load(MachineType::Int8(), BytecodeArrayTaggedPointer(),
    264               IntPtrAdd(BytecodeOffset(), operand_offset));
    265 }
    266 
    267 compiler::Node* InterpreterAssembler::BytecodeOperandReadUnaligned(
    268     int relative_offset, MachineType result_type) {
    269   static const int kMaxCount = 4;
    270   DCHECK(!TargetSupportsUnalignedAccess());
    271 
    272   int count;
    273   switch (result_type.representation()) {
    274     case MachineRepresentation::kWord16:
    275       count = 2;
    276       break;
    277     case MachineRepresentation::kWord32:
    278       count = 4;
    279       break;
    280     default:
    281       UNREACHABLE();
    282       break;
    283   }
    284   MachineType msb_type =
    285       result_type.IsSigned() ? MachineType::Int8() : MachineType::Uint8();
    286 
    287 #if V8_TARGET_LITTLE_ENDIAN
    288   const int kStep = -1;
    289   int msb_offset = count - 1;
    290 #elif V8_TARGET_BIG_ENDIAN
    291   const int kStep = 1;
    292   int msb_offset = 0;
    293 #else
    294 #error "Unknown Architecture"
    295 #endif
    296 
    297   // Read the most signicant bytecode into bytes[0] and then in order
    298   // down to least significant in bytes[count - 1].
    299   DCHECK(count <= kMaxCount);
    300   compiler::Node* bytes[kMaxCount];
    301   for (int i = 0; i < count; i++) {
    302     MachineType machine_type = (i == 0) ? msb_type : MachineType::Uint8();
    303     Node* offset = IntPtrConstant(relative_offset + msb_offset + i * kStep);
    304     Node* array_offset = IntPtrAdd(BytecodeOffset(), offset);
    305     bytes[i] = Load(machine_type, BytecodeArrayTaggedPointer(), array_offset);
    306   }
    307 
    308   // Pack LSB to MSB.
    309   Node* result = bytes[--count];
    310   for (int i = 1; --count >= 0; i++) {
    311     Node* shift = Int32Constant(i * kBitsPerByte);
    312     Node* value = Word32Shl(bytes[count], shift);
    313     result = Word32Or(value, result);
    314   }
    315   return result;
    316 }
    317 
    318 Node* InterpreterAssembler::BytecodeOperandUnsignedShort(int operand_index) {
    319   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
    320   DCHECK_EQ(
    321       OperandSize::kShort,
    322       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()));
    323   int operand_offset =
    324       Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
    325   if (TargetSupportsUnalignedAccess()) {
    326     return Load(MachineType::Uint16(), BytecodeArrayTaggedPointer(),
    327                 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
    328   } else {
    329     return BytecodeOperandReadUnaligned(operand_offset, MachineType::Uint16());
    330   }
    331 }
    332 
    333 Node* InterpreterAssembler::BytecodeOperandSignedShort(int operand_index) {
    334   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
    335   DCHECK_EQ(
    336       OperandSize::kShort,
    337       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()));
    338   int operand_offset =
    339       Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
    340   if (TargetSupportsUnalignedAccess()) {
    341     return Load(MachineType::Int16(), BytecodeArrayTaggedPointer(),
    342                 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
    343   } else {
    344     return BytecodeOperandReadUnaligned(operand_offset, MachineType::Int16());
    345   }
    346 }
    347 
    348 Node* InterpreterAssembler::BytecodeOperandUnsignedQuad(int operand_index) {
    349   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
    350   DCHECK_EQ(OperandSize::kQuad, Bytecodes::GetOperandSize(
    351                                     bytecode_, operand_index, operand_scale()));
    352   int operand_offset =
    353       Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
    354   if (TargetSupportsUnalignedAccess()) {
    355     return Load(MachineType::Uint32(), BytecodeArrayTaggedPointer(),
    356                 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
    357   } else {
    358     return BytecodeOperandReadUnaligned(operand_offset, MachineType::Uint32());
    359   }
    360 }
    361 
    362 Node* InterpreterAssembler::BytecodeOperandSignedQuad(int operand_index) {
    363   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
    364   DCHECK_EQ(OperandSize::kQuad, Bytecodes::GetOperandSize(
    365                                     bytecode_, operand_index, operand_scale()));
    366   int operand_offset =
    367       Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
    368   if (TargetSupportsUnalignedAccess()) {
    369     return Load(MachineType::Int32(), BytecodeArrayTaggedPointer(),
    370                 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
    371   } else {
    372     return BytecodeOperandReadUnaligned(operand_offset, MachineType::Int32());
    373   }
    374 }
    375 
    376 Node* InterpreterAssembler::BytecodeSignedOperand(int operand_index,
    377                                                   OperandSize operand_size) {
    378   DCHECK(!Bytecodes::IsUnsignedOperandType(
    379       Bytecodes::GetOperandType(bytecode_, operand_index)));
    380   switch (operand_size) {
    381     case OperandSize::kByte:
    382       return BytecodeOperandSignedByte(operand_index);
    383     case OperandSize::kShort:
    384       return BytecodeOperandSignedShort(operand_index);
    385     case OperandSize::kQuad:
    386       return BytecodeOperandSignedQuad(operand_index);
    387     case OperandSize::kNone:
    388       UNREACHABLE();
    389   }
    390   return nullptr;
    391 }
    392 
    393 Node* InterpreterAssembler::BytecodeUnsignedOperand(int operand_index,
    394                                                     OperandSize operand_size) {
    395   DCHECK(Bytecodes::IsUnsignedOperandType(
    396       Bytecodes::GetOperandType(bytecode_, operand_index)));
    397   switch (operand_size) {
    398     case OperandSize::kByte:
    399       return BytecodeOperandUnsignedByte(operand_index);
    400     case OperandSize::kShort:
    401       return BytecodeOperandUnsignedShort(operand_index);
    402     case OperandSize::kQuad:
    403       return BytecodeOperandUnsignedQuad(operand_index);
    404     case OperandSize::kNone:
    405       UNREACHABLE();
    406   }
    407   return nullptr;
    408 }
    409 
    410 Node* InterpreterAssembler::BytecodeOperandCount(int operand_index) {
    411   DCHECK_EQ(OperandType::kRegCount,
    412             Bytecodes::GetOperandType(bytecode_, operand_index));
    413   OperandSize operand_size =
    414       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    415   return BytecodeUnsignedOperand(operand_index, operand_size);
    416 }
    417 
    418 Node* InterpreterAssembler::BytecodeOperandFlag(int operand_index) {
    419   DCHECK_EQ(OperandType::kFlag8,
    420             Bytecodes::GetOperandType(bytecode_, operand_index));
    421   OperandSize operand_size =
    422       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    423   DCHECK_EQ(operand_size, OperandSize::kByte);
    424   return BytecodeUnsignedOperand(operand_index, operand_size);
    425 }
    426 
    427 Node* InterpreterAssembler::BytecodeOperandUImm(int operand_index) {
    428   DCHECK_EQ(OperandType::kUImm,
    429             Bytecodes::GetOperandType(bytecode_, operand_index));
    430   OperandSize operand_size =
    431       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    432   return BytecodeUnsignedOperand(operand_index, operand_size);
    433 }
    434 
    435 Node* InterpreterAssembler::BytecodeOperandUImmWord(int operand_index) {
    436   return ChangeUint32ToWord(BytecodeOperandUImm(operand_index));
    437 }
    438 
    439 Node* InterpreterAssembler::BytecodeOperandImm(int operand_index) {
    440   DCHECK_EQ(OperandType::kImm,
    441             Bytecodes::GetOperandType(bytecode_, operand_index));
    442   OperandSize operand_size =
    443       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    444   return BytecodeSignedOperand(operand_index, operand_size);
    445 }
    446 
    447 Node* InterpreterAssembler::BytecodeOperandImmIntPtr(int operand_index) {
    448   return ChangeInt32ToIntPtr(BytecodeOperandImm(operand_index));
    449 }
    450 
    451 Node* InterpreterAssembler::BytecodeOperandImmSmi(int operand_index) {
    452   return SmiFromWord32(BytecodeOperandImm(operand_index));
    453 }
    454 
    455 Node* InterpreterAssembler::BytecodeOperandIdx(int operand_index) {
    456   DCHECK(OperandType::kIdx ==
    457          Bytecodes::GetOperandType(bytecode_, operand_index));
    458   OperandSize operand_size =
    459       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    460   return ChangeUint32ToWord(
    461       BytecodeUnsignedOperand(operand_index, operand_size));
    462 }
    463 
    464 Node* InterpreterAssembler::BytecodeOperandIdxSmi(int operand_index) {
    465   return SmiTag(BytecodeOperandIdx(operand_index));
    466 }
    467 
    468 Node* InterpreterAssembler::BytecodeOperandReg(int operand_index) {
    469   DCHECK(Bytecodes::IsRegisterOperandType(
    470       Bytecodes::GetOperandType(bytecode_, operand_index)));
    471   OperandSize operand_size =
    472       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    473   return ChangeInt32ToIntPtr(
    474       BytecodeSignedOperand(operand_index, operand_size));
    475 }
    476 
    477 Node* InterpreterAssembler::BytecodeOperandRuntimeId(int operand_index) {
    478   DCHECK(OperandType::kRuntimeId ==
    479          Bytecodes::GetOperandType(bytecode_, operand_index));
    480   OperandSize operand_size =
    481       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    482   DCHECK_EQ(operand_size, OperandSize::kShort);
    483   return BytecodeUnsignedOperand(operand_index, operand_size);
    484 }
    485 
    486 Node* InterpreterAssembler::BytecodeOperandIntrinsicId(int operand_index) {
    487   DCHECK(OperandType::kIntrinsicId ==
    488          Bytecodes::GetOperandType(bytecode_, operand_index));
    489   OperandSize operand_size =
    490       Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
    491   DCHECK_EQ(operand_size, OperandSize::kByte);
    492   return BytecodeUnsignedOperand(operand_index, operand_size);
    493 }
    494 
    495 Node* InterpreterAssembler::LoadConstantPoolEntry(Node* index) {
    496   Node* constant_pool = LoadObjectField(BytecodeArrayTaggedPointer(),
    497                                         BytecodeArray::kConstantPoolOffset);
    498   return LoadFixedArrayElement(constant_pool, index);
    499 }
    500 
    501 Node* InterpreterAssembler::LoadAndUntagConstantPoolEntry(Node* index) {
    502   return SmiUntag(LoadConstantPoolEntry(index));
    503 }
    504 
    505 Node* InterpreterAssembler::LoadFeedbackVector() {
    506   Node* function = LoadRegister(Register::function_closure());
    507   Node* cell = LoadObjectField(function, JSFunction::kFeedbackVectorOffset);
    508   Node* vector = LoadObjectField(cell, Cell::kValueOffset);
    509   return vector;
    510 }
    511 
    512 void InterpreterAssembler::SaveBytecodeOffset() {
    513   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    514   StoreAndTagRegister(BytecodeOffset(), Register::bytecode_offset());
    515   saved_bytecode_offset_ = true;
    516 }
    517 
    518 void InterpreterAssembler::CallPrologue() {
    519   if (!saved_bytecode_offset_) {
    520     // If there are multiple calls in the bytecode handler, you need to spill
    521     // before each of them, unless SaveBytecodeOffset has explicitly been called
    522     // in a path that dominates _all_ of those calls. Therefore don't set
    523     // saved_bytecode_offset_ to true or call SaveBytecodeOffset.
    524     StoreAndTagRegister(BytecodeOffset(), Register::bytecode_offset());
    525   }
    526 
    527   if (FLAG_debug_code && !disable_stack_check_across_call_) {
    528     DCHECK(stack_pointer_before_call_ == nullptr);
    529     stack_pointer_before_call_ = LoadStackPointer();
    530   }
    531   made_call_ = true;
    532 }
    533 
    534 void InterpreterAssembler::CallEpilogue() {
    535   if (FLAG_debug_code && !disable_stack_check_across_call_) {
    536     Node* stack_pointer_after_call = LoadStackPointer();
    537     Node* stack_pointer_before_call = stack_pointer_before_call_;
    538     stack_pointer_before_call_ = nullptr;
    539     AbortIfWordNotEqual(stack_pointer_before_call, stack_pointer_after_call,
    540                         kUnexpectedStackPointer);
    541   }
    542 }
    543 
    544 Node* InterpreterAssembler::IncrementCallCount(Node* feedback_vector,
    545                                                Node* slot_id) {
    546   Comment("increment call count");
    547   Node* call_count_slot = IntPtrAdd(slot_id, IntPtrConstant(1));
    548   Node* call_count = LoadFixedArrayElement(feedback_vector, call_count_slot);
    549   Node* new_count = SmiAdd(call_count, SmiConstant(1));
    550   // Count is Smi, so we don't need a write barrier.
    551   return StoreFixedArrayElement(feedback_vector, call_count_slot, new_count,
    552                                 SKIP_WRITE_BARRIER);
    553 }
    554 
    555 Node* InterpreterAssembler::CallJSWithFeedback(Node* function, Node* context,
    556                                                Node* first_arg, Node* arg_count,
    557                                                Node* slot_id,
    558                                                Node* feedback_vector,
    559                                                TailCallMode tail_call_mode) {
    560   // Static checks to assert it is safe to examine the type feedback element.
    561   // We don't know that we have a weak cell. We might have a private symbol
    562   // or an AllocationSite, but the memory is safe to examine.
    563   // AllocationSite::kTransitionInfoOffset - contains a Smi or pointer to
    564   // FixedArray.
    565   // WeakCell::kValueOffset - contains a JSFunction or Smi(0)
    566   // Symbol::kHashFieldSlot - if the low bit is 1, then the hash is not
    567   // computed, meaning that it can't appear to be a pointer. If the low bit is
    568   // 0, then hash is computed, but the 0 bit prevents the field from appearing
    569   // to be a pointer.
    570   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    571   DCHECK(Bytecodes::IsCallOrConstruct(bytecode_));
    572   STATIC_ASSERT(WeakCell::kSize >= kPointerSize);
    573   STATIC_ASSERT(AllocationSite::kTransitionInfoOffset ==
    574                     WeakCell::kValueOffset &&
    575                 WeakCell::kValueOffset == Symbol::kHashFieldSlot);
    576 
    577   Variable return_value(this, MachineRepresentation::kTagged);
    578   Label call_function(this), extra_checks(this, Label::kDeferred), call(this),
    579       end(this);
    580 
    581   // The checks. First, does function match the recorded monomorphic target?
    582   Node* feedback_element = LoadFixedArrayElement(feedback_vector, slot_id);
    583   Node* feedback_value = LoadWeakCellValueUnchecked(feedback_element);
    584   Node* is_monomorphic = WordEqual(function, feedback_value);
    585   GotoIfNot(is_monomorphic, &extra_checks);
    586 
    587   // The compare above could have been a SMI/SMI comparison. Guard against
    588   // this convincing us that we have a monomorphic JSFunction.
    589   Node* is_smi = TaggedIsSmi(function);
    590   Branch(is_smi, &extra_checks, &call_function);
    591 
    592   Bind(&call_function);
    593   {
    594     // Increment the call count.
    595     IncrementCallCount(feedback_vector, slot_id);
    596 
    597     // Call using call function builtin.
    598     Callable callable = CodeFactory::InterpreterPushArgsAndCall(
    599         isolate(), tail_call_mode, InterpreterPushArgsMode::kJSFunction);
    600     Node* code_target = HeapConstant(callable.code());
    601     Node* ret_value = CallStub(callable.descriptor(), code_target, context,
    602                                arg_count, first_arg, function);
    603     return_value.Bind(ret_value);
    604     Goto(&end);
    605   }
    606 
    607   Bind(&extra_checks);
    608   {
    609     Label check_initialized(this), mark_megamorphic(this),
    610         create_allocation_site(this);
    611 
    612     Comment("check if megamorphic");
    613     // Check if it is a megamorphic target.
    614     Node* is_megamorphic =
    615         WordEqual(feedback_element,
    616                   HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())));
    617     GotoIf(is_megamorphic, &call);
    618 
    619     Comment("check if it is an allocation site");
    620     GotoIfNot(IsAllocationSiteMap(LoadMap(feedback_element)),
    621               &check_initialized);
    622 
    623     // If it is not the Array() function, mark megamorphic.
    624     Node* context_slot = LoadContextElement(LoadNativeContext(context),
    625                                             Context::ARRAY_FUNCTION_INDEX);
    626     Node* is_array_function = WordEqual(context_slot, function);
    627     GotoIfNot(is_array_function, &mark_megamorphic);
    628 
    629     // It is a monomorphic Array function. Increment the call count.
    630     IncrementCallCount(feedback_vector, slot_id);
    631 
    632     // Call ArrayConstructorStub.
    633     Callable callable_call =
    634         CodeFactory::InterpreterPushArgsAndConstructArray(isolate());
    635     Node* code_target_call = HeapConstant(callable_call.code());
    636     Node* ret_value =
    637         CallStub(callable_call.descriptor(), code_target_call, context,
    638                  arg_count, function, feedback_element, first_arg);
    639     return_value.Bind(ret_value);
    640     Goto(&end);
    641 
    642     Bind(&check_initialized);
    643     {
    644       Comment("check if uninitialized");
    645       // Check if it is uninitialized target first.
    646       Node* is_uninitialized = WordEqual(
    647           feedback_element,
    648           HeapConstant(FeedbackVector::UninitializedSentinel(isolate())));
    649       GotoIfNot(is_uninitialized, &mark_megamorphic);
    650 
    651       Comment("handle_unitinitialized");
    652       // If it is not a JSFunction mark it as megamorphic.
    653       Node* is_smi = TaggedIsSmi(function);
    654       GotoIf(is_smi, &mark_megamorphic);
    655 
    656       // Check if function is an object of JSFunction type.
    657       Node* instance_type = LoadInstanceType(function);
    658       Node* is_js_function =
    659           Word32Equal(instance_type, Int32Constant(JS_FUNCTION_TYPE));
    660       GotoIfNot(is_js_function, &mark_megamorphic);
    661 
    662       // Check if it is the Array() function.
    663       Node* context_slot = LoadContextElement(LoadNativeContext(context),
    664                                               Context::ARRAY_FUNCTION_INDEX);
    665       Node* is_array_function = WordEqual(context_slot, function);
    666       GotoIf(is_array_function, &create_allocation_site);
    667 
    668       // Check if the function belongs to the same native context.
    669       Node* native_context = LoadNativeContext(
    670           LoadObjectField(function, JSFunction::kContextOffset));
    671       Node* is_same_native_context =
    672           WordEqual(native_context, LoadNativeContext(context));
    673       GotoIfNot(is_same_native_context, &mark_megamorphic);
    674 
    675       CreateWeakCellInFeedbackVector(feedback_vector, SmiTag(slot_id),
    676                                      function);
    677 
    678       // Call using call function builtin.
    679       Goto(&call_function);
    680     }
    681 
    682     Bind(&create_allocation_site);
    683     {
    684       CreateAllocationSiteInFeedbackVector(feedback_vector, SmiTag(slot_id));
    685 
    686       // Call using CallFunction builtin. CallICs have a PREMONOMORPHIC state.
    687       // They start collecting feedback only when a call is executed the second
    688       // time. So, do not pass any feedback here.
    689       Goto(&call_function);
    690     }
    691 
    692     Bind(&mark_megamorphic);
    693     {
    694       // Mark it as a megamorphic.
    695       // MegamorphicSentinel is created as a part of Heap::InitialObjects
    696       // and will not move during a GC. So it is safe to skip write barrier.
    697       DCHECK(Heap::RootIsImmortalImmovable(Heap::kmegamorphic_symbolRootIndex));
    698       StoreFixedArrayElement(
    699           feedback_vector, slot_id,
    700           HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())),
    701           SKIP_WRITE_BARRIER);
    702       Goto(&call);
    703     }
    704   }
    705 
    706   Bind(&call);
    707   {
    708     Comment("Increment call count and call using Call builtin");
    709     // Increment the call count.
    710     IncrementCallCount(feedback_vector, slot_id);
    711 
    712     // Call using call builtin.
    713     Callable callable_call = CodeFactory::InterpreterPushArgsAndCall(
    714         isolate(), tail_call_mode, InterpreterPushArgsMode::kOther);
    715     Node* code_target_call = HeapConstant(callable_call.code());
    716     Node* ret_value = CallStub(callable_call.descriptor(), code_target_call,
    717                                context, arg_count, first_arg, function);
    718     return_value.Bind(ret_value);
    719     Goto(&end);
    720   }
    721 
    722   Bind(&end);
    723   return return_value.value();
    724 }
    725 
    726 Node* InterpreterAssembler::CallJS(Node* function, Node* context,
    727                                    Node* first_arg, Node* arg_count,
    728                                    TailCallMode tail_call_mode) {
    729   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    730   DCHECK(Bytecodes::IsCallOrConstruct(bytecode_));
    731   Callable callable = CodeFactory::InterpreterPushArgsAndCall(
    732       isolate(), tail_call_mode, InterpreterPushArgsMode::kOther);
    733   Node* code_target = HeapConstant(callable.code());
    734 
    735   return CallStub(callable.descriptor(), code_target, context, arg_count,
    736                   first_arg, function);
    737 }
    738 
    739 Node* InterpreterAssembler::CallJSWithSpread(Node* function, Node* context,
    740                                              Node* first_arg, Node* arg_count) {
    741   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    742   Callable callable = CodeFactory::InterpreterPushArgsAndCall(
    743       isolate(), TailCallMode::kDisallow,
    744       InterpreterPushArgsMode::kWithFinalSpread);
    745   Node* code_target = HeapConstant(callable.code());
    746 
    747   return CallStub(callable.descriptor(), code_target, context, arg_count,
    748                   first_arg, function);
    749 }
    750 
    751 Node* InterpreterAssembler::Construct(Node* constructor, Node* context,
    752                                       Node* new_target, Node* first_arg,
    753                                       Node* arg_count, Node* slot_id,
    754                                       Node* feedback_vector) {
    755   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    756   Variable return_value(this, MachineRepresentation::kTagged);
    757   Variable allocation_feedback(this, MachineRepresentation::kTagged);
    758   Label call_construct_function(this, &allocation_feedback),
    759       extra_checks(this, Label::kDeferred), call_construct(this), end(this);
    760 
    761   // Slot id of 0 is used to indicate no type feedback is available.
    762   STATIC_ASSERT(FeedbackVector::kReservedIndexCount > 0);
    763   Node* is_feedback_unavailable = WordEqual(slot_id, IntPtrConstant(0));
    764   GotoIf(is_feedback_unavailable, &call_construct);
    765 
    766   // Check that the constructor is not a smi.
    767   Node* is_smi = TaggedIsSmi(constructor);
    768   GotoIf(is_smi, &call_construct);
    769 
    770   // Check that constructor is a JSFunction.
    771   Node* instance_type = LoadInstanceType(constructor);
    772   Node* is_js_function =
    773       Word32Equal(instance_type, Int32Constant(JS_FUNCTION_TYPE));
    774   GotoIfNot(is_js_function, &call_construct);
    775 
    776   // Check if it is a monomorphic constructor.
    777   Node* feedback_element = LoadFixedArrayElement(feedback_vector, slot_id);
    778   Node* feedback_value = LoadWeakCellValueUnchecked(feedback_element);
    779   Node* is_monomorphic = WordEqual(constructor, feedback_value);
    780   allocation_feedback.Bind(UndefinedConstant());
    781   Branch(is_monomorphic, &call_construct_function, &extra_checks);
    782 
    783   Bind(&call_construct_function);
    784   {
    785     Comment("call using ConstructFunction");
    786     IncrementCallCount(feedback_vector, slot_id);
    787     Callable callable_function = CodeFactory::InterpreterPushArgsAndConstruct(
    788         isolate(), InterpreterPushArgsMode::kJSFunction);
    789     return_value.Bind(CallStub(callable_function.descriptor(),
    790                                HeapConstant(callable_function.code()), context,
    791                                arg_count, new_target, constructor,
    792                                allocation_feedback.value(), first_arg));
    793     Goto(&end);
    794   }
    795 
    796   Bind(&extra_checks);
    797   {
    798     Label check_allocation_site(this), check_initialized(this),
    799         initialize(this), mark_megamorphic(this);
    800 
    801     // Check if it is a megamorphic target.
    802     Comment("check if megamorphic");
    803     Node* is_megamorphic =
    804         WordEqual(feedback_element,
    805                   HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())));
    806     GotoIf(is_megamorphic, &call_construct_function);
    807 
    808     Comment("check if weak cell");
    809     Node* is_weak_cell = WordEqual(LoadMap(feedback_element),
    810                                    LoadRoot(Heap::kWeakCellMapRootIndex));
    811     GotoIfNot(is_weak_cell, &check_allocation_site);
    812 
    813     // If the weak cell is cleared, we have a new chance to become
    814     // monomorphic.
    815     Comment("check if weak cell is cleared");
    816     Node* is_smi = TaggedIsSmi(feedback_value);
    817     Branch(is_smi, &initialize, &mark_megamorphic);
    818 
    819     Bind(&check_allocation_site);
    820     {
    821       Comment("check if it is an allocation site");
    822       Node* is_allocation_site =
    823           WordEqual(LoadObjectField(feedback_element, 0),
    824                     LoadRoot(Heap::kAllocationSiteMapRootIndex));
    825       GotoIfNot(is_allocation_site, &check_initialized);
    826 
    827       // Make sure the function is the Array() function.
    828       Node* context_slot = LoadContextElement(LoadNativeContext(context),
    829                                               Context::ARRAY_FUNCTION_INDEX);
    830       Node* is_array_function = WordEqual(context_slot, constructor);
    831       GotoIfNot(is_array_function, &mark_megamorphic);
    832 
    833       allocation_feedback.Bind(feedback_element);
    834       Goto(&call_construct_function);
    835     }
    836 
    837     Bind(&check_initialized);
    838     {
    839       // Check if it is uninitialized.
    840       Comment("check if uninitialized");
    841       Node* is_uninitialized = WordEqual(
    842           feedback_element, LoadRoot(Heap::kuninitialized_symbolRootIndex));
    843       Branch(is_uninitialized, &initialize, &mark_megamorphic);
    844     }
    845 
    846     Bind(&initialize);
    847     {
    848       Label create_allocation_site(this), create_weak_cell(this);
    849       Comment("initialize the feedback element");
    850       // Create an allocation site if the function is an array function,
    851       // otherwise create a weak cell.
    852       Node* context_slot = LoadContextElement(LoadNativeContext(context),
    853                                               Context::ARRAY_FUNCTION_INDEX);
    854       Node* is_array_function = WordEqual(context_slot, constructor);
    855       Branch(is_array_function, &create_allocation_site, &create_weak_cell);
    856 
    857       Bind(&create_allocation_site);
    858       {
    859         Node* site = CreateAllocationSiteInFeedbackVector(feedback_vector,
    860                                                           SmiTag(slot_id));
    861         allocation_feedback.Bind(site);
    862         Goto(&call_construct_function);
    863       }
    864 
    865       Bind(&create_weak_cell);
    866       {
    867         CreateWeakCellInFeedbackVector(feedback_vector, SmiTag(slot_id),
    868                                        constructor);
    869         Goto(&call_construct_function);
    870       }
    871     }
    872 
    873     Bind(&mark_megamorphic);
    874     {
    875       // MegamorphicSentinel is an immortal immovable object so
    876       // write-barrier is not needed.
    877       Comment("transition to megamorphic");
    878       DCHECK(Heap::RootIsImmortalImmovable(Heap::kmegamorphic_symbolRootIndex));
    879       StoreFixedArrayElement(
    880           feedback_vector, slot_id,
    881           HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())),
    882           SKIP_WRITE_BARRIER);
    883       Goto(&call_construct_function);
    884     }
    885   }
    886 
    887   Bind(&call_construct);
    888   {
    889     Comment("call using Construct builtin");
    890     Callable callable = CodeFactory::InterpreterPushArgsAndConstruct(
    891         isolate(), InterpreterPushArgsMode::kOther);
    892     Node* code_target = HeapConstant(callable.code());
    893     return_value.Bind(CallStub(callable.descriptor(), code_target, context,
    894                                arg_count, new_target, constructor,
    895                                UndefinedConstant(), first_arg));
    896     Goto(&end);
    897   }
    898 
    899   Bind(&end);
    900   return return_value.value();
    901 }
    902 
    903 Node* InterpreterAssembler::ConstructWithSpread(Node* constructor,
    904                                                 Node* context, Node* new_target,
    905                                                 Node* first_arg,
    906                                                 Node* arg_count) {
    907   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    908   Variable return_value(this, MachineRepresentation::kTagged);
    909   Comment("call using ConstructWithSpread");
    910   Callable callable = CodeFactory::InterpreterPushArgsAndConstruct(
    911       isolate(), InterpreterPushArgsMode::kWithFinalSpread);
    912   Node* code_target = HeapConstant(callable.code());
    913   return_value.Bind(CallStub(callable.descriptor(), code_target, context,
    914                              arg_count, new_target, constructor,
    915                              UndefinedConstant(), first_arg));
    916 
    917   return return_value.value();
    918 }
    919 
    920 Node* InterpreterAssembler::CallRuntimeN(Node* function_id, Node* context,
    921                                          Node* first_arg, Node* arg_count,
    922                                          int result_size) {
    923   DCHECK(Bytecodes::MakesCallAlongCriticalPath(bytecode_));
    924   DCHECK(Bytecodes::IsCallRuntime(bytecode_));
    925   Callable callable = CodeFactory::InterpreterCEntry(isolate(), result_size);
    926   Node* code_target = HeapConstant(callable.code());
    927 
    928   // Get the function entry from the function id.
    929   Node* function_table = ExternalConstant(
    930       ExternalReference::runtime_function_table_address(isolate()));
    931   Node* function_offset =
    932       Int32Mul(function_id, Int32Constant(sizeof(Runtime::Function)));
    933   Node* function =
    934       IntPtrAdd(function_table, ChangeUint32ToWord(function_offset));
    935   Node* function_entry =
    936       Load(MachineType::Pointer(), function,
    937            IntPtrConstant(offsetof(Runtime::Function, entry)));
    938 
    939   return CallStubR(callable.descriptor(), result_size, code_target, context,
    940                    arg_count, first_arg, function_entry);
    941 }
    942 
    943 void InterpreterAssembler::UpdateInterruptBudget(Node* weight, bool backward) {
    944   Label ok(this), interrupt_check(this, Label::kDeferred), end(this);
    945   Node* budget_offset =
    946       IntPtrConstant(BytecodeArray::kInterruptBudgetOffset - kHeapObjectTag);
    947 
    948   // Update budget by |weight| and check if it reaches zero.
    949   Variable new_budget(this, MachineRepresentation::kWord32);
    950   Node* old_budget =
    951       Load(MachineType::Int32(), BytecodeArrayTaggedPointer(), budget_offset);
    952   if (backward) {
    953     new_budget.Bind(Int32Sub(old_budget, weight));
    954   } else {
    955     new_budget.Bind(Int32Add(old_budget, weight));
    956   }
    957   Node* condition =
    958       Int32GreaterThanOrEqual(new_budget.value(), Int32Constant(0));
    959   Branch(condition, &ok, &interrupt_check);
    960 
    961   // Perform interrupt and reset budget.
    962   Bind(&interrupt_check);
    963   {
    964     CallRuntime(Runtime::kInterrupt, GetContext());
    965     new_budget.Bind(Int32Constant(Interpreter::InterruptBudget()));
    966     Goto(&ok);
    967   }
    968 
    969   // Update budget.
    970   Bind(&ok);
    971   StoreNoWriteBarrier(MachineRepresentation::kWord32,
    972                       BytecodeArrayTaggedPointer(), budget_offset,
    973                       new_budget.value());
    974 }
    975 
    976 Node* InterpreterAssembler::Advance() {
    977   return Advance(Bytecodes::Size(bytecode_, operand_scale_));
    978 }
    979 
    980 Node* InterpreterAssembler::Advance(int delta) {
    981   return Advance(IntPtrConstant(delta));
    982 }
    983 
    984 Node* InterpreterAssembler::Advance(Node* delta, bool backward) {
    985   if (FLAG_trace_ignition) {
    986     TraceBytecode(Runtime::kInterpreterTraceBytecodeExit);
    987   }
    988   Node* next_offset = backward ? IntPtrSub(BytecodeOffset(), delta)
    989                                : IntPtrAdd(BytecodeOffset(), delta);
    990   bytecode_offset_.Bind(next_offset);
    991   return next_offset;
    992 }
    993 
    994 Node* InterpreterAssembler::Jump(Node* delta, bool backward) {
    995   DCHECK(!Bytecodes::IsStarLookahead(bytecode_, operand_scale_));
    996 
    997   UpdateInterruptBudget(TruncateWordToWord32(delta), backward);
    998   Node* new_bytecode_offset = Advance(delta, backward);
    999   Node* target_bytecode = LoadBytecode(new_bytecode_offset);
   1000   return DispatchToBytecode(target_bytecode, new_bytecode_offset);
   1001 }
   1002 
   1003 Node* InterpreterAssembler::Jump(Node* delta) { return Jump(delta, false); }
   1004 
   1005 Node* InterpreterAssembler::JumpBackward(Node* delta) {
   1006   return Jump(delta, true);
   1007 }
   1008 
   1009 void InterpreterAssembler::JumpConditional(Node* condition, Node* delta) {
   1010   Label match(this), no_match(this);
   1011 
   1012   Branch(condition, &match, &no_match);
   1013   Bind(&match);
   1014   Jump(delta);
   1015   Bind(&no_match);
   1016   Dispatch();
   1017 }
   1018 
   1019 void InterpreterAssembler::JumpIfWordEqual(Node* lhs, Node* rhs, Node* delta) {
   1020   JumpConditional(WordEqual(lhs, rhs), delta);
   1021 }
   1022 
   1023 void InterpreterAssembler::JumpIfWordNotEqual(Node* lhs, Node* rhs,
   1024                                               Node* delta) {
   1025   JumpConditional(WordNotEqual(lhs, rhs), delta);
   1026 }
   1027 
   1028 Node* InterpreterAssembler::LoadBytecode(compiler::Node* bytecode_offset) {
   1029   Node* bytecode =
   1030       Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(), bytecode_offset);
   1031   return ChangeUint32ToWord(bytecode);
   1032 }
   1033 
   1034 Node* InterpreterAssembler::StarDispatchLookahead(Node* target_bytecode) {
   1035   Label do_inline_star(this), done(this);
   1036 
   1037   Variable var_bytecode(this, MachineType::PointerRepresentation());
   1038   var_bytecode.Bind(target_bytecode);
   1039 
   1040   Node* star_bytecode = IntPtrConstant(static_cast<int>(Bytecode::kStar));
   1041   Node* is_star = WordEqual(target_bytecode, star_bytecode);
   1042   Branch(is_star, &do_inline_star, &done);
   1043 
   1044   Bind(&do_inline_star);
   1045   {
   1046     InlineStar();
   1047     var_bytecode.Bind(LoadBytecode(BytecodeOffset()));
   1048     Goto(&done);
   1049   }
   1050   Bind(&done);
   1051   return var_bytecode.value();
   1052 }
   1053 
   1054 void InterpreterAssembler::InlineStar() {
   1055   Bytecode previous_bytecode = bytecode_;
   1056   AccumulatorUse previous_acc_use = accumulator_use_;
   1057 
   1058   bytecode_ = Bytecode::kStar;
   1059   accumulator_use_ = AccumulatorUse::kNone;
   1060 
   1061   if (FLAG_trace_ignition) {
   1062     TraceBytecode(Runtime::kInterpreterTraceBytecodeEntry);
   1063   }
   1064   StoreRegister(GetAccumulator(), BytecodeOperandReg(0));
   1065 
   1066   DCHECK_EQ(accumulator_use_, Bytecodes::GetAccumulatorUse(bytecode_));
   1067 
   1068   Advance();
   1069   bytecode_ = previous_bytecode;
   1070   accumulator_use_ = previous_acc_use;
   1071 }
   1072 
   1073 Node* InterpreterAssembler::Dispatch() {
   1074   Comment("========= Dispatch");
   1075   DCHECK_IMPLIES(Bytecodes::MakesCallAlongCriticalPath(bytecode_), made_call_);
   1076   Node* target_offset = Advance();
   1077   Node* target_bytecode = LoadBytecode(target_offset);
   1078 
   1079   if (Bytecodes::IsStarLookahead(bytecode_, operand_scale_)) {
   1080     target_bytecode = StarDispatchLookahead(target_bytecode);
   1081   }
   1082   return DispatchToBytecode(target_bytecode, BytecodeOffset());
   1083 }
   1084 
   1085 Node* InterpreterAssembler::DispatchToBytecode(Node* target_bytecode,
   1086                                                Node* new_bytecode_offset) {
   1087   if (FLAG_trace_ignition_dispatches) {
   1088     TraceBytecodeDispatch(target_bytecode);
   1089   }
   1090 
   1091   Node* target_code_entry =
   1092       Load(MachineType::Pointer(), DispatchTableRawPointer(),
   1093            WordShl(target_bytecode, IntPtrConstant(kPointerSizeLog2)));
   1094 
   1095   return DispatchToBytecodeHandlerEntry(target_code_entry, new_bytecode_offset);
   1096 }
   1097 
   1098 Node* InterpreterAssembler::DispatchToBytecodeHandler(Node* handler,
   1099                                                       Node* bytecode_offset) {
   1100   // TODO(ishell): Add CSA::CodeEntryPoint(code).
   1101   Node* handler_entry =
   1102       IntPtrAdd(BitcastTaggedToWord(handler),
   1103                 IntPtrConstant(Code::kHeaderSize - kHeapObjectTag));
   1104   return DispatchToBytecodeHandlerEntry(handler_entry, bytecode_offset);
   1105 }
   1106 
   1107 Node* InterpreterAssembler::DispatchToBytecodeHandlerEntry(
   1108     Node* handler_entry, Node* bytecode_offset) {
   1109   InterpreterDispatchDescriptor descriptor(isolate());
   1110   return TailCallBytecodeDispatch(
   1111       descriptor, handler_entry, GetAccumulatorUnchecked(), bytecode_offset,
   1112       BytecodeArrayTaggedPointer(), DispatchTableRawPointer());
   1113 }
   1114 
   1115 void InterpreterAssembler::DispatchWide(OperandScale operand_scale) {
   1116   // Dispatching a wide bytecode requires treating the prefix
   1117   // bytecode a base pointer into the dispatch table and dispatching
   1118   // the bytecode that follows relative to this base.
   1119   //
   1120   //   Indices 0-255 correspond to bytecodes with operand_scale == 0
   1121   //   Indices 256-511 correspond to bytecodes with operand_scale == 1
   1122   //   Indices 512-767 correspond to bytecodes with operand_scale == 2
   1123   DCHECK_IMPLIES(Bytecodes::MakesCallAlongCriticalPath(bytecode_), made_call_);
   1124   Node* next_bytecode_offset = Advance(1);
   1125   Node* next_bytecode = LoadBytecode(next_bytecode_offset);
   1126 
   1127   if (FLAG_trace_ignition_dispatches) {
   1128     TraceBytecodeDispatch(next_bytecode);
   1129   }
   1130 
   1131   Node* base_index;
   1132   switch (operand_scale) {
   1133     case OperandScale::kDouble:
   1134       base_index = IntPtrConstant(1 << kBitsPerByte);
   1135       break;
   1136     case OperandScale::kQuadruple:
   1137       base_index = IntPtrConstant(2 << kBitsPerByte);
   1138       break;
   1139     default:
   1140       UNREACHABLE();
   1141       base_index = nullptr;
   1142   }
   1143   Node* target_index = IntPtrAdd(base_index, next_bytecode);
   1144   Node* target_code_entry =
   1145       Load(MachineType::Pointer(), DispatchTableRawPointer(),
   1146            WordShl(target_index, kPointerSizeLog2));
   1147 
   1148   DispatchToBytecodeHandlerEntry(target_code_entry, next_bytecode_offset);
   1149 }
   1150 
   1151 Node* InterpreterAssembler::TruncateTaggedToWord32WithFeedback(
   1152     Node* context, Node* value, Variable* var_type_feedback) {
   1153   // We might need to loop once due to ToNumber conversion.
   1154   Variable var_value(this, MachineRepresentation::kTagged),
   1155       var_result(this, MachineRepresentation::kWord32);
   1156   Variable* loop_vars[] = {&var_value, var_type_feedback};
   1157   Label loop(this, 2, loop_vars), done_loop(this, &var_result);
   1158   var_value.Bind(value);
   1159   var_type_feedback->Bind(SmiConstant(BinaryOperationFeedback::kNone));
   1160   Goto(&loop);
   1161   Bind(&loop);
   1162   {
   1163     // Load the current {value}.
   1164     value = var_value.value();
   1165 
   1166     // Check if the {value} is a Smi or a HeapObject.
   1167     Label if_valueissmi(this), if_valueisnotsmi(this);
   1168     Branch(TaggedIsSmi(value), &if_valueissmi, &if_valueisnotsmi);
   1169 
   1170     Bind(&if_valueissmi);
   1171     {
   1172       // Convert the Smi {value}.
   1173       var_result.Bind(SmiToWord32(value));
   1174       var_type_feedback->Bind(
   1175           SmiOr(var_type_feedback->value(),
   1176                 SmiConstant(BinaryOperationFeedback::kSignedSmall)));
   1177       Goto(&done_loop);
   1178     }
   1179 
   1180     Bind(&if_valueisnotsmi);
   1181     {
   1182       // Check if {value} is a HeapNumber.
   1183       Label if_valueisheapnumber(this),
   1184           if_valueisnotheapnumber(this, Label::kDeferred);
   1185       Node* value_map = LoadMap(value);
   1186       Branch(IsHeapNumberMap(value_map), &if_valueisheapnumber,
   1187              &if_valueisnotheapnumber);
   1188 
   1189       Bind(&if_valueisheapnumber);
   1190       {
   1191         // Truncate the floating point value.
   1192         var_result.Bind(TruncateHeapNumberValueToWord32(value));
   1193         var_type_feedback->Bind(
   1194             SmiOr(var_type_feedback->value(),
   1195                   SmiConstant(BinaryOperationFeedback::kNumber)));
   1196         Goto(&done_loop);
   1197       }
   1198 
   1199       Bind(&if_valueisnotheapnumber);
   1200       {
   1201         // We do not require an Or with earlier feedback here because once we
   1202         // convert the value to a number, we cannot reach this path. We can
   1203         // only reach this path on the first pass when the feedback is kNone.
   1204         CSA_ASSERT(this, SmiEqual(var_type_feedback->value(),
   1205                                   SmiConstant(BinaryOperationFeedback::kNone)));
   1206 
   1207         Label if_valueisoddball(this),
   1208             if_valueisnotoddball(this, Label::kDeferred);
   1209         Node* is_oddball = Word32Equal(LoadMapInstanceType(value_map),
   1210                                        Int32Constant(ODDBALL_TYPE));
   1211         Branch(is_oddball, &if_valueisoddball, &if_valueisnotoddball);
   1212 
   1213         Bind(&if_valueisoddball);
   1214         {
   1215           // Convert Oddball to a Number and perform checks again.
   1216           var_value.Bind(LoadObjectField(value, Oddball::kToNumberOffset));
   1217           var_type_feedback->Bind(
   1218               SmiConstant(BinaryOperationFeedback::kNumberOrOddball));
   1219           Goto(&loop);
   1220         }
   1221 
   1222         Bind(&if_valueisnotoddball);
   1223         {
   1224           // Convert the {value} to a Number first.
   1225           Callable callable = CodeFactory::NonNumberToNumber(isolate());
   1226           var_value.Bind(CallStub(callable, context, value));
   1227           var_type_feedback->Bind(SmiConstant(BinaryOperationFeedback::kAny));
   1228           Goto(&loop);
   1229         }
   1230       }
   1231     }
   1232   }
   1233   Bind(&done_loop);
   1234   return var_result.value();
   1235 }
   1236 
   1237 void InterpreterAssembler::UpdateInterruptBudgetOnReturn() {
   1238   // TODO(rmcilroy): Investigate whether it is worth supporting self
   1239   // optimization of primitive functions like FullCodegen.
   1240 
   1241   // Update profiling count by -BytecodeOffset to simulate backedge to start of
   1242   // function.
   1243   Node* profiling_weight =
   1244       Int32Sub(Int32Constant(kHeapObjectTag + BytecodeArray::kHeaderSize),
   1245                TruncateWordToWord32(BytecodeOffset()));
   1246   UpdateInterruptBudget(profiling_weight, false);
   1247 }
   1248 
   1249 Node* InterpreterAssembler::StackCheckTriggeredInterrupt() {
   1250   Node* sp = LoadStackPointer();
   1251   Node* stack_limit = Load(
   1252       MachineType::Pointer(),
   1253       ExternalConstant(ExternalReference::address_of_stack_limit(isolate())));
   1254   return UintPtrLessThan(sp, stack_limit);
   1255 }
   1256 
   1257 Node* InterpreterAssembler::LoadOSRNestingLevel() {
   1258   return LoadObjectField(BytecodeArrayTaggedPointer(),
   1259                          BytecodeArray::kOSRNestingLevelOffset,
   1260                          MachineType::Int8());
   1261 }
   1262 
   1263 void InterpreterAssembler::Abort(BailoutReason bailout_reason) {
   1264   disable_stack_check_across_call_ = true;
   1265   Node* abort_id = SmiTag(Int32Constant(bailout_reason));
   1266   CallRuntime(Runtime::kAbort, GetContext(), abort_id);
   1267   disable_stack_check_across_call_ = false;
   1268 }
   1269 
   1270 void InterpreterAssembler::AbortIfWordNotEqual(Node* lhs, Node* rhs,
   1271                                                BailoutReason bailout_reason) {
   1272   Label ok(this), abort(this, Label::kDeferred);
   1273   Branch(WordEqual(lhs, rhs), &ok, &abort);
   1274 
   1275   Bind(&abort);
   1276   Abort(bailout_reason);
   1277   Goto(&ok);
   1278 
   1279   Bind(&ok);
   1280 }
   1281 
   1282 void InterpreterAssembler::MaybeDropFrames(Node* context) {
   1283   Node* restart_fp_address =
   1284       ExternalConstant(ExternalReference::debug_restart_fp_address(isolate()));
   1285 
   1286   Node* restart_fp = Load(MachineType::Pointer(), restart_fp_address);
   1287   Node* null = IntPtrConstant(0);
   1288 
   1289   Label ok(this), drop_frames(this);
   1290   Branch(IntPtrEqual(restart_fp, null), &ok, &drop_frames);
   1291 
   1292   Bind(&drop_frames);
   1293   // We don't expect this call to return since the frame dropper tears down
   1294   // the stack and jumps into the function on the target frame to restart it.
   1295   CallStub(CodeFactory::FrameDropperTrampoline(isolate()), context, restart_fp);
   1296   Abort(kUnexpectedReturnFromFrameDropper);
   1297   Goto(&ok);
   1298 
   1299   Bind(&ok);
   1300 }
   1301 
   1302 void InterpreterAssembler::TraceBytecode(Runtime::FunctionId function_id) {
   1303   CallRuntime(function_id, GetContext(), BytecodeArrayTaggedPointer(),
   1304               SmiTag(BytecodeOffset()), GetAccumulatorUnchecked());
   1305 }
   1306 
   1307 void InterpreterAssembler::TraceBytecodeDispatch(Node* target_bytecode) {
   1308   Node* counters_table = ExternalConstant(
   1309       ExternalReference::interpreter_dispatch_counters(isolate()));
   1310   Node* source_bytecode_table_index = IntPtrConstant(
   1311       static_cast<int>(bytecode_) * (static_cast<int>(Bytecode::kLast) + 1));
   1312 
   1313   Node* counter_offset =
   1314       WordShl(IntPtrAdd(source_bytecode_table_index, target_bytecode),
   1315               IntPtrConstant(kPointerSizeLog2));
   1316   Node* old_counter =
   1317       Load(MachineType::IntPtr(), counters_table, counter_offset);
   1318 
   1319   Label counter_ok(this), counter_saturated(this, Label::kDeferred);
   1320 
   1321   Node* counter_reached_max = WordEqual(
   1322       old_counter, IntPtrConstant(std::numeric_limits<uintptr_t>::max()));
   1323   Branch(counter_reached_max, &counter_saturated, &counter_ok);
   1324 
   1325   Bind(&counter_ok);
   1326   {
   1327     Node* new_counter = IntPtrAdd(old_counter, IntPtrConstant(1));
   1328     StoreNoWriteBarrier(MachineType::PointerRepresentation(), counters_table,
   1329                         counter_offset, new_counter);
   1330     Goto(&counter_saturated);
   1331   }
   1332 
   1333   Bind(&counter_saturated);
   1334 }
   1335 
   1336 // static
   1337 bool InterpreterAssembler::TargetSupportsUnalignedAccess() {
   1338 #if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64
   1339   return false;
   1340 #elif V8_TARGET_ARCH_IA32 || V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_X87 || \
   1341     V8_TARGET_ARCH_S390 || V8_TARGET_ARCH_ARM || V8_TARGET_ARCH_ARM64 || \
   1342     V8_TARGET_ARCH_PPC
   1343   return true;
   1344 #else
   1345 #error "Unknown Architecture"
   1346 #endif
   1347 }
   1348 
   1349 Node* InterpreterAssembler::RegisterCount() {
   1350   Node* bytecode_array = LoadRegister(Register::bytecode_array());
   1351   Node* frame_size = LoadObjectField(
   1352       bytecode_array, BytecodeArray::kFrameSizeOffset, MachineType::Uint32());
   1353   return WordShr(ChangeUint32ToWord(frame_size),
   1354                  IntPtrConstant(kPointerSizeLog2));
   1355 }
   1356 
   1357 Node* InterpreterAssembler::ExportRegisterFile(Node* array) {
   1358   Node* register_count = RegisterCount();
   1359   if (FLAG_debug_code) {
   1360     Node* array_size = LoadAndUntagFixedArrayBaseLength(array);
   1361     AbortIfWordNotEqual(array_size, register_count,
   1362                         kInvalidRegisterFileInGenerator);
   1363   }
   1364 
   1365   Variable var_index(this, MachineType::PointerRepresentation());
   1366   var_index.Bind(IntPtrConstant(0));
   1367 
   1368   // Iterate over register file and write values into array.
   1369   // The mapping of register to array index must match that used in
   1370   // BytecodeGraphBuilder::VisitResumeGenerator.
   1371   Label loop(this, &var_index), done_loop(this);
   1372   Goto(&loop);
   1373   Bind(&loop);
   1374   {
   1375     Node* index = var_index.value();
   1376     GotoIfNot(UintPtrLessThan(index, register_count), &done_loop);
   1377 
   1378     Node* reg_index = IntPtrSub(IntPtrConstant(Register(0).ToOperand()), index);
   1379     Node* value = LoadRegister(reg_index);
   1380 
   1381     StoreFixedArrayElement(array, index, value);
   1382 
   1383     var_index.Bind(IntPtrAdd(index, IntPtrConstant(1)));
   1384     Goto(&loop);
   1385   }
   1386   Bind(&done_loop);
   1387 
   1388   return array;
   1389 }
   1390 
   1391 Node* InterpreterAssembler::ImportRegisterFile(Node* array) {
   1392   Node* register_count = RegisterCount();
   1393   if (FLAG_debug_code) {
   1394     Node* array_size = LoadAndUntagFixedArrayBaseLength(array);
   1395     AbortIfWordNotEqual(array_size, register_count,
   1396                         kInvalidRegisterFileInGenerator);
   1397   }
   1398 
   1399   Variable var_index(this, MachineType::PointerRepresentation());
   1400   var_index.Bind(IntPtrConstant(0));
   1401 
   1402   // Iterate over array and write values into register file.  Also erase the
   1403   // array contents to not keep them alive artificially.
   1404   Label loop(this, &var_index), done_loop(this);
   1405   Goto(&loop);
   1406   Bind(&loop);
   1407   {
   1408     Node* index = var_index.value();
   1409     GotoIfNot(UintPtrLessThan(index, register_count), &done_loop);
   1410 
   1411     Node* value = LoadFixedArrayElement(array, index);
   1412 
   1413     Node* reg_index = IntPtrSub(IntPtrConstant(Register(0).ToOperand()), index);
   1414     StoreRegister(value, reg_index);
   1415 
   1416     StoreFixedArrayElement(array, index, StaleRegisterConstant());
   1417 
   1418     var_index.Bind(IntPtrAdd(index, IntPtrConstant(1)));
   1419     Goto(&loop);
   1420   }
   1421   Bind(&done_loop);
   1422 
   1423   return array;
   1424 }
   1425 
   1426 }  // namespace interpreter
   1427 }  // namespace internal
   1428 }  // namespace v8
   1429