1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // 3 // Redistribution and use in source and binary forms, with or without 4 // modification, are permitted provided that the following conditions are 5 // met: 6 // 7 // * Redistributions of source code must retain the above copyright 8 // notice, this list of conditions and the following disclaimer. 9 // * Redistributions in binary form must reproduce the above 10 // copyright notice, this list of conditions and the following 11 // disclaimer in the documentation and/or other materials provided 12 // with the distribution. 13 // * Neither the name of Google Inc. nor the names of its 14 // contributors may be used to endorse or promote products derived 15 // from this software without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #if V8_TARGET_ARCH_ARM64 30 31 #define ARM64_DEFINE_REG_STATICS 32 #include "src/arm64/assembler-arm64.h" 33 34 #include "src/arm64/assembler-arm64-inl.h" 35 #include "src/arm64/frames-arm64.h" 36 #include "src/base/bits.h" 37 #include "src/base/cpu.h" 38 #include "src/register-configuration.h" 39 40 namespace v8 { 41 namespace internal { 42 43 44 // ----------------------------------------------------------------------------- 45 // CpuFeatures implementation. 46 47 void CpuFeatures::ProbeImpl(bool cross_compile) { 48 // AArch64 has no configuration options, no further probing is required. 49 supported_ = 0; 50 51 // Only use statically determined features for cross compile (snapshot). 52 if (cross_compile) return; 53 54 // We used to probe for coherent cache support, but on older CPUs it 55 // causes crashes (crbug.com/524337), and newer CPUs don't even have 56 // the feature any more. 57 } 58 59 void CpuFeatures::PrintTarget() { } 60 void CpuFeatures::PrintFeatures() {} 61 62 // ----------------------------------------------------------------------------- 63 // CPURegList utilities. 64 65 CPURegister CPURegList::PopLowestIndex() { 66 DCHECK(IsValid()); 67 if (IsEmpty()) { 68 return NoCPUReg; 69 } 70 int index = CountTrailingZeros(list_, kRegListSizeInBits); 71 DCHECK((1 << index) & list_); 72 Remove(index); 73 return CPURegister::Create(index, size_, type_); 74 } 75 76 77 CPURegister CPURegList::PopHighestIndex() { 78 DCHECK(IsValid()); 79 if (IsEmpty()) { 80 return NoCPUReg; 81 } 82 int index = CountLeadingZeros(list_, kRegListSizeInBits); 83 index = kRegListSizeInBits - 1 - index; 84 DCHECK((1 << index) & list_); 85 Remove(index); 86 return CPURegister::Create(index, size_, type_); 87 } 88 89 90 void CPURegList::RemoveCalleeSaved() { 91 if (type() == CPURegister::kRegister) { 92 Remove(GetCalleeSaved(RegisterSizeInBits())); 93 } else if (type() == CPURegister::kFPRegister) { 94 Remove(GetCalleeSavedFP(RegisterSizeInBits())); 95 } else { 96 DCHECK(type() == CPURegister::kNoRegister); 97 DCHECK(IsEmpty()); 98 // The list must already be empty, so do nothing. 99 } 100 } 101 102 103 CPURegList CPURegList::GetCalleeSaved(int size) { 104 return CPURegList(CPURegister::kRegister, size, 19, 29); 105 } 106 107 108 CPURegList CPURegList::GetCalleeSavedFP(int size) { 109 return CPURegList(CPURegister::kFPRegister, size, 8, 15); 110 } 111 112 113 CPURegList CPURegList::GetCallerSaved(int size) { 114 // Registers x0-x18 and lr (x30) are caller-saved. 115 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18); 116 list.Combine(lr); 117 return list; 118 } 119 120 121 CPURegList CPURegList::GetCallerSavedFP(int size) { 122 // Registers d0-d7 and d16-d31 are caller-saved. 123 CPURegList list = CPURegList(CPURegister::kFPRegister, size, 0, 7); 124 list.Combine(CPURegList(CPURegister::kFPRegister, size, 16, 31)); 125 return list; 126 } 127 128 129 // This function defines the list of registers which are associated with a 130 // safepoint slot. Safepoint register slots are saved contiguously on the stack. 131 // MacroAssembler::SafepointRegisterStackIndex handles mapping from register 132 // code to index in the safepoint register slots. Any change here can affect 133 // this mapping. 134 CPURegList CPURegList::GetSafepointSavedRegisters() { 135 CPURegList list = CPURegList::GetCalleeSaved(); 136 list.Combine( 137 CPURegList(CPURegister::kRegister, kXRegSizeInBits, kJSCallerSaved)); 138 139 // Note that unfortunately we can't use symbolic names for registers and have 140 // to directly use register codes. This is because this function is used to 141 // initialize some static variables and we can't rely on register variables 142 // to be initialized due to static initialization order issues in C++. 143 144 // Drop ip0 and ip1 (i.e. x16 and x17), as they should not be expected to be 145 // preserved outside of the macro assembler. 146 list.Remove(16); 147 list.Remove(17); 148 149 // Add x18 to the safepoint list, as although it's not in kJSCallerSaved, it 150 // is a caller-saved register according to the procedure call standard. 151 list.Combine(18); 152 153 // Drop jssp as the stack pointer doesn't need to be included. 154 list.Remove(28); 155 156 // Add the link register (x30) to the safepoint list. 157 list.Combine(30); 158 159 return list; 160 } 161 162 163 // ----------------------------------------------------------------------------- 164 // Implementation of RelocInfo 165 166 const int RelocInfo::kApplyMask = 1 << RelocInfo::INTERNAL_REFERENCE; 167 168 169 bool RelocInfo::IsCodedSpecially() { 170 // The deserializer needs to know whether a pointer is specially coded. Being 171 // specially coded on ARM64 means that it is a movz/movk sequence. We don't 172 // generate those for relocatable pointers. 173 return false; 174 } 175 176 177 bool RelocInfo::IsInConstantPool() { 178 Instruction* instr = reinterpret_cast<Instruction*>(pc_); 179 return instr->IsLdrLiteralX(); 180 } 181 182 Address RelocInfo::wasm_memory_reference() { 183 DCHECK(IsWasmMemoryReference(rmode_)); 184 return Memory::Address_at(Assembler::target_pointer_address_at(pc_)); 185 } 186 187 uint32_t RelocInfo::wasm_memory_size_reference() { 188 DCHECK(IsWasmMemorySizeReference(rmode_)); 189 return Memory::uint32_at(Assembler::target_pointer_address_at(pc_)); 190 } 191 192 Address RelocInfo::wasm_global_reference() { 193 DCHECK(IsWasmGlobalReference(rmode_)); 194 return Memory::Address_at(Assembler::target_pointer_address_at(pc_)); 195 } 196 197 void RelocInfo::unchecked_update_wasm_memory_reference( 198 Address address, ICacheFlushMode flush_mode) { 199 Assembler::set_target_address_at(isolate_, pc_, host_, address, flush_mode); 200 } 201 202 void RelocInfo::unchecked_update_wasm_memory_size(uint32_t size, 203 ICacheFlushMode flush_mode) { 204 Memory::uint32_at(Assembler::target_pointer_address_at(pc_)) = size; 205 } 206 207 Register GetAllocatableRegisterThatIsNotOneOf(Register reg1, Register reg2, 208 Register reg3, Register reg4) { 209 CPURegList regs(reg1, reg2, reg3, reg4); 210 const RegisterConfiguration* config = RegisterConfiguration::Crankshaft(); 211 for (int i = 0; i < config->num_allocatable_double_registers(); ++i) { 212 int code = config->GetAllocatableDoubleCode(i); 213 Register candidate = Register::from_code(code); 214 if (regs.IncludesAliasOf(candidate)) continue; 215 return candidate; 216 } 217 UNREACHABLE(); 218 return NoReg; 219 } 220 221 222 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2, 223 const CPURegister& reg3, const CPURegister& reg4, 224 const CPURegister& reg5, const CPURegister& reg6, 225 const CPURegister& reg7, const CPURegister& reg8) { 226 int number_of_valid_regs = 0; 227 int number_of_valid_fpregs = 0; 228 229 RegList unique_regs = 0; 230 RegList unique_fpregs = 0; 231 232 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8}; 233 234 for (unsigned i = 0; i < arraysize(regs); i++) { 235 if (regs[i].IsRegister()) { 236 number_of_valid_regs++; 237 unique_regs |= regs[i].Bit(); 238 } else if (regs[i].IsFPRegister()) { 239 number_of_valid_fpregs++; 240 unique_fpregs |= regs[i].Bit(); 241 } else { 242 DCHECK(!regs[i].IsValid()); 243 } 244 } 245 246 int number_of_unique_regs = 247 CountSetBits(unique_regs, sizeof(unique_regs) * kBitsPerByte); 248 int number_of_unique_fpregs = 249 CountSetBits(unique_fpregs, sizeof(unique_fpregs) * kBitsPerByte); 250 251 DCHECK(number_of_valid_regs >= number_of_unique_regs); 252 DCHECK(number_of_valid_fpregs >= number_of_unique_fpregs); 253 254 return (number_of_valid_regs != number_of_unique_regs) || 255 (number_of_valid_fpregs != number_of_unique_fpregs); 256 } 257 258 259 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2, 260 const CPURegister& reg3, const CPURegister& reg4, 261 const CPURegister& reg5, const CPURegister& reg6, 262 const CPURegister& reg7, const CPURegister& reg8) { 263 DCHECK(reg1.IsValid()); 264 bool match = true; 265 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1); 266 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1); 267 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1); 268 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1); 269 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1); 270 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1); 271 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1); 272 return match; 273 } 274 275 276 void Immediate::InitializeHandle(Handle<Object> handle) { 277 AllowDeferredHandleDereference using_raw_address; 278 279 // Verify all Objects referred by code are NOT in new space. 280 Object* obj = *handle; 281 if (obj->IsHeapObject()) { 282 value_ = reinterpret_cast<intptr_t>(handle.location()); 283 rmode_ = RelocInfo::EMBEDDED_OBJECT; 284 } else { 285 STATIC_ASSERT(sizeof(intptr_t) == sizeof(int64_t)); 286 value_ = reinterpret_cast<intptr_t>(obj); 287 rmode_ = RelocInfo::NONE64; 288 } 289 } 290 291 292 bool Operand::NeedsRelocation(const Assembler* assembler) const { 293 RelocInfo::Mode rmode = immediate_.rmode(); 294 295 if (rmode == RelocInfo::EXTERNAL_REFERENCE) { 296 return assembler->serializer_enabled(); 297 } 298 299 return !RelocInfo::IsNone(rmode); 300 } 301 302 303 // Constant Pool. 304 void ConstPool::RecordEntry(intptr_t data, 305 RelocInfo::Mode mode) { 306 DCHECK(mode != RelocInfo::COMMENT && mode != RelocInfo::CONST_POOL && 307 mode != RelocInfo::VENEER_POOL && 308 mode != RelocInfo::CODE_AGE_SEQUENCE && 309 mode != RelocInfo::DEOPT_SCRIPT_OFFSET && 310 mode != RelocInfo::DEOPT_INLINING_ID && 311 mode != RelocInfo::DEOPT_REASON && mode != RelocInfo::DEOPT_ID); 312 uint64_t raw_data = static_cast<uint64_t>(data); 313 int offset = assm_->pc_offset(); 314 if (IsEmpty()) { 315 first_use_ = offset; 316 } 317 318 std::pair<uint64_t, int> entry = std::make_pair(raw_data, offset); 319 if (CanBeShared(mode)) { 320 shared_entries_.insert(entry); 321 if (shared_entries_.count(entry.first) == 1) { 322 shared_entries_count++; 323 } 324 } else { 325 unique_entries_.push_back(entry); 326 } 327 328 if (EntryCount() > Assembler::kApproxMaxPoolEntryCount) { 329 // Request constant pool emission after the next instruction. 330 assm_->SetNextConstPoolCheckIn(1); 331 } 332 } 333 334 335 int ConstPool::DistanceToFirstUse() { 336 DCHECK(first_use_ >= 0); 337 return assm_->pc_offset() - first_use_; 338 } 339 340 341 int ConstPool::MaxPcOffset() { 342 // There are no pending entries in the pool so we can never get out of 343 // range. 344 if (IsEmpty()) return kMaxInt; 345 346 // Entries are not necessarily emitted in the order they are added so in the 347 // worst case the first constant pool use will be accessing the last entry. 348 return first_use_ + kMaxLoadLiteralRange - WorstCaseSize(); 349 } 350 351 352 int ConstPool::WorstCaseSize() { 353 if (IsEmpty()) return 0; 354 355 // Max size prologue: 356 // b over 357 // ldr xzr, #pool_size 358 // blr xzr 359 // nop 360 // All entries are 64-bit for now. 361 return 4 * kInstructionSize + EntryCount() * kPointerSize; 362 } 363 364 365 int ConstPool::SizeIfEmittedAtCurrentPc(bool require_jump) { 366 if (IsEmpty()) return 0; 367 368 // Prologue is: 369 // b over ;; if require_jump 370 // ldr xzr, #pool_size 371 // blr xzr 372 // nop ;; if not 64-bit aligned 373 int prologue_size = require_jump ? kInstructionSize : 0; 374 prologue_size += 2 * kInstructionSize; 375 prologue_size += IsAligned(assm_->pc_offset() + prologue_size, 8) ? 376 0 : kInstructionSize; 377 378 // All entries are 64-bit for now. 379 return prologue_size + EntryCount() * kPointerSize; 380 } 381 382 383 void ConstPool::Emit(bool require_jump) { 384 DCHECK(!assm_->is_const_pool_blocked()); 385 // Prevent recursive pool emission and protect from veneer pools. 386 Assembler::BlockPoolsScope block_pools(assm_); 387 388 int size = SizeIfEmittedAtCurrentPc(require_jump); 389 Label size_check; 390 assm_->bind(&size_check); 391 392 assm_->RecordConstPool(size); 393 // Emit the constant pool. It is preceded by an optional branch if 394 // require_jump and a header which will: 395 // 1) Encode the size of the constant pool, for use by the disassembler. 396 // 2) Terminate the program, to try to prevent execution from accidentally 397 // flowing into the constant pool. 398 // 3) align the pool entries to 64-bit. 399 // The header is therefore made of up to three arm64 instructions: 400 // ldr xzr, #<size of the constant pool in 32-bit words> 401 // blr xzr 402 // nop 403 // 404 // If executed, the header will likely segfault and lr will point to the 405 // instruction following the offending blr. 406 // TODO(all): Make the alignment part less fragile. Currently code is 407 // allocated as a byte array so there are no guarantees the alignment will 408 // be preserved on compaction. Currently it works as allocation seems to be 409 // 64-bit aligned. 410 411 // Emit branch if required 412 Label after_pool; 413 if (require_jump) { 414 assm_->b(&after_pool); 415 } 416 417 // Emit the header. 418 assm_->RecordComment("[ Constant Pool"); 419 EmitMarker(); 420 EmitGuard(); 421 assm_->Align(8); 422 423 // Emit constant pool entries. 424 // TODO(all): currently each relocated constant is 64 bits, consider adding 425 // support for 32-bit entries. 426 EmitEntries(); 427 assm_->RecordComment("]"); 428 429 if (after_pool.is_linked()) { 430 assm_->bind(&after_pool); 431 } 432 433 DCHECK(assm_->SizeOfCodeGeneratedSince(&size_check) == 434 static_cast<unsigned>(size)); 435 } 436 437 438 void ConstPool::Clear() { 439 shared_entries_.clear(); 440 shared_entries_count = 0; 441 unique_entries_.clear(); 442 first_use_ = -1; 443 } 444 445 446 bool ConstPool::CanBeShared(RelocInfo::Mode mode) { 447 // Constant pool currently does not support 32-bit entries. 448 DCHECK(mode != RelocInfo::NONE32); 449 450 return RelocInfo::IsNone(mode) || 451 (!assm_->serializer_enabled() && 452 (mode >= RelocInfo::FIRST_SHAREABLE_RELOC_MODE)); 453 } 454 455 456 void ConstPool::EmitMarker() { 457 // A constant pool size is expressed in number of 32-bits words. 458 // Currently all entries are 64-bit. 459 // + 1 is for the crash guard. 460 // + 0/1 for alignment. 461 int word_count = EntryCount() * 2 + 1 + 462 (IsAligned(assm_->pc_offset(), 8) ? 0 : 1); 463 assm_->Emit(LDR_x_lit | 464 Assembler::ImmLLiteral(word_count) | 465 Assembler::Rt(xzr)); 466 } 467 468 469 MemOperand::PairResult MemOperand::AreConsistentForPair( 470 const MemOperand& operandA, 471 const MemOperand& operandB, 472 int access_size_log2) { 473 DCHECK(access_size_log2 >= 0); 474 DCHECK(access_size_log2 <= 3); 475 // Step one: check that they share the same base, that the mode is Offset 476 // and that the offset is a multiple of access size. 477 if (!operandA.base().Is(operandB.base()) || 478 (operandA.addrmode() != Offset) || 479 (operandB.addrmode() != Offset) || 480 ((operandA.offset() & ((1 << access_size_log2) - 1)) != 0)) { 481 return kNotPair; 482 } 483 // Step two: check that the offsets are contiguous and that the range 484 // is OK for ldp/stp. 485 if ((operandB.offset() == operandA.offset() + (1 << access_size_log2)) && 486 is_int7(operandA.offset() >> access_size_log2)) { 487 return kPairAB; 488 } 489 if ((operandA.offset() == operandB.offset() + (1 << access_size_log2)) && 490 is_int7(operandB.offset() >> access_size_log2)) { 491 return kPairBA; 492 } 493 return kNotPair; 494 } 495 496 497 void ConstPool::EmitGuard() { 498 #ifdef DEBUG 499 Instruction* instr = reinterpret_cast<Instruction*>(assm_->pc()); 500 DCHECK(instr->preceding()->IsLdrLiteralX() && 501 instr->preceding()->Rt() == xzr.code()); 502 #endif 503 assm_->EmitPoolGuard(); 504 } 505 506 507 void ConstPool::EmitEntries() { 508 DCHECK(IsAligned(assm_->pc_offset(), 8)); 509 510 typedef std::multimap<uint64_t, int>::const_iterator SharedEntriesIterator; 511 SharedEntriesIterator value_it; 512 // Iterate through the keys (constant pool values). 513 for (value_it = shared_entries_.begin(); 514 value_it != shared_entries_.end(); 515 value_it = shared_entries_.upper_bound(value_it->first)) { 516 std::pair<SharedEntriesIterator, SharedEntriesIterator> range; 517 uint64_t data = value_it->first; 518 range = shared_entries_.equal_range(data); 519 SharedEntriesIterator offset_it; 520 // Iterate through the offsets of a given key. 521 for (offset_it = range.first; offset_it != range.second; offset_it++) { 522 Instruction* instr = assm_->InstructionAt(offset_it->second); 523 524 // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0. 525 DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0); 526 instr->SetImmPCOffsetTarget(assm_->isolate(), assm_->pc()); 527 } 528 assm_->dc64(data); 529 } 530 shared_entries_.clear(); 531 shared_entries_count = 0; 532 533 // Emit unique entries. 534 std::vector<std::pair<uint64_t, int> >::const_iterator unique_it; 535 for (unique_it = unique_entries_.begin(); 536 unique_it != unique_entries_.end(); 537 unique_it++) { 538 Instruction* instr = assm_->InstructionAt(unique_it->second); 539 540 // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0. 541 DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0); 542 instr->SetImmPCOffsetTarget(assm_->isolate(), assm_->pc()); 543 assm_->dc64(unique_it->first); 544 } 545 unique_entries_.clear(); 546 first_use_ = -1; 547 } 548 549 550 // Assembler 551 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size) 552 : AssemblerBase(isolate, buffer, buffer_size), 553 constpool_(this), 554 recorded_ast_id_(TypeFeedbackId::None()), 555 unresolved_branches_() { 556 const_pool_blocked_nesting_ = 0; 557 veneer_pool_blocked_nesting_ = 0; 558 Reset(); 559 } 560 561 562 Assembler::~Assembler() { 563 DCHECK(constpool_.IsEmpty()); 564 DCHECK(const_pool_blocked_nesting_ == 0); 565 DCHECK(veneer_pool_blocked_nesting_ == 0); 566 } 567 568 569 void Assembler::Reset() { 570 #ifdef DEBUG 571 DCHECK((pc_ >= buffer_) && (pc_ < buffer_ + buffer_size_)); 572 DCHECK(const_pool_blocked_nesting_ == 0); 573 DCHECK(veneer_pool_blocked_nesting_ == 0); 574 DCHECK(unresolved_branches_.empty()); 575 memset(buffer_, 0, pc_ - buffer_); 576 #endif 577 pc_ = buffer_; 578 reloc_info_writer.Reposition(reinterpret_cast<byte*>(buffer_ + buffer_size_), 579 reinterpret_cast<byte*>(pc_)); 580 constpool_.Clear(); 581 next_constant_pool_check_ = 0; 582 next_veneer_pool_check_ = kMaxInt; 583 no_const_pool_before_ = 0; 584 ClearRecordedAstId(); 585 } 586 587 588 void Assembler::GetCode(CodeDesc* desc) { 589 // Emit constant pool if necessary. 590 CheckConstPool(true, false); 591 DCHECK(constpool_.IsEmpty()); 592 593 // Set up code descriptor. 594 if (desc) { 595 desc->buffer = reinterpret_cast<byte*>(buffer_); 596 desc->buffer_size = buffer_size_; 597 desc->instr_size = pc_offset(); 598 desc->reloc_size = 599 static_cast<int>((reinterpret_cast<byte*>(buffer_) + buffer_size_) - 600 reloc_info_writer.pos()); 601 desc->origin = this; 602 desc->constant_pool_size = 0; 603 desc->unwinding_info_size = 0; 604 desc->unwinding_info = nullptr; 605 } 606 } 607 608 609 void Assembler::Align(int m) { 610 DCHECK(m >= 4 && base::bits::IsPowerOfTwo32(m)); 611 while ((pc_offset() & (m - 1)) != 0) { 612 nop(); 613 } 614 } 615 616 617 void Assembler::CheckLabelLinkChain(Label const * label) { 618 #ifdef DEBUG 619 if (label->is_linked()) { 620 static const int kMaxLinksToCheck = 64; // Avoid O(n2) behaviour. 621 int links_checked = 0; 622 int64_t linkoffset = label->pos(); 623 bool end_of_chain = false; 624 while (!end_of_chain) { 625 if (++links_checked > kMaxLinksToCheck) break; 626 Instruction * link = InstructionAt(linkoffset); 627 int64_t linkpcoffset = link->ImmPCOffset(); 628 int64_t prevlinkoffset = linkoffset + linkpcoffset; 629 630 end_of_chain = (linkoffset == prevlinkoffset); 631 linkoffset = linkoffset + linkpcoffset; 632 } 633 } 634 #endif 635 } 636 637 638 void Assembler::RemoveBranchFromLabelLinkChain(Instruction* branch, 639 Label* label, 640 Instruction* label_veneer) { 641 DCHECK(label->is_linked()); 642 643 CheckLabelLinkChain(label); 644 645 Instruction* link = InstructionAt(label->pos()); 646 Instruction* prev_link = link; 647 Instruction* next_link; 648 bool end_of_chain = false; 649 650 while (link != branch && !end_of_chain) { 651 next_link = link->ImmPCOffsetTarget(); 652 end_of_chain = (link == next_link); 653 prev_link = link; 654 link = next_link; 655 } 656 657 DCHECK(branch == link); 658 next_link = branch->ImmPCOffsetTarget(); 659 660 if (branch == prev_link) { 661 // The branch is the first instruction in the chain. 662 if (branch == next_link) { 663 // It is also the last instruction in the chain, so it is the only branch 664 // currently referring to this label. 665 label->Unuse(); 666 } else { 667 label->link_to( 668 static_cast<int>(reinterpret_cast<byte*>(next_link) - buffer_)); 669 } 670 671 } else if (branch == next_link) { 672 // The branch is the last (but not also the first) instruction in the chain. 673 prev_link->SetImmPCOffsetTarget(isolate(), prev_link); 674 675 } else { 676 // The branch is in the middle of the chain. 677 if (prev_link->IsTargetInImmPCOffsetRange(next_link)) { 678 prev_link->SetImmPCOffsetTarget(isolate(), next_link); 679 } else if (label_veneer != NULL) { 680 // Use the veneer for all previous links in the chain. 681 prev_link->SetImmPCOffsetTarget(isolate(), prev_link); 682 683 end_of_chain = false; 684 link = next_link; 685 while (!end_of_chain) { 686 next_link = link->ImmPCOffsetTarget(); 687 end_of_chain = (link == next_link); 688 link->SetImmPCOffsetTarget(isolate(), label_veneer); 689 link = next_link; 690 } 691 } else { 692 // The assert below will fire. 693 // Some other work could be attempted to fix up the chain, but it would be 694 // rather complicated. If we crash here, we may want to consider using an 695 // other mechanism than a chain of branches. 696 // 697 // Note that this situation currently should not happen, as we always call 698 // this function with a veneer to the target label. 699 // However this could happen with a MacroAssembler in the following state: 700 // [previous code] 701 // B(label); 702 // [20KB code] 703 // Tbz(label); // First tbz. Pointing to unconditional branch. 704 // [20KB code] 705 // Tbz(label); // Second tbz. Pointing to the first tbz. 706 // [more code] 707 // and this function is called to remove the first tbz from the label link 708 // chain. Since tbz has a range of +-32KB, the second tbz cannot point to 709 // the unconditional branch. 710 CHECK(prev_link->IsTargetInImmPCOffsetRange(next_link)); 711 UNREACHABLE(); 712 } 713 } 714 715 CheckLabelLinkChain(label); 716 } 717 718 719 void Assembler::bind(Label* label) { 720 // Bind label to the address at pc_. All instructions (most likely branches) 721 // that are linked to this label will be updated to point to the newly-bound 722 // label. 723 724 DCHECK(!label->is_near_linked()); 725 DCHECK(!label->is_bound()); 726 727 DeleteUnresolvedBranchInfoForLabel(label); 728 729 // If the label is linked, the link chain looks something like this: 730 // 731 // |--I----I-------I-------L 732 // |---------------------->| pc_offset 733 // |-------------->| linkoffset = label->pos() 734 // |<------| link->ImmPCOffset() 735 // |------>| prevlinkoffset = linkoffset + link->ImmPCOffset() 736 // 737 // On each iteration, the last link is updated and then removed from the 738 // chain until only one remains. At that point, the label is bound. 739 // 740 // If the label is not linked, no preparation is required before binding. 741 while (label->is_linked()) { 742 int linkoffset = label->pos(); 743 Instruction* link = InstructionAt(linkoffset); 744 int prevlinkoffset = linkoffset + static_cast<int>(link->ImmPCOffset()); 745 746 CheckLabelLinkChain(label); 747 748 DCHECK(linkoffset >= 0); 749 DCHECK(linkoffset < pc_offset()); 750 DCHECK((linkoffset > prevlinkoffset) || 751 (linkoffset - prevlinkoffset == kStartOfLabelLinkChain)); 752 DCHECK(prevlinkoffset >= 0); 753 754 // Update the link to point to the label. 755 if (link->IsUnresolvedInternalReference()) { 756 // Internal references do not get patched to an instruction but directly 757 // to an address. 758 internal_reference_positions_.push_back(linkoffset); 759 PatchingAssembler patcher(isolate(), link, 2); 760 patcher.dc64(reinterpret_cast<uintptr_t>(pc_)); 761 } else { 762 link->SetImmPCOffsetTarget(isolate(), 763 reinterpret_cast<Instruction*>(pc_)); 764 } 765 766 // Link the label to the previous link in the chain. 767 if (linkoffset - prevlinkoffset == kStartOfLabelLinkChain) { 768 // We hit kStartOfLabelLinkChain, so the chain is fully processed. 769 label->Unuse(); 770 } else { 771 // Update the label for the next iteration. 772 label->link_to(prevlinkoffset); 773 } 774 } 775 label->bind_to(pc_offset()); 776 777 DCHECK(label->is_bound()); 778 DCHECK(!label->is_linked()); 779 } 780 781 782 int Assembler::LinkAndGetByteOffsetTo(Label* label) { 783 DCHECK(sizeof(*pc_) == 1); 784 CheckLabelLinkChain(label); 785 786 int offset; 787 if (label->is_bound()) { 788 // The label is bound, so it does not need to be updated. Referring 789 // instructions must link directly to the label as they will not be 790 // updated. 791 // 792 // In this case, label->pos() returns the offset of the label from the 793 // start of the buffer. 794 // 795 // Note that offset can be zero for self-referential instructions. (This 796 // could be useful for ADR, for example.) 797 offset = label->pos() - pc_offset(); 798 DCHECK(offset <= 0); 799 } else { 800 if (label->is_linked()) { 801 // The label is linked, so the referring instruction should be added onto 802 // the end of the label's link chain. 803 // 804 // In this case, label->pos() returns the offset of the last linked 805 // instruction from the start of the buffer. 806 offset = label->pos() - pc_offset(); 807 DCHECK(offset != kStartOfLabelLinkChain); 808 // Note that the offset here needs to be PC-relative only so that the 809 // first instruction in a buffer can link to an unbound label. Otherwise, 810 // the offset would be 0 for this case, and 0 is reserved for 811 // kStartOfLabelLinkChain. 812 } else { 813 // The label is unused, so it now becomes linked and the referring 814 // instruction is at the start of the new link chain. 815 offset = kStartOfLabelLinkChain; 816 } 817 // The instruction at pc is now the last link in the label's chain. 818 label->link_to(pc_offset()); 819 } 820 821 return offset; 822 } 823 824 825 void Assembler::DeleteUnresolvedBranchInfoForLabelTraverse(Label* label) { 826 DCHECK(label->is_linked()); 827 CheckLabelLinkChain(label); 828 829 int link_offset = label->pos(); 830 int link_pcoffset; 831 bool end_of_chain = false; 832 833 while (!end_of_chain) { 834 Instruction * link = InstructionAt(link_offset); 835 link_pcoffset = static_cast<int>(link->ImmPCOffset()); 836 837 // ADR instructions are not handled by veneers. 838 if (link->IsImmBranch()) { 839 int max_reachable_pc = 840 static_cast<int>(InstructionOffset(link) + 841 Instruction::ImmBranchRange(link->BranchType())); 842 typedef std::multimap<int, FarBranchInfo>::iterator unresolved_info_it; 843 std::pair<unresolved_info_it, unresolved_info_it> range; 844 range = unresolved_branches_.equal_range(max_reachable_pc); 845 unresolved_info_it it; 846 for (it = range.first; it != range.second; ++it) { 847 if (it->second.pc_offset_ == link_offset) { 848 unresolved_branches_.erase(it); 849 break; 850 } 851 } 852 } 853 854 end_of_chain = (link_pcoffset == 0); 855 link_offset = link_offset + link_pcoffset; 856 } 857 } 858 859 860 void Assembler::DeleteUnresolvedBranchInfoForLabel(Label* label) { 861 if (unresolved_branches_.empty()) { 862 DCHECK(next_veneer_pool_check_ == kMaxInt); 863 return; 864 } 865 866 if (label->is_linked()) { 867 // Branches to this label will be resolved when the label is bound, normally 868 // just after all the associated info has been deleted. 869 DeleteUnresolvedBranchInfoForLabelTraverse(label); 870 } 871 if (unresolved_branches_.empty()) { 872 next_veneer_pool_check_ = kMaxInt; 873 } else { 874 next_veneer_pool_check_ = 875 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 876 } 877 } 878 879 880 void Assembler::StartBlockConstPool() { 881 if (const_pool_blocked_nesting_++ == 0) { 882 // Prevent constant pool checks happening by setting the next check to 883 // the biggest possible offset. 884 next_constant_pool_check_ = kMaxInt; 885 } 886 } 887 888 889 void Assembler::EndBlockConstPool() { 890 if (--const_pool_blocked_nesting_ == 0) { 891 // Check the constant pool hasn't been blocked for too long. 892 DCHECK(pc_offset() < constpool_.MaxPcOffset()); 893 // Two cases: 894 // * no_const_pool_before_ >= next_constant_pool_check_ and the emission is 895 // still blocked 896 // * no_const_pool_before_ < next_constant_pool_check_ and the next emit 897 // will trigger a check. 898 next_constant_pool_check_ = no_const_pool_before_; 899 } 900 } 901 902 903 bool Assembler::is_const_pool_blocked() const { 904 return (const_pool_blocked_nesting_ > 0) || 905 (pc_offset() < no_const_pool_before_); 906 } 907 908 909 bool Assembler::IsConstantPoolAt(Instruction* instr) { 910 // The constant pool marker is made of two instructions. These instructions 911 // will never be emitted by the JIT, so checking for the first one is enough: 912 // 0: ldr xzr, #<size of pool> 913 bool result = instr->IsLdrLiteralX() && (instr->Rt() == kZeroRegCode); 914 915 // It is still worth asserting the marker is complete. 916 // 4: blr xzr 917 DCHECK(!result || (instr->following()->IsBranchAndLinkToRegister() && 918 instr->following()->Rn() == kZeroRegCode)); 919 920 return result; 921 } 922 923 924 int Assembler::ConstantPoolSizeAt(Instruction* instr) { 925 #ifdef USE_SIMULATOR 926 // Assembler::debug() embeds constants directly into the instruction stream. 927 // Although this is not a genuine constant pool, treat it like one to avoid 928 // disassembling the constants. 929 if ((instr->Mask(ExceptionMask) == HLT) && 930 (instr->ImmException() == kImmExceptionIsDebug)) { 931 const char* message = 932 reinterpret_cast<const char*>( 933 instr->InstructionAtOffset(kDebugMessageOffset)); 934 int size = static_cast<int>(kDebugMessageOffset + strlen(message) + 1); 935 return RoundUp(size, kInstructionSize) / kInstructionSize; 936 } 937 // Same for printf support, see MacroAssembler::CallPrintf(). 938 if ((instr->Mask(ExceptionMask) == HLT) && 939 (instr->ImmException() == kImmExceptionIsPrintf)) { 940 return kPrintfLength / kInstructionSize; 941 } 942 #endif 943 if (IsConstantPoolAt(instr)) { 944 return instr->ImmLLiteral(); 945 } else { 946 return -1; 947 } 948 } 949 950 951 void Assembler::EmitPoolGuard() { 952 // We must generate only one instruction as this is used in scopes that 953 // control the size of the code generated. 954 Emit(BLR | Rn(xzr)); 955 } 956 957 958 void Assembler::StartBlockVeneerPool() { 959 ++veneer_pool_blocked_nesting_; 960 } 961 962 963 void Assembler::EndBlockVeneerPool() { 964 if (--veneer_pool_blocked_nesting_ == 0) { 965 // Check the veneer pool hasn't been blocked for too long. 966 DCHECK(unresolved_branches_.empty() || 967 (pc_offset() < unresolved_branches_first_limit())); 968 } 969 } 970 971 972 void Assembler::br(const Register& xn) { 973 DCHECK(xn.Is64Bits()); 974 Emit(BR | Rn(xn)); 975 } 976 977 978 void Assembler::blr(const Register& xn) { 979 DCHECK(xn.Is64Bits()); 980 // The pattern 'blr xzr' is used as a guard to detect when execution falls 981 // through the constant pool. It should not be emitted. 982 DCHECK(!xn.Is(xzr)); 983 Emit(BLR | Rn(xn)); 984 } 985 986 987 void Assembler::ret(const Register& xn) { 988 DCHECK(xn.Is64Bits()); 989 Emit(RET | Rn(xn)); 990 } 991 992 993 void Assembler::b(int imm26) { 994 Emit(B | ImmUncondBranch(imm26)); 995 } 996 997 998 void Assembler::b(Label* label) { 999 b(LinkAndGetInstructionOffsetTo(label)); 1000 } 1001 1002 1003 void Assembler::b(int imm19, Condition cond) { 1004 Emit(B_cond | ImmCondBranch(imm19) | cond); 1005 } 1006 1007 1008 void Assembler::b(Label* label, Condition cond) { 1009 b(LinkAndGetInstructionOffsetTo(label), cond); 1010 } 1011 1012 1013 void Assembler::bl(int imm26) { 1014 Emit(BL | ImmUncondBranch(imm26)); 1015 } 1016 1017 1018 void Assembler::bl(Label* label) { 1019 bl(LinkAndGetInstructionOffsetTo(label)); 1020 } 1021 1022 1023 void Assembler::cbz(const Register& rt, 1024 int imm19) { 1025 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt)); 1026 } 1027 1028 1029 void Assembler::cbz(const Register& rt, 1030 Label* label) { 1031 cbz(rt, LinkAndGetInstructionOffsetTo(label)); 1032 } 1033 1034 1035 void Assembler::cbnz(const Register& rt, 1036 int imm19) { 1037 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt)); 1038 } 1039 1040 1041 void Assembler::cbnz(const Register& rt, 1042 Label* label) { 1043 cbnz(rt, LinkAndGetInstructionOffsetTo(label)); 1044 } 1045 1046 1047 void Assembler::tbz(const Register& rt, 1048 unsigned bit_pos, 1049 int imm14) { 1050 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits))); 1051 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 1052 } 1053 1054 1055 void Assembler::tbz(const Register& rt, 1056 unsigned bit_pos, 1057 Label* label) { 1058 tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 1059 } 1060 1061 1062 void Assembler::tbnz(const Register& rt, 1063 unsigned bit_pos, 1064 int imm14) { 1065 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits))); 1066 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 1067 } 1068 1069 1070 void Assembler::tbnz(const Register& rt, 1071 unsigned bit_pos, 1072 Label* label) { 1073 tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 1074 } 1075 1076 1077 void Assembler::adr(const Register& rd, int imm21) { 1078 DCHECK(rd.Is64Bits()); 1079 Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd)); 1080 } 1081 1082 1083 void Assembler::adr(const Register& rd, Label* label) { 1084 adr(rd, LinkAndGetByteOffsetTo(label)); 1085 } 1086 1087 1088 void Assembler::add(const Register& rd, 1089 const Register& rn, 1090 const Operand& operand) { 1091 AddSub(rd, rn, operand, LeaveFlags, ADD); 1092 } 1093 1094 1095 void Assembler::adds(const Register& rd, 1096 const Register& rn, 1097 const Operand& operand) { 1098 AddSub(rd, rn, operand, SetFlags, ADD); 1099 } 1100 1101 1102 void Assembler::cmn(const Register& rn, 1103 const Operand& operand) { 1104 Register zr = AppropriateZeroRegFor(rn); 1105 adds(zr, rn, operand); 1106 } 1107 1108 1109 void Assembler::sub(const Register& rd, 1110 const Register& rn, 1111 const Operand& operand) { 1112 AddSub(rd, rn, operand, LeaveFlags, SUB); 1113 } 1114 1115 1116 void Assembler::subs(const Register& rd, 1117 const Register& rn, 1118 const Operand& operand) { 1119 AddSub(rd, rn, operand, SetFlags, SUB); 1120 } 1121 1122 1123 void Assembler::cmp(const Register& rn, const Operand& operand) { 1124 Register zr = AppropriateZeroRegFor(rn); 1125 subs(zr, rn, operand); 1126 } 1127 1128 1129 void Assembler::neg(const Register& rd, const Operand& operand) { 1130 Register zr = AppropriateZeroRegFor(rd); 1131 sub(rd, zr, operand); 1132 } 1133 1134 1135 void Assembler::negs(const Register& rd, const Operand& operand) { 1136 Register zr = AppropriateZeroRegFor(rd); 1137 subs(rd, zr, operand); 1138 } 1139 1140 1141 void Assembler::adc(const Register& rd, 1142 const Register& rn, 1143 const Operand& operand) { 1144 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC); 1145 } 1146 1147 1148 void Assembler::adcs(const Register& rd, 1149 const Register& rn, 1150 const Operand& operand) { 1151 AddSubWithCarry(rd, rn, operand, SetFlags, ADC); 1152 } 1153 1154 1155 void Assembler::sbc(const Register& rd, 1156 const Register& rn, 1157 const Operand& operand) { 1158 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC); 1159 } 1160 1161 1162 void Assembler::sbcs(const Register& rd, 1163 const Register& rn, 1164 const Operand& operand) { 1165 AddSubWithCarry(rd, rn, operand, SetFlags, SBC); 1166 } 1167 1168 1169 void Assembler::ngc(const Register& rd, const Operand& operand) { 1170 Register zr = AppropriateZeroRegFor(rd); 1171 sbc(rd, zr, operand); 1172 } 1173 1174 1175 void Assembler::ngcs(const Register& rd, const Operand& operand) { 1176 Register zr = AppropriateZeroRegFor(rd); 1177 sbcs(rd, zr, operand); 1178 } 1179 1180 1181 // Logical instructions. 1182 void Assembler::and_(const Register& rd, 1183 const Register& rn, 1184 const Operand& operand) { 1185 Logical(rd, rn, operand, AND); 1186 } 1187 1188 1189 void Assembler::ands(const Register& rd, 1190 const Register& rn, 1191 const Operand& operand) { 1192 Logical(rd, rn, operand, ANDS); 1193 } 1194 1195 1196 void Assembler::tst(const Register& rn, 1197 const Operand& operand) { 1198 ands(AppropriateZeroRegFor(rn), rn, operand); 1199 } 1200 1201 1202 void Assembler::bic(const Register& rd, 1203 const Register& rn, 1204 const Operand& operand) { 1205 Logical(rd, rn, operand, BIC); 1206 } 1207 1208 1209 void Assembler::bics(const Register& rd, 1210 const Register& rn, 1211 const Operand& operand) { 1212 Logical(rd, rn, operand, BICS); 1213 } 1214 1215 1216 void Assembler::orr(const Register& rd, 1217 const Register& rn, 1218 const Operand& operand) { 1219 Logical(rd, rn, operand, ORR); 1220 } 1221 1222 1223 void Assembler::orn(const Register& rd, 1224 const Register& rn, 1225 const Operand& operand) { 1226 Logical(rd, rn, operand, ORN); 1227 } 1228 1229 1230 void Assembler::eor(const Register& rd, 1231 const Register& rn, 1232 const Operand& operand) { 1233 Logical(rd, rn, operand, EOR); 1234 } 1235 1236 1237 void Assembler::eon(const Register& rd, 1238 const Register& rn, 1239 const Operand& operand) { 1240 Logical(rd, rn, operand, EON); 1241 } 1242 1243 1244 void Assembler::lslv(const Register& rd, 1245 const Register& rn, 1246 const Register& rm) { 1247 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1248 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1249 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd)); 1250 } 1251 1252 1253 void Assembler::lsrv(const Register& rd, 1254 const Register& rn, 1255 const Register& rm) { 1256 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1257 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1258 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd)); 1259 } 1260 1261 1262 void Assembler::asrv(const Register& rd, 1263 const Register& rn, 1264 const Register& rm) { 1265 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1266 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1267 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd)); 1268 } 1269 1270 1271 void Assembler::rorv(const Register& rd, 1272 const Register& rn, 1273 const Register& rm) { 1274 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1275 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1276 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd)); 1277 } 1278 1279 1280 // Bitfield operations. 1281 void Assembler::bfm(const Register& rd, const Register& rn, int immr, 1282 int imms) { 1283 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1284 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1285 Emit(SF(rd) | BFM | N | 1286 ImmR(immr, rd.SizeInBits()) | 1287 ImmS(imms, rn.SizeInBits()) | 1288 Rn(rn) | Rd(rd)); 1289 } 1290 1291 1292 void Assembler::sbfm(const Register& rd, const Register& rn, int immr, 1293 int imms) { 1294 DCHECK(rd.Is64Bits() || rn.Is32Bits()); 1295 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1296 Emit(SF(rd) | SBFM | N | 1297 ImmR(immr, rd.SizeInBits()) | 1298 ImmS(imms, rn.SizeInBits()) | 1299 Rn(rn) | Rd(rd)); 1300 } 1301 1302 1303 void Assembler::ubfm(const Register& rd, const Register& rn, int immr, 1304 int imms) { 1305 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1306 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1307 Emit(SF(rd) | UBFM | N | 1308 ImmR(immr, rd.SizeInBits()) | 1309 ImmS(imms, rn.SizeInBits()) | 1310 Rn(rn) | Rd(rd)); 1311 } 1312 1313 1314 void Assembler::extr(const Register& rd, const Register& rn, const Register& rm, 1315 int lsb) { 1316 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1317 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1318 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1319 Emit(SF(rd) | EXTR | N | Rm(rm) | 1320 ImmS(lsb, rn.SizeInBits()) | Rn(rn) | Rd(rd)); 1321 } 1322 1323 1324 void Assembler::csel(const Register& rd, 1325 const Register& rn, 1326 const Register& rm, 1327 Condition cond) { 1328 ConditionalSelect(rd, rn, rm, cond, CSEL); 1329 } 1330 1331 1332 void Assembler::csinc(const Register& rd, 1333 const Register& rn, 1334 const Register& rm, 1335 Condition cond) { 1336 ConditionalSelect(rd, rn, rm, cond, CSINC); 1337 } 1338 1339 1340 void Assembler::csinv(const Register& rd, 1341 const Register& rn, 1342 const Register& rm, 1343 Condition cond) { 1344 ConditionalSelect(rd, rn, rm, cond, CSINV); 1345 } 1346 1347 1348 void Assembler::csneg(const Register& rd, 1349 const Register& rn, 1350 const Register& rm, 1351 Condition cond) { 1352 ConditionalSelect(rd, rn, rm, cond, CSNEG); 1353 } 1354 1355 1356 void Assembler::cset(const Register &rd, Condition cond) { 1357 DCHECK((cond != al) && (cond != nv)); 1358 Register zr = AppropriateZeroRegFor(rd); 1359 csinc(rd, zr, zr, NegateCondition(cond)); 1360 } 1361 1362 1363 void Assembler::csetm(const Register &rd, Condition cond) { 1364 DCHECK((cond != al) && (cond != nv)); 1365 Register zr = AppropriateZeroRegFor(rd); 1366 csinv(rd, zr, zr, NegateCondition(cond)); 1367 } 1368 1369 1370 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) { 1371 DCHECK((cond != al) && (cond != nv)); 1372 csinc(rd, rn, rn, NegateCondition(cond)); 1373 } 1374 1375 1376 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) { 1377 DCHECK((cond != al) && (cond != nv)); 1378 csinv(rd, rn, rn, NegateCondition(cond)); 1379 } 1380 1381 1382 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) { 1383 DCHECK((cond != al) && (cond != nv)); 1384 csneg(rd, rn, rn, NegateCondition(cond)); 1385 } 1386 1387 1388 void Assembler::ConditionalSelect(const Register& rd, 1389 const Register& rn, 1390 const Register& rm, 1391 Condition cond, 1392 ConditionalSelectOp op) { 1393 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1394 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1395 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd)); 1396 } 1397 1398 1399 void Assembler::ccmn(const Register& rn, 1400 const Operand& operand, 1401 StatusFlags nzcv, 1402 Condition cond) { 1403 ConditionalCompare(rn, operand, nzcv, cond, CCMN); 1404 } 1405 1406 1407 void Assembler::ccmp(const Register& rn, 1408 const Operand& operand, 1409 StatusFlags nzcv, 1410 Condition cond) { 1411 ConditionalCompare(rn, operand, nzcv, cond, CCMP); 1412 } 1413 1414 1415 void Assembler::DataProcessing3Source(const Register& rd, 1416 const Register& rn, 1417 const Register& rm, 1418 const Register& ra, 1419 DataProcessing3SourceOp op) { 1420 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd)); 1421 } 1422 1423 1424 void Assembler::mul(const Register& rd, 1425 const Register& rn, 1426 const Register& rm) { 1427 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1428 Register zr = AppropriateZeroRegFor(rn); 1429 DataProcessing3Source(rd, rn, rm, zr, MADD); 1430 } 1431 1432 1433 void Assembler::madd(const Register& rd, 1434 const Register& rn, 1435 const Register& rm, 1436 const Register& ra) { 1437 DCHECK(AreSameSizeAndType(rd, rn, rm, ra)); 1438 DataProcessing3Source(rd, rn, rm, ra, MADD); 1439 } 1440 1441 1442 void Assembler::mneg(const Register& rd, 1443 const Register& rn, 1444 const Register& rm) { 1445 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1446 Register zr = AppropriateZeroRegFor(rn); 1447 DataProcessing3Source(rd, rn, rm, zr, MSUB); 1448 } 1449 1450 1451 void Assembler::msub(const Register& rd, 1452 const Register& rn, 1453 const Register& rm, 1454 const Register& ra) { 1455 DCHECK(AreSameSizeAndType(rd, rn, rm, ra)); 1456 DataProcessing3Source(rd, rn, rm, ra, MSUB); 1457 } 1458 1459 1460 void Assembler::smaddl(const Register& rd, 1461 const Register& rn, 1462 const Register& rm, 1463 const Register& ra) { 1464 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1465 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1466 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x); 1467 } 1468 1469 1470 void Assembler::smsubl(const Register& rd, 1471 const Register& rn, 1472 const Register& rm, 1473 const Register& ra) { 1474 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1475 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1476 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x); 1477 } 1478 1479 1480 void Assembler::umaddl(const Register& rd, 1481 const Register& rn, 1482 const Register& rm, 1483 const Register& ra) { 1484 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1485 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1486 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x); 1487 } 1488 1489 1490 void Assembler::umsubl(const Register& rd, 1491 const Register& rn, 1492 const Register& rm, 1493 const Register& ra) { 1494 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1495 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1496 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x); 1497 } 1498 1499 1500 void Assembler::smull(const Register& rd, 1501 const Register& rn, 1502 const Register& rm) { 1503 DCHECK(rd.Is64Bits()); 1504 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1505 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x); 1506 } 1507 1508 1509 void Assembler::smulh(const Register& rd, 1510 const Register& rn, 1511 const Register& rm) { 1512 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1513 DataProcessing3Source(rd, rn, rm, xzr, SMULH_x); 1514 } 1515 1516 1517 void Assembler::sdiv(const Register& rd, 1518 const Register& rn, 1519 const Register& rm) { 1520 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1521 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1522 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1523 } 1524 1525 1526 void Assembler::udiv(const Register& rd, 1527 const Register& rn, 1528 const Register& rm) { 1529 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1530 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1531 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1532 } 1533 1534 1535 void Assembler::rbit(const Register& rd, 1536 const Register& rn) { 1537 DataProcessing1Source(rd, rn, RBIT); 1538 } 1539 1540 1541 void Assembler::rev16(const Register& rd, 1542 const Register& rn) { 1543 DataProcessing1Source(rd, rn, REV16); 1544 } 1545 1546 1547 void Assembler::rev32(const Register& rd, 1548 const Register& rn) { 1549 DCHECK(rd.Is64Bits()); 1550 DataProcessing1Source(rd, rn, REV); 1551 } 1552 1553 1554 void Assembler::rev(const Register& rd, 1555 const Register& rn) { 1556 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w); 1557 } 1558 1559 1560 void Assembler::clz(const Register& rd, 1561 const Register& rn) { 1562 DataProcessing1Source(rd, rn, CLZ); 1563 } 1564 1565 1566 void Assembler::cls(const Register& rd, 1567 const Register& rn) { 1568 DataProcessing1Source(rd, rn, CLS); 1569 } 1570 1571 1572 void Assembler::ldp(const CPURegister& rt, 1573 const CPURegister& rt2, 1574 const MemOperand& src) { 1575 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2)); 1576 } 1577 1578 1579 void Assembler::stp(const CPURegister& rt, 1580 const CPURegister& rt2, 1581 const MemOperand& dst) { 1582 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2)); 1583 } 1584 1585 1586 void Assembler::ldpsw(const Register& rt, 1587 const Register& rt2, 1588 const MemOperand& src) { 1589 DCHECK(rt.Is64Bits()); 1590 LoadStorePair(rt, rt2, src, LDPSW_x); 1591 } 1592 1593 1594 void Assembler::LoadStorePair(const CPURegister& rt, 1595 const CPURegister& rt2, 1596 const MemOperand& addr, 1597 LoadStorePairOp op) { 1598 // 'rt' and 'rt2' can only be aliased for stores. 1599 DCHECK(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2)); 1600 DCHECK(AreSameSizeAndType(rt, rt2)); 1601 DCHECK(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op))); 1602 int offset = static_cast<int>(addr.offset()); 1603 1604 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | 1605 ImmLSPair(offset, CalcLSPairDataSize(op)); 1606 1607 Instr addrmodeop; 1608 if (addr.IsImmediateOffset()) { 1609 addrmodeop = LoadStorePairOffsetFixed; 1610 } else { 1611 // Pre-index and post-index modes. 1612 DCHECK(!rt.Is(addr.base())); 1613 DCHECK(!rt2.Is(addr.base())); 1614 DCHECK(addr.offset() != 0); 1615 if (addr.IsPreIndex()) { 1616 addrmodeop = LoadStorePairPreIndexFixed; 1617 } else { 1618 DCHECK(addr.IsPostIndex()); 1619 addrmodeop = LoadStorePairPostIndexFixed; 1620 } 1621 } 1622 Emit(addrmodeop | memop); 1623 } 1624 1625 1626 // Memory instructions. 1627 void Assembler::ldrb(const Register& rt, const MemOperand& src) { 1628 LoadStore(rt, src, LDRB_w); 1629 } 1630 1631 1632 void Assembler::strb(const Register& rt, const MemOperand& dst) { 1633 LoadStore(rt, dst, STRB_w); 1634 } 1635 1636 1637 void Assembler::ldrsb(const Register& rt, const MemOperand& src) { 1638 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w); 1639 } 1640 1641 1642 void Assembler::ldrh(const Register& rt, const MemOperand& src) { 1643 LoadStore(rt, src, LDRH_w); 1644 } 1645 1646 1647 void Assembler::strh(const Register& rt, const MemOperand& dst) { 1648 LoadStore(rt, dst, STRH_w); 1649 } 1650 1651 1652 void Assembler::ldrsh(const Register& rt, const MemOperand& src) { 1653 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w); 1654 } 1655 1656 1657 void Assembler::ldr(const CPURegister& rt, const MemOperand& src) { 1658 LoadStore(rt, src, LoadOpFor(rt)); 1659 } 1660 1661 1662 void Assembler::str(const CPURegister& rt, const MemOperand& src) { 1663 LoadStore(rt, src, StoreOpFor(rt)); 1664 } 1665 1666 1667 void Assembler::ldrsw(const Register& rt, const MemOperand& src) { 1668 DCHECK(rt.Is64Bits()); 1669 LoadStore(rt, src, LDRSW_x); 1670 } 1671 1672 1673 void Assembler::ldr_pcrel(const CPURegister& rt, int imm19) { 1674 // The pattern 'ldr xzr, #offset' is used to indicate the beginning of a 1675 // constant pool. It should not be emitted. 1676 DCHECK(!rt.IsZero()); 1677 Emit(LoadLiteralOpFor(rt) | ImmLLiteral(imm19) | Rt(rt)); 1678 } 1679 1680 1681 void Assembler::ldr(const CPURegister& rt, const Immediate& imm) { 1682 // Currently we only support 64-bit literals. 1683 DCHECK(rt.Is64Bits()); 1684 1685 RecordRelocInfo(imm.rmode(), imm.value()); 1686 BlockConstPoolFor(1); 1687 // The load will be patched when the constpool is emitted, patching code 1688 // expect a load literal with offset 0. 1689 ldr_pcrel(rt, 0); 1690 } 1691 1692 void Assembler::ldar(const Register& rt, const Register& rn) { 1693 DCHECK(rn.Is64Bits()); 1694 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x; 1695 Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1696 } 1697 1698 void Assembler::ldaxr(const Register& rt, const Register& rn) { 1699 DCHECK(rn.Is64Bits()); 1700 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x; 1701 Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1702 } 1703 1704 void Assembler::stlr(const Register& rt, const Register& rn) { 1705 DCHECK(rn.Is64Bits()); 1706 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x; 1707 Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1708 } 1709 1710 void Assembler::stlxr(const Register& rs, const Register& rt, 1711 const Register& rn) { 1712 DCHECK(rs.Is32Bits()); 1713 DCHECK(rn.Is64Bits()); 1714 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x; 1715 Emit(op | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt)); 1716 } 1717 1718 void Assembler::ldarb(const Register& rt, const Register& rn) { 1719 DCHECK(rt.Is32Bits()); 1720 DCHECK(rn.Is64Bits()); 1721 Emit(LDAR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1722 } 1723 1724 void Assembler::ldaxrb(const Register& rt, const Register& rn) { 1725 DCHECK(rt.Is32Bits()); 1726 DCHECK(rn.Is64Bits()); 1727 Emit(LDAXR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1728 } 1729 1730 void Assembler::stlrb(const Register& rt, const Register& rn) { 1731 DCHECK(rt.Is32Bits()); 1732 DCHECK(rn.Is64Bits()); 1733 Emit(STLR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1734 } 1735 1736 void Assembler::stlxrb(const Register& rs, const Register& rt, 1737 const Register& rn) { 1738 DCHECK(rs.Is32Bits()); 1739 DCHECK(rt.Is32Bits()); 1740 DCHECK(rn.Is64Bits()); 1741 Emit(STLXR_b | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt)); 1742 } 1743 1744 void Assembler::ldarh(const Register& rt, const Register& rn) { 1745 DCHECK(rt.Is32Bits()); 1746 DCHECK(rn.Is64Bits()); 1747 Emit(LDAR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1748 } 1749 1750 void Assembler::ldaxrh(const Register& rt, const Register& rn) { 1751 DCHECK(rt.Is32Bits()); 1752 DCHECK(rn.Is64Bits()); 1753 Emit(LDAXR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1754 } 1755 1756 void Assembler::stlrh(const Register& rt, const Register& rn) { 1757 DCHECK(rt.Is32Bits()); 1758 DCHECK(rn.Is64Bits()); 1759 Emit(STLR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1760 } 1761 1762 void Assembler::stlxrh(const Register& rs, const Register& rt, 1763 const Register& rn) { 1764 DCHECK(rs.Is32Bits()); 1765 DCHECK(rt.Is32Bits()); 1766 DCHECK(rn.Is64Bits()); 1767 Emit(STLXR_h | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt)); 1768 } 1769 1770 void Assembler::mov(const Register& rd, const Register& rm) { 1771 // Moves involving the stack pointer are encoded as add immediate with 1772 // second operand of zero. Otherwise, orr with first operand zr is 1773 // used. 1774 if (rd.IsSP() || rm.IsSP()) { 1775 add(rd, rm, 0); 1776 } else { 1777 orr(rd, AppropriateZeroRegFor(rd), rm); 1778 } 1779 } 1780 1781 1782 void Assembler::mvn(const Register& rd, const Operand& operand) { 1783 orn(rd, AppropriateZeroRegFor(rd), operand); 1784 } 1785 1786 1787 void Assembler::mrs(const Register& rt, SystemRegister sysreg) { 1788 DCHECK(rt.Is64Bits()); 1789 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt)); 1790 } 1791 1792 1793 void Assembler::msr(SystemRegister sysreg, const Register& rt) { 1794 DCHECK(rt.Is64Bits()); 1795 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg)); 1796 } 1797 1798 1799 void Assembler::hint(SystemHint code) { 1800 Emit(HINT | ImmHint(code) | Rt(xzr)); 1801 } 1802 1803 1804 void Assembler::dmb(BarrierDomain domain, BarrierType type) { 1805 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 1806 } 1807 1808 1809 void Assembler::dsb(BarrierDomain domain, BarrierType type) { 1810 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 1811 } 1812 1813 1814 void Assembler::isb() { 1815 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll)); 1816 } 1817 1818 1819 void Assembler::fmov(FPRegister fd, double imm) { 1820 DCHECK(fd.Is64Bits()); 1821 DCHECK(IsImmFP64(imm)); 1822 Emit(FMOV_d_imm | Rd(fd) | ImmFP64(imm)); 1823 } 1824 1825 1826 void Assembler::fmov(FPRegister fd, float imm) { 1827 DCHECK(fd.Is32Bits()); 1828 DCHECK(IsImmFP32(imm)); 1829 Emit(FMOV_s_imm | Rd(fd) | ImmFP32(imm)); 1830 } 1831 1832 1833 void Assembler::fmov(Register rd, FPRegister fn) { 1834 DCHECK(rd.SizeInBits() == fn.SizeInBits()); 1835 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd; 1836 Emit(op | Rd(rd) | Rn(fn)); 1837 } 1838 1839 1840 void Assembler::fmov(FPRegister fd, Register rn) { 1841 DCHECK(fd.SizeInBits() == rn.SizeInBits()); 1842 FPIntegerConvertOp op = fd.Is32Bits() ? FMOV_sw : FMOV_dx; 1843 Emit(op | Rd(fd) | Rn(rn)); 1844 } 1845 1846 1847 void Assembler::fmov(FPRegister fd, FPRegister fn) { 1848 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1849 Emit(FPType(fd) | FMOV | Rd(fd) | Rn(fn)); 1850 } 1851 1852 1853 void Assembler::fadd(const FPRegister& fd, 1854 const FPRegister& fn, 1855 const FPRegister& fm) { 1856 FPDataProcessing2Source(fd, fn, fm, FADD); 1857 } 1858 1859 1860 void Assembler::fsub(const FPRegister& fd, 1861 const FPRegister& fn, 1862 const FPRegister& fm) { 1863 FPDataProcessing2Source(fd, fn, fm, FSUB); 1864 } 1865 1866 1867 void Assembler::fmul(const FPRegister& fd, 1868 const FPRegister& fn, 1869 const FPRegister& fm) { 1870 FPDataProcessing2Source(fd, fn, fm, FMUL); 1871 } 1872 1873 1874 void Assembler::fmadd(const FPRegister& fd, 1875 const FPRegister& fn, 1876 const FPRegister& fm, 1877 const FPRegister& fa) { 1878 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMADD_s : FMADD_d); 1879 } 1880 1881 1882 void Assembler::fmsub(const FPRegister& fd, 1883 const FPRegister& fn, 1884 const FPRegister& fm, 1885 const FPRegister& fa) { 1886 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMSUB_s : FMSUB_d); 1887 } 1888 1889 1890 void Assembler::fnmadd(const FPRegister& fd, 1891 const FPRegister& fn, 1892 const FPRegister& fm, 1893 const FPRegister& fa) { 1894 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMADD_s : FNMADD_d); 1895 } 1896 1897 1898 void Assembler::fnmsub(const FPRegister& fd, 1899 const FPRegister& fn, 1900 const FPRegister& fm, 1901 const FPRegister& fa) { 1902 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMSUB_s : FNMSUB_d); 1903 } 1904 1905 1906 void Assembler::fdiv(const FPRegister& fd, 1907 const FPRegister& fn, 1908 const FPRegister& fm) { 1909 FPDataProcessing2Source(fd, fn, fm, FDIV); 1910 } 1911 1912 1913 void Assembler::fmax(const FPRegister& fd, 1914 const FPRegister& fn, 1915 const FPRegister& fm) { 1916 FPDataProcessing2Source(fd, fn, fm, FMAX); 1917 } 1918 1919 1920 void Assembler::fmaxnm(const FPRegister& fd, 1921 const FPRegister& fn, 1922 const FPRegister& fm) { 1923 FPDataProcessing2Source(fd, fn, fm, FMAXNM); 1924 } 1925 1926 1927 void Assembler::fmin(const FPRegister& fd, 1928 const FPRegister& fn, 1929 const FPRegister& fm) { 1930 FPDataProcessing2Source(fd, fn, fm, FMIN); 1931 } 1932 1933 1934 void Assembler::fminnm(const FPRegister& fd, 1935 const FPRegister& fn, 1936 const FPRegister& fm) { 1937 FPDataProcessing2Source(fd, fn, fm, FMINNM); 1938 } 1939 1940 1941 void Assembler::fabs(const FPRegister& fd, 1942 const FPRegister& fn) { 1943 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1944 FPDataProcessing1Source(fd, fn, FABS); 1945 } 1946 1947 1948 void Assembler::fneg(const FPRegister& fd, 1949 const FPRegister& fn) { 1950 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1951 FPDataProcessing1Source(fd, fn, FNEG); 1952 } 1953 1954 1955 void Assembler::fsqrt(const FPRegister& fd, 1956 const FPRegister& fn) { 1957 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1958 FPDataProcessing1Source(fd, fn, FSQRT); 1959 } 1960 1961 1962 void Assembler::frinta(const FPRegister& fd, 1963 const FPRegister& fn) { 1964 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1965 FPDataProcessing1Source(fd, fn, FRINTA); 1966 } 1967 1968 1969 void Assembler::frintm(const FPRegister& fd, 1970 const FPRegister& fn) { 1971 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1972 FPDataProcessing1Source(fd, fn, FRINTM); 1973 } 1974 1975 1976 void Assembler::frintn(const FPRegister& fd, 1977 const FPRegister& fn) { 1978 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1979 FPDataProcessing1Source(fd, fn, FRINTN); 1980 } 1981 1982 1983 void Assembler::frintp(const FPRegister& fd, const FPRegister& fn) { 1984 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1985 FPDataProcessing1Source(fd, fn, FRINTP); 1986 } 1987 1988 1989 void Assembler::frintz(const FPRegister& fd, 1990 const FPRegister& fn) { 1991 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1992 FPDataProcessing1Source(fd, fn, FRINTZ); 1993 } 1994 1995 1996 void Assembler::fcmp(const FPRegister& fn, 1997 const FPRegister& fm) { 1998 DCHECK(fn.SizeInBits() == fm.SizeInBits()); 1999 Emit(FPType(fn) | FCMP | Rm(fm) | Rn(fn)); 2000 } 2001 2002 2003 void Assembler::fcmp(const FPRegister& fn, 2004 double value) { 2005 USE(value); 2006 // Although the fcmp instruction can strictly only take an immediate value of 2007 // +0.0, we don't need to check for -0.0 because the sign of 0.0 doesn't 2008 // affect the result of the comparison. 2009 DCHECK(value == 0.0); 2010 Emit(FPType(fn) | FCMP_zero | Rn(fn)); 2011 } 2012 2013 2014 void Assembler::fccmp(const FPRegister& fn, 2015 const FPRegister& fm, 2016 StatusFlags nzcv, 2017 Condition cond) { 2018 DCHECK(fn.SizeInBits() == fm.SizeInBits()); 2019 Emit(FPType(fn) | FCCMP | Rm(fm) | Cond(cond) | Rn(fn) | Nzcv(nzcv)); 2020 } 2021 2022 2023 void Assembler::fcsel(const FPRegister& fd, 2024 const FPRegister& fn, 2025 const FPRegister& fm, 2026 Condition cond) { 2027 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 2028 DCHECK(fd.SizeInBits() == fm.SizeInBits()); 2029 Emit(FPType(fd) | FCSEL | Rm(fm) | Cond(cond) | Rn(fn) | Rd(fd)); 2030 } 2031 2032 2033 void Assembler::FPConvertToInt(const Register& rd, 2034 const FPRegister& fn, 2035 FPIntegerConvertOp op) { 2036 Emit(SF(rd) | FPType(fn) | op | Rn(fn) | Rd(rd)); 2037 } 2038 2039 2040 void Assembler::fcvt(const FPRegister& fd, 2041 const FPRegister& fn) { 2042 if (fd.Is64Bits()) { 2043 // Convert float to double. 2044 DCHECK(fn.Is32Bits()); 2045 FPDataProcessing1Source(fd, fn, FCVT_ds); 2046 } else { 2047 // Convert double to float. 2048 DCHECK(fn.Is64Bits()); 2049 FPDataProcessing1Source(fd, fn, FCVT_sd); 2050 } 2051 } 2052 2053 2054 void Assembler::fcvtau(const Register& rd, const FPRegister& fn) { 2055 FPConvertToInt(rd, fn, FCVTAU); 2056 } 2057 2058 2059 void Assembler::fcvtas(const Register& rd, const FPRegister& fn) { 2060 FPConvertToInt(rd, fn, FCVTAS); 2061 } 2062 2063 2064 void Assembler::fcvtmu(const Register& rd, const FPRegister& fn) { 2065 FPConvertToInt(rd, fn, FCVTMU); 2066 } 2067 2068 2069 void Assembler::fcvtms(const Register& rd, const FPRegister& fn) { 2070 FPConvertToInt(rd, fn, FCVTMS); 2071 } 2072 2073 2074 void Assembler::fcvtnu(const Register& rd, const FPRegister& fn) { 2075 FPConvertToInt(rd, fn, FCVTNU); 2076 } 2077 2078 2079 void Assembler::fcvtns(const Register& rd, const FPRegister& fn) { 2080 FPConvertToInt(rd, fn, FCVTNS); 2081 } 2082 2083 2084 void Assembler::fcvtzu(const Register& rd, const FPRegister& fn) { 2085 FPConvertToInt(rd, fn, FCVTZU); 2086 } 2087 2088 2089 void Assembler::fcvtzs(const Register& rd, const FPRegister& fn) { 2090 FPConvertToInt(rd, fn, FCVTZS); 2091 } 2092 2093 2094 void Assembler::scvtf(const FPRegister& fd, 2095 const Register& rn, 2096 unsigned fbits) { 2097 if (fbits == 0) { 2098 Emit(SF(rn) | FPType(fd) | SCVTF | Rn(rn) | Rd(fd)); 2099 } else { 2100 Emit(SF(rn) | FPType(fd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 2101 Rd(fd)); 2102 } 2103 } 2104 2105 2106 void Assembler::ucvtf(const FPRegister& fd, 2107 const Register& rn, 2108 unsigned fbits) { 2109 if (fbits == 0) { 2110 Emit(SF(rn) | FPType(fd) | UCVTF | Rn(rn) | Rd(fd)); 2111 } else { 2112 Emit(SF(rn) | FPType(fd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 2113 Rd(fd)); 2114 } 2115 } 2116 2117 2118 void Assembler::dcptr(Label* label) { 2119 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 2120 if (label->is_bound()) { 2121 // The label is bound, so it does not need to be updated and the internal 2122 // reference should be emitted. 2123 // 2124 // In this case, label->pos() returns the offset of the label from the 2125 // start of the buffer. 2126 internal_reference_positions_.push_back(pc_offset()); 2127 dc64(reinterpret_cast<uintptr_t>(buffer_ + label->pos())); 2128 } else { 2129 int32_t offset; 2130 if (label->is_linked()) { 2131 // The label is linked, so the internal reference should be added 2132 // onto the end of the label's link chain. 2133 // 2134 // In this case, label->pos() returns the offset of the last linked 2135 // instruction from the start of the buffer. 2136 offset = label->pos() - pc_offset(); 2137 DCHECK(offset != kStartOfLabelLinkChain); 2138 } else { 2139 // The label is unused, so it now becomes linked and the internal 2140 // reference is at the start of the new link chain. 2141 offset = kStartOfLabelLinkChain; 2142 } 2143 // The instruction at pc is now the last link in the label's chain. 2144 label->link_to(pc_offset()); 2145 2146 // Traditionally the offset to the previous instruction in the chain is 2147 // encoded in the instruction payload (e.g. branch range) but internal 2148 // references are not instructions so while unbound they are encoded as 2149 // two consecutive brk instructions. The two 16-bit immediates are used 2150 // to encode the offset. 2151 offset >>= kInstructionSizeLog2; 2152 DCHECK(is_int32(offset)); 2153 uint32_t high16 = unsigned_bitextract_32(31, 16, offset); 2154 uint32_t low16 = unsigned_bitextract_32(15, 0, offset); 2155 2156 brk(high16); 2157 brk(low16); 2158 } 2159 } 2160 2161 2162 // Note: 2163 // Below, a difference in case for the same letter indicates a 2164 // negated bit. 2165 // If b is 1, then B is 0. 2166 Instr Assembler::ImmFP32(float imm) { 2167 DCHECK(IsImmFP32(imm)); 2168 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000 2169 uint32_t bits = float_to_rawbits(imm); 2170 // bit7: a000.0000 2171 uint32_t bit7 = ((bits >> 31) & 0x1) << 7; 2172 // bit6: 0b00.0000 2173 uint32_t bit6 = ((bits >> 29) & 0x1) << 6; 2174 // bit5_to_0: 00cd.efgh 2175 uint32_t bit5_to_0 = (bits >> 19) & 0x3f; 2176 2177 return (bit7 | bit6 | bit5_to_0) << ImmFP_offset; 2178 } 2179 2180 2181 Instr Assembler::ImmFP64(double imm) { 2182 DCHECK(IsImmFP64(imm)); 2183 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 2184 // 0000.0000.0000.0000.0000.0000.0000.0000 2185 uint64_t bits = double_to_rawbits(imm); 2186 // bit7: a000.0000 2187 uint64_t bit7 = ((bits >> 63) & 0x1) << 7; 2188 // bit6: 0b00.0000 2189 uint64_t bit6 = ((bits >> 61) & 0x1) << 6; 2190 // bit5_to_0: 00cd.efgh 2191 uint64_t bit5_to_0 = (bits >> 48) & 0x3f; 2192 2193 return static_cast<Instr>((bit7 | bit6 | bit5_to_0) << ImmFP_offset); 2194 } 2195 2196 2197 // Code generation helpers. 2198 void Assembler::MoveWide(const Register& rd, 2199 uint64_t imm, 2200 int shift, 2201 MoveWideImmediateOp mov_op) { 2202 // Ignore the top 32 bits of an immediate if we're moving to a W register. 2203 if (rd.Is32Bits()) { 2204 // Check that the top 32 bits are zero (a positive 32-bit number) or top 2205 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits). 2206 DCHECK(((imm >> kWRegSizeInBits) == 0) || 2207 ((imm >> (kWRegSizeInBits - 1)) == 0x1ffffffff)); 2208 imm &= kWRegMask; 2209 } 2210 2211 if (shift >= 0) { 2212 // Explicit shift specified. 2213 DCHECK((shift == 0) || (shift == 16) || (shift == 32) || (shift == 48)); 2214 DCHECK(rd.Is64Bits() || (shift == 0) || (shift == 16)); 2215 shift /= 16; 2216 } else { 2217 // Calculate a new immediate and shift combination to encode the immediate 2218 // argument. 2219 shift = 0; 2220 if ((imm & ~0xffffUL) == 0) { 2221 // Nothing to do. 2222 } else if ((imm & ~(0xffffUL << 16)) == 0) { 2223 imm >>= 16; 2224 shift = 1; 2225 } else if ((imm & ~(0xffffUL << 32)) == 0) { 2226 DCHECK(rd.Is64Bits()); 2227 imm >>= 32; 2228 shift = 2; 2229 } else if ((imm & ~(0xffffUL << 48)) == 0) { 2230 DCHECK(rd.Is64Bits()); 2231 imm >>= 48; 2232 shift = 3; 2233 } 2234 } 2235 2236 DCHECK(is_uint16(imm)); 2237 2238 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | 2239 ImmMoveWide(static_cast<int>(imm)) | ShiftMoveWide(shift)); 2240 } 2241 2242 2243 void Assembler::AddSub(const Register& rd, 2244 const Register& rn, 2245 const Operand& operand, 2246 FlagsUpdate S, 2247 AddSubOp op) { 2248 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2249 DCHECK(!operand.NeedsRelocation(this)); 2250 if (operand.IsImmediate()) { 2251 int64_t immediate = operand.ImmediateValue(); 2252 DCHECK(IsImmAddSub(immediate)); 2253 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 2254 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) | 2255 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn)); 2256 } else if (operand.IsShiftedRegister()) { 2257 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits()); 2258 DCHECK(operand.shift() != ROR); 2259 2260 // For instructions of the form: 2261 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ] 2262 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ] 2263 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ] 2264 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ] 2265 // or their 64-bit register equivalents, convert the operand from shifted to 2266 // extended register mode, and emit an add/sub extended instruction. 2267 if (rn.IsSP() || rd.IsSP()) { 2268 DCHECK(!(rd.IsSP() && (S == SetFlags))); 2269 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S, 2270 AddSubExtendedFixed | op); 2271 } else { 2272 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op); 2273 } 2274 } else { 2275 DCHECK(operand.IsExtendedRegister()); 2276 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op); 2277 } 2278 } 2279 2280 2281 void Assembler::AddSubWithCarry(const Register& rd, 2282 const Register& rn, 2283 const Operand& operand, 2284 FlagsUpdate S, 2285 AddSubWithCarryOp op) { 2286 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2287 DCHECK(rd.SizeInBits() == operand.reg().SizeInBits()); 2288 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 2289 DCHECK(!operand.NeedsRelocation(this)); 2290 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd)); 2291 } 2292 2293 2294 void Assembler::hlt(int code) { 2295 DCHECK(is_uint16(code)); 2296 Emit(HLT | ImmException(code)); 2297 } 2298 2299 2300 void Assembler::brk(int code) { 2301 DCHECK(is_uint16(code)); 2302 Emit(BRK | ImmException(code)); 2303 } 2304 2305 2306 void Assembler::EmitStringData(const char* string) { 2307 size_t len = strlen(string) + 1; 2308 DCHECK(RoundUp(len, kInstructionSize) <= static_cast<size_t>(kGap)); 2309 EmitData(string, static_cast<int>(len)); 2310 // Pad with NULL characters until pc_ is aligned. 2311 const char pad[] = {'\0', '\0', '\0', '\0'}; 2312 STATIC_ASSERT(sizeof(pad) == kInstructionSize); 2313 EmitData(pad, RoundUp(pc_offset(), kInstructionSize) - pc_offset()); 2314 } 2315 2316 2317 void Assembler::debug(const char* message, uint32_t code, Instr params) { 2318 #ifdef USE_SIMULATOR 2319 // Don't generate simulator specific code if we are building a snapshot, which 2320 // might be run on real hardware. 2321 if (!serializer_enabled()) { 2322 // The arguments to the debug marker need to be contiguous in memory, so 2323 // make sure we don't try to emit pools. 2324 BlockPoolsScope scope(this); 2325 2326 Label start; 2327 bind(&start); 2328 2329 // Refer to instructions-arm64.h for a description of the marker and its 2330 // arguments. 2331 hlt(kImmExceptionIsDebug); 2332 DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugCodeOffset); 2333 dc32(code); 2334 DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugParamsOffset); 2335 dc32(params); 2336 DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugMessageOffset); 2337 EmitStringData(message); 2338 hlt(kImmExceptionIsUnreachable); 2339 2340 return; 2341 } 2342 // Fall through if Serializer is enabled. 2343 #endif 2344 2345 if (params & BREAK) { 2346 hlt(kImmExceptionIsDebug); 2347 } 2348 } 2349 2350 2351 void Assembler::Logical(const Register& rd, 2352 const Register& rn, 2353 const Operand& operand, 2354 LogicalOp op) { 2355 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2356 DCHECK(!operand.NeedsRelocation(this)); 2357 if (operand.IsImmediate()) { 2358 int64_t immediate = operand.ImmediateValue(); 2359 unsigned reg_size = rd.SizeInBits(); 2360 2361 DCHECK(immediate != 0); 2362 DCHECK(immediate != -1); 2363 DCHECK(rd.Is64Bits() || is_uint32(immediate)); 2364 2365 // If the operation is NOT, invert the operation and immediate. 2366 if ((op & NOT) == NOT) { 2367 op = static_cast<LogicalOp>(op & ~NOT); 2368 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask); 2369 } 2370 2371 unsigned n, imm_s, imm_r; 2372 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 2373 // Immediate can be encoded in the instruction. 2374 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 2375 } else { 2376 // This case is handled in the macro assembler. 2377 UNREACHABLE(); 2378 } 2379 } else { 2380 DCHECK(operand.IsShiftedRegister()); 2381 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits()); 2382 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed); 2383 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op); 2384 } 2385 } 2386 2387 2388 void Assembler::LogicalImmediate(const Register& rd, 2389 const Register& rn, 2390 unsigned n, 2391 unsigned imm_s, 2392 unsigned imm_r, 2393 LogicalOp op) { 2394 unsigned reg_size = rd.SizeInBits(); 2395 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd); 2396 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) | 2397 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg | 2398 Rn(rn)); 2399 } 2400 2401 2402 void Assembler::ConditionalCompare(const Register& rn, 2403 const Operand& operand, 2404 StatusFlags nzcv, 2405 Condition cond, 2406 ConditionalCompareOp op) { 2407 Instr ccmpop; 2408 DCHECK(!operand.NeedsRelocation(this)); 2409 if (operand.IsImmediate()) { 2410 int64_t immediate = operand.ImmediateValue(); 2411 DCHECK(IsImmConditionalCompare(immediate)); 2412 ccmpop = ConditionalCompareImmediateFixed | op | 2413 ImmCondCmp(static_cast<unsigned>(immediate)); 2414 } else { 2415 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 2416 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg()); 2417 } 2418 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv)); 2419 } 2420 2421 2422 void Assembler::DataProcessing1Source(const Register& rd, 2423 const Register& rn, 2424 DataProcessing1SourceOp op) { 2425 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2426 Emit(SF(rn) | op | Rn(rn) | Rd(rd)); 2427 } 2428 2429 2430 void Assembler::FPDataProcessing1Source(const FPRegister& fd, 2431 const FPRegister& fn, 2432 FPDataProcessing1SourceOp op) { 2433 Emit(FPType(fn) | op | Rn(fn) | Rd(fd)); 2434 } 2435 2436 2437 void Assembler::FPDataProcessing2Source(const FPRegister& fd, 2438 const FPRegister& fn, 2439 const FPRegister& fm, 2440 FPDataProcessing2SourceOp op) { 2441 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 2442 DCHECK(fd.SizeInBits() == fm.SizeInBits()); 2443 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd)); 2444 } 2445 2446 2447 void Assembler::FPDataProcessing3Source(const FPRegister& fd, 2448 const FPRegister& fn, 2449 const FPRegister& fm, 2450 const FPRegister& fa, 2451 FPDataProcessing3SourceOp op) { 2452 DCHECK(AreSameSizeAndType(fd, fn, fm, fa)); 2453 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd) | Ra(fa)); 2454 } 2455 2456 2457 void Assembler::EmitShift(const Register& rd, 2458 const Register& rn, 2459 Shift shift, 2460 unsigned shift_amount) { 2461 switch (shift) { 2462 case LSL: 2463 lsl(rd, rn, shift_amount); 2464 break; 2465 case LSR: 2466 lsr(rd, rn, shift_amount); 2467 break; 2468 case ASR: 2469 asr(rd, rn, shift_amount); 2470 break; 2471 case ROR: 2472 ror(rd, rn, shift_amount); 2473 break; 2474 default: 2475 UNREACHABLE(); 2476 } 2477 } 2478 2479 2480 void Assembler::EmitExtendShift(const Register& rd, 2481 const Register& rn, 2482 Extend extend, 2483 unsigned left_shift) { 2484 DCHECK(rd.SizeInBits() >= rn.SizeInBits()); 2485 unsigned reg_size = rd.SizeInBits(); 2486 // Use the correct size of register. 2487 Register rn_ = Register::Create(rn.code(), rd.SizeInBits()); 2488 // Bits extracted are high_bit:0. 2489 unsigned high_bit = (8 << (extend & 0x3)) - 1; 2490 // Number of bits left in the result that are not introduced by the shift. 2491 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1); 2492 2493 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) { 2494 switch (extend) { 2495 case UXTB: 2496 case UXTH: 2497 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break; 2498 case SXTB: 2499 case SXTH: 2500 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break; 2501 case UXTX: 2502 case SXTX: { 2503 DCHECK(rn.SizeInBits() == kXRegSizeInBits); 2504 // Nothing to extend. Just shift. 2505 lsl(rd, rn_, left_shift); 2506 break; 2507 } 2508 default: UNREACHABLE(); 2509 } 2510 } else { 2511 // No need to extend as the extended bits would be shifted away. 2512 lsl(rd, rn_, left_shift); 2513 } 2514 } 2515 2516 2517 void Assembler::DataProcShiftedRegister(const Register& rd, 2518 const Register& rn, 2519 const Operand& operand, 2520 FlagsUpdate S, 2521 Instr op) { 2522 DCHECK(operand.IsShiftedRegister()); 2523 DCHECK(rn.Is64Bits() || (rn.Is32Bits() && is_uint5(operand.shift_amount()))); 2524 DCHECK(!operand.NeedsRelocation(this)); 2525 Emit(SF(rd) | op | Flags(S) | 2526 ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) | 2527 Rm(operand.reg()) | Rn(rn) | Rd(rd)); 2528 } 2529 2530 2531 void Assembler::DataProcExtendedRegister(const Register& rd, 2532 const Register& rn, 2533 const Operand& operand, 2534 FlagsUpdate S, 2535 Instr op) { 2536 DCHECK(!operand.NeedsRelocation(this)); 2537 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 2538 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | 2539 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) | 2540 dest_reg | RnSP(rn)); 2541 } 2542 2543 2544 bool Assembler::IsImmAddSub(int64_t immediate) { 2545 return is_uint12(immediate) || 2546 (is_uint12(immediate >> 12) && ((immediate & 0xfff) == 0)); 2547 } 2548 2549 void Assembler::LoadStore(const CPURegister& rt, 2550 const MemOperand& addr, 2551 LoadStoreOp op) { 2552 Instr memop = op | Rt(rt) | RnSP(addr.base()); 2553 2554 if (addr.IsImmediateOffset()) { 2555 LSDataSize size = CalcLSDataSize(op); 2556 if (IsImmLSScaled(addr.offset(), size)) { 2557 int offset = static_cast<int>(addr.offset()); 2558 // Use the scaled addressing mode. 2559 Emit(LoadStoreUnsignedOffsetFixed | memop | 2560 ImmLSUnsigned(offset >> size)); 2561 } else if (IsImmLSUnscaled(addr.offset())) { 2562 int offset = static_cast<int>(addr.offset()); 2563 // Use the unscaled addressing mode. 2564 Emit(LoadStoreUnscaledOffsetFixed | memop | ImmLS(offset)); 2565 } else { 2566 // This case is handled in the macro assembler. 2567 UNREACHABLE(); 2568 } 2569 } else if (addr.IsRegisterOffset()) { 2570 Extend ext = addr.extend(); 2571 Shift shift = addr.shift(); 2572 unsigned shift_amount = addr.shift_amount(); 2573 2574 // LSL is encoded in the option field as UXTX. 2575 if (shift == LSL) { 2576 ext = UXTX; 2577 } 2578 2579 // Shifts are encoded in one bit, indicating a left shift by the memory 2580 // access size. 2581 DCHECK((shift_amount == 0) || 2582 (shift_amount == static_cast<unsigned>(CalcLSDataSize(op)))); 2583 Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) | 2584 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0)); 2585 } else { 2586 // Pre-index and post-index modes. 2587 DCHECK(!rt.Is(addr.base())); 2588 if (IsImmLSUnscaled(addr.offset())) { 2589 int offset = static_cast<int>(addr.offset()); 2590 if (addr.IsPreIndex()) { 2591 Emit(LoadStorePreIndexFixed | memop | ImmLS(offset)); 2592 } else { 2593 DCHECK(addr.IsPostIndex()); 2594 Emit(LoadStorePostIndexFixed | memop | ImmLS(offset)); 2595 } 2596 } else { 2597 // This case is handled in the macro assembler. 2598 UNREACHABLE(); 2599 } 2600 } 2601 } 2602 2603 2604 bool Assembler::IsImmLSUnscaled(int64_t offset) { 2605 return is_int9(offset); 2606 } 2607 2608 2609 bool Assembler::IsImmLSScaled(int64_t offset, LSDataSize size) { 2610 bool offset_is_size_multiple = (((offset >> size) << size) == offset); 2611 return offset_is_size_multiple && is_uint12(offset >> size); 2612 } 2613 2614 2615 bool Assembler::IsImmLSPair(int64_t offset, LSDataSize size) { 2616 bool offset_is_size_multiple = (((offset >> size) << size) == offset); 2617 return offset_is_size_multiple && is_int7(offset >> size); 2618 } 2619 2620 2621 bool Assembler::IsImmLLiteral(int64_t offset) { 2622 int inst_size = static_cast<int>(kInstructionSizeLog2); 2623 bool offset_is_inst_multiple = 2624 (((offset >> inst_size) << inst_size) == offset); 2625 return offset_is_inst_multiple && is_intn(offset, ImmLLiteral_width); 2626 } 2627 2628 2629 // Test if a given value can be encoded in the immediate field of a logical 2630 // instruction. 2631 // If it can be encoded, the function returns true, and values pointed to by n, 2632 // imm_s and imm_r are updated with immediates encoded in the format required 2633 // by the corresponding fields in the logical instruction. 2634 // If it can not be encoded, the function returns false, and the values pointed 2635 // to by n, imm_s and imm_r are undefined. 2636 bool Assembler::IsImmLogical(uint64_t value, 2637 unsigned width, 2638 unsigned* n, 2639 unsigned* imm_s, 2640 unsigned* imm_r) { 2641 DCHECK((n != NULL) && (imm_s != NULL) && (imm_r != NULL)); 2642 DCHECK((width == kWRegSizeInBits) || (width == kXRegSizeInBits)); 2643 2644 bool negate = false; 2645 2646 // Logical immediates are encoded using parameters n, imm_s and imm_r using 2647 // the following table: 2648 // 2649 // N imms immr size S R 2650 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr) 2651 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr) 2652 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr) 2653 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr) 2654 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr) 2655 // 0 11110s xxxxxr 2 UInt(s) UInt(r) 2656 // (s bits must not be all set) 2657 // 2658 // A pattern is constructed of size bits, where the least significant S+1 bits 2659 // are set. The pattern is rotated right by R, and repeated across a 32 or 2660 // 64-bit value, depending on destination register width. 2661 // 2662 // Put another way: the basic format of a logical immediate is a single 2663 // contiguous stretch of 1 bits, repeated across the whole word at intervals 2664 // given by a power of 2. To identify them quickly, we first locate the 2665 // lowest stretch of 1 bits, then the next 1 bit above that; that combination 2666 // is different for every logical immediate, so it gives us all the 2667 // information we need to identify the only logical immediate that our input 2668 // could be, and then we simply check if that's the value we actually have. 2669 // 2670 // (The rotation parameter does give the possibility of the stretch of 1 bits 2671 // going 'round the end' of the word. To deal with that, we observe that in 2672 // any situation where that happens the bitwise NOT of the value is also a 2673 // valid logical immediate. So we simply invert the input whenever its low bit 2674 // is set, and then we know that the rotated case can't arise.) 2675 2676 if (value & 1) { 2677 // If the low bit is 1, negate the value, and set a flag to remember that we 2678 // did (so that we can adjust the return values appropriately). 2679 negate = true; 2680 value = ~value; 2681 } 2682 2683 if (width == kWRegSizeInBits) { 2684 // To handle 32-bit logical immediates, the very easiest thing is to repeat 2685 // the input value twice to make a 64-bit word. The correct encoding of that 2686 // as a logical immediate will also be the correct encoding of the 32-bit 2687 // value. 2688 2689 // The most-significant 32 bits may not be zero (ie. negate is true) so 2690 // shift the value left before duplicating it. 2691 value <<= kWRegSizeInBits; 2692 value |= value >> kWRegSizeInBits; 2693 } 2694 2695 // The basic analysis idea: imagine our input word looks like this. 2696 // 2697 // 0011111000111110001111100011111000111110001111100011111000111110 2698 // c b a 2699 // |<--d-->| 2700 // 2701 // We find the lowest set bit (as an actual power-of-2 value, not its index) 2702 // and call it a. Then we add a to our original number, which wipes out the 2703 // bottommost stretch of set bits and replaces it with a 1 carried into the 2704 // next zero bit. Then we look for the new lowest set bit, which is in 2705 // position b, and subtract it, so now our number is just like the original 2706 // but with the lowest stretch of set bits completely gone. Now we find the 2707 // lowest set bit again, which is position c in the diagram above. Then we'll 2708 // measure the distance d between bit positions a and c (using CLZ), and that 2709 // tells us that the only valid logical immediate that could possibly be equal 2710 // to this number is the one in which a stretch of bits running from a to just 2711 // below b is replicated every d bits. 2712 uint64_t a = LargestPowerOf2Divisor(value); 2713 uint64_t value_plus_a = value + a; 2714 uint64_t b = LargestPowerOf2Divisor(value_plus_a); 2715 uint64_t value_plus_a_minus_b = value_plus_a - b; 2716 uint64_t c = LargestPowerOf2Divisor(value_plus_a_minus_b); 2717 2718 int d, clz_a, out_n; 2719 uint64_t mask; 2720 2721 if (c != 0) { 2722 // The general case, in which there is more than one stretch of set bits. 2723 // Compute the repeat distance d, and set up a bitmask covering the basic 2724 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all 2725 // of these cases the N bit of the output will be zero. 2726 clz_a = CountLeadingZeros(a, kXRegSizeInBits); 2727 int clz_c = CountLeadingZeros(c, kXRegSizeInBits); 2728 d = clz_a - clz_c; 2729 mask = ((V8_UINT64_C(1) << d) - 1); 2730 out_n = 0; 2731 } else { 2732 // Handle degenerate cases. 2733 // 2734 // If any of those 'find lowest set bit' operations didn't find a set bit at 2735 // all, then the word will have been zero thereafter, so in particular the 2736 // last lowest_set_bit operation will have returned zero. So we can test for 2737 // all the special case conditions in one go by seeing if c is zero. 2738 if (a == 0) { 2739 // The input was zero (or all 1 bits, which will come to here too after we 2740 // inverted it at the start of the function), for which we just return 2741 // false. 2742 return false; 2743 } else { 2744 // Otherwise, if c was zero but a was not, then there's just one stretch 2745 // of set bits in our word, meaning that we have the trivial case of 2746 // d == 64 and only one 'repetition'. Set up all the same variables as in 2747 // the general case above, and set the N bit in the output. 2748 clz_a = CountLeadingZeros(a, kXRegSizeInBits); 2749 d = 64; 2750 mask = ~V8_UINT64_C(0); 2751 out_n = 1; 2752 } 2753 } 2754 2755 // If the repeat period d is not a power of two, it can't be encoded. 2756 if (!IS_POWER_OF_TWO(d)) { 2757 return false; 2758 } 2759 2760 if (((b - a) & ~mask) != 0) { 2761 // If the bit stretch (b - a) does not fit within the mask derived from the 2762 // repeat period, then fail. 2763 return false; 2764 } 2765 2766 // The only possible option is b - a repeated every d bits. Now we're going to 2767 // actually construct the valid logical immediate derived from that 2768 // specification, and see if it equals our original input. 2769 // 2770 // To repeat a value every d bits, we multiply it by a number of the form 2771 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can 2772 // be derived using a table lookup on CLZ(d). 2773 static const uint64_t multipliers[] = { 2774 0x0000000000000001UL, 2775 0x0000000100000001UL, 2776 0x0001000100010001UL, 2777 0x0101010101010101UL, 2778 0x1111111111111111UL, 2779 0x5555555555555555UL, 2780 }; 2781 int multiplier_idx = CountLeadingZeros(d, kXRegSizeInBits) - 57; 2782 // Ensure that the index to the multipliers array is within bounds. 2783 DCHECK((multiplier_idx >= 0) && 2784 (static_cast<size_t>(multiplier_idx) < arraysize(multipliers))); 2785 uint64_t multiplier = multipliers[multiplier_idx]; 2786 uint64_t candidate = (b - a) * multiplier; 2787 2788 if (value != candidate) { 2789 // The candidate pattern doesn't match our input value, so fail. 2790 return false; 2791 } 2792 2793 // We have a match! This is a valid logical immediate, so now we have to 2794 // construct the bits and pieces of the instruction encoding that generates 2795 // it. 2796 2797 // Count the set bits in our basic stretch. The special case of clz(0) == -1 2798 // makes the answer come out right for stretches that reach the very top of 2799 // the word (e.g. numbers like 0xffffc00000000000). 2800 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSizeInBits); 2801 int s = clz_a - clz_b; 2802 2803 // Decide how many bits to rotate right by, to put the low bit of that basic 2804 // stretch in position a. 2805 int r; 2806 if (negate) { 2807 // If we inverted the input right at the start of this function, here's 2808 // where we compensate: the number of set bits becomes the number of clear 2809 // bits, and the rotation count is based on position b rather than position 2810 // a (since b is the location of the 'lowest' 1 bit after inversion). 2811 s = d - s; 2812 r = (clz_b + 1) & (d - 1); 2813 } else { 2814 r = (clz_a + 1) & (d - 1); 2815 } 2816 2817 // Now we're done, except for having to encode the S output in such a way that 2818 // it gives both the number of set bits and the length of the repeated 2819 // segment. The s field is encoded like this: 2820 // 2821 // imms size S 2822 // ssssss 64 UInt(ssssss) 2823 // 0sssss 32 UInt(sssss) 2824 // 10ssss 16 UInt(ssss) 2825 // 110sss 8 UInt(sss) 2826 // 1110ss 4 UInt(ss) 2827 // 11110s 2 UInt(s) 2828 // 2829 // So we 'or' (-d << 1) with our computed s to form imms. 2830 *n = out_n; 2831 *imm_s = ((-d << 1) | (s - 1)) & 0x3f; 2832 *imm_r = r; 2833 2834 return true; 2835 } 2836 2837 2838 bool Assembler::IsImmConditionalCompare(int64_t immediate) { 2839 return is_uint5(immediate); 2840 } 2841 2842 2843 bool Assembler::IsImmFP32(float imm) { 2844 // Valid values will have the form: 2845 // aBbb.bbbc.defg.h000.0000.0000.0000.0000 2846 uint32_t bits = float_to_rawbits(imm); 2847 // bits[19..0] are cleared. 2848 if ((bits & 0x7ffff) != 0) { 2849 return false; 2850 } 2851 2852 // bits[29..25] are all set or all cleared. 2853 uint32_t b_pattern = (bits >> 16) & 0x3e00; 2854 if (b_pattern != 0 && b_pattern != 0x3e00) { 2855 return false; 2856 } 2857 2858 // bit[30] and bit[29] are opposite. 2859 if (((bits ^ (bits << 1)) & 0x40000000) == 0) { 2860 return false; 2861 } 2862 2863 return true; 2864 } 2865 2866 2867 bool Assembler::IsImmFP64(double imm) { 2868 // Valid values will have the form: 2869 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 2870 // 0000.0000.0000.0000.0000.0000.0000.0000 2871 uint64_t bits = double_to_rawbits(imm); 2872 // bits[47..0] are cleared. 2873 if ((bits & 0xffffffffffffL) != 0) { 2874 return false; 2875 } 2876 2877 // bits[61..54] are all set or all cleared. 2878 uint32_t b_pattern = (bits >> 48) & 0x3fc0; 2879 if (b_pattern != 0 && b_pattern != 0x3fc0) { 2880 return false; 2881 } 2882 2883 // bit[62] and bit[61] are opposite. 2884 if (((bits ^ (bits << 1)) & 0x4000000000000000L) == 0) { 2885 return false; 2886 } 2887 2888 return true; 2889 } 2890 2891 2892 void Assembler::GrowBuffer() { 2893 if (!own_buffer_) FATAL("external code buffer is too small"); 2894 2895 // Compute new buffer size. 2896 CodeDesc desc; // the new buffer 2897 if (buffer_size_ < 1 * MB) { 2898 desc.buffer_size = 2 * buffer_size_; 2899 } else { 2900 desc.buffer_size = buffer_size_ + 1 * MB; 2901 } 2902 CHECK_GT(desc.buffer_size, 0); // No overflow. 2903 2904 byte* buffer = reinterpret_cast<byte*>(buffer_); 2905 2906 // Set up new buffer. 2907 desc.buffer = NewArray<byte>(desc.buffer_size); 2908 desc.origin = this; 2909 2910 desc.instr_size = pc_offset(); 2911 desc.reloc_size = 2912 static_cast<int>((buffer + buffer_size_) - reloc_info_writer.pos()); 2913 2914 // Copy the data. 2915 intptr_t pc_delta = desc.buffer - buffer; 2916 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 2917 (buffer + buffer_size_); 2918 memmove(desc.buffer, buffer, desc.instr_size); 2919 memmove(reloc_info_writer.pos() + rc_delta, 2920 reloc_info_writer.pos(), desc.reloc_size); 2921 2922 // Switch buffers. 2923 DeleteArray(buffer_); 2924 buffer_ = desc.buffer; 2925 buffer_size_ = desc.buffer_size; 2926 pc_ = reinterpret_cast<byte*>(pc_) + pc_delta; 2927 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 2928 reloc_info_writer.last_pc() + pc_delta); 2929 2930 // None of our relocation types are pc relative pointing outside the code 2931 // buffer nor pc absolute pointing inside the code buffer, so there is no need 2932 // to relocate any emitted relocation entries. 2933 2934 // Relocate internal references. 2935 for (auto pos : internal_reference_positions_) { 2936 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 2937 *p += pc_delta; 2938 } 2939 2940 // Pending relocation entries are also relative, no need to relocate. 2941 } 2942 2943 2944 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { 2945 // We do not try to reuse pool constants. 2946 RelocInfo rinfo(isolate(), reinterpret_cast<byte*>(pc_), rmode, data, NULL); 2947 if (((rmode >= RelocInfo::COMMENT) && 2948 (rmode <= RelocInfo::DEBUG_BREAK_SLOT_AT_TAIL_CALL)) || 2949 (rmode == RelocInfo::INTERNAL_REFERENCE) || 2950 (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) || 2951 (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) || 2952 (rmode == RelocInfo::DEOPT_INLINING_ID) || 2953 (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID) || 2954 (rmode == RelocInfo::GENERATOR_CONTINUATION)) { 2955 // Adjust code for new modes. 2956 DCHECK(RelocInfo::IsDebugBreakSlot(rmode) || RelocInfo::IsComment(rmode) || 2957 RelocInfo::IsDeoptReason(rmode) || RelocInfo::IsDeoptId(rmode) || 2958 RelocInfo::IsDeoptPosition(rmode) || 2959 RelocInfo::IsInternalReference(rmode) || 2960 RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode) || 2961 RelocInfo::IsGeneratorContinuation(rmode)); 2962 // These modes do not need an entry in the constant pool. 2963 } else { 2964 constpool_.RecordEntry(data, rmode); 2965 // Make sure the constant pool is not emitted in place of the next 2966 // instruction for which we just recorded relocation info. 2967 BlockConstPoolFor(1); 2968 } 2969 2970 if (!RelocInfo::IsNone(rmode)) { 2971 // Don't record external references unless the heap will be serialized. 2972 if (rmode == RelocInfo::EXTERNAL_REFERENCE && 2973 !serializer_enabled() && !emit_debug_code()) { 2974 return; 2975 } 2976 DCHECK(buffer_space() >= kMaxRelocSize); // too late to grow buffer here 2977 if (rmode == RelocInfo::CODE_TARGET_WITH_ID) { 2978 RelocInfo reloc_info_with_ast_id(isolate(), reinterpret_cast<byte*>(pc_), 2979 rmode, RecordedAstId().ToInt(), NULL); 2980 ClearRecordedAstId(); 2981 reloc_info_writer.Write(&reloc_info_with_ast_id); 2982 } else { 2983 reloc_info_writer.Write(&rinfo); 2984 } 2985 } 2986 } 2987 2988 2989 void Assembler::BlockConstPoolFor(int instructions) { 2990 int pc_limit = pc_offset() + instructions * kInstructionSize; 2991 if (no_const_pool_before_ < pc_limit) { 2992 no_const_pool_before_ = pc_limit; 2993 // Make sure the pool won't be blocked for too long. 2994 DCHECK(pc_limit < constpool_.MaxPcOffset()); 2995 } 2996 2997 if (next_constant_pool_check_ < no_const_pool_before_) { 2998 next_constant_pool_check_ = no_const_pool_before_; 2999 } 3000 } 3001 3002 3003 void Assembler::CheckConstPool(bool force_emit, bool require_jump) { 3004 // Some short sequence of instruction mustn't be broken up by constant pool 3005 // emission, such sequences are protected by calls to BlockConstPoolFor and 3006 // BlockConstPoolScope. 3007 if (is_const_pool_blocked()) { 3008 // Something is wrong if emission is forced and blocked at the same time. 3009 DCHECK(!force_emit); 3010 return; 3011 } 3012 3013 // There is nothing to do if there are no pending constant pool entries. 3014 if (constpool_.IsEmpty()) { 3015 // Calculate the offset of the next check. 3016 SetNextConstPoolCheckIn(kCheckConstPoolInterval); 3017 return; 3018 } 3019 3020 // We emit a constant pool when: 3021 // * requested to do so by parameter force_emit (e.g. after each function). 3022 // * the distance to the first instruction accessing the constant pool is 3023 // kApproxMaxDistToConstPool or more. 3024 // * the number of entries in the pool is kApproxMaxPoolEntryCount or more. 3025 int dist = constpool_.DistanceToFirstUse(); 3026 int count = constpool_.EntryCount(); 3027 if (!force_emit && 3028 (dist < kApproxMaxDistToConstPool) && 3029 (count < kApproxMaxPoolEntryCount)) { 3030 return; 3031 } 3032 3033 3034 // Emit veneers for branches that would go out of range during emission of the 3035 // constant pool. 3036 int worst_case_size = constpool_.WorstCaseSize(); 3037 CheckVeneerPool(false, require_jump, 3038 kVeneerDistanceMargin + worst_case_size); 3039 3040 // Check that the code buffer is large enough before emitting the constant 3041 // pool (this includes the gap to the relocation information). 3042 int needed_space = worst_case_size + kGap + 1 * kInstructionSize; 3043 while (buffer_space() <= needed_space) { 3044 GrowBuffer(); 3045 } 3046 3047 Label size_check; 3048 bind(&size_check); 3049 constpool_.Emit(require_jump); 3050 DCHECK(SizeOfCodeGeneratedSince(&size_check) <= 3051 static_cast<unsigned>(worst_case_size)); 3052 3053 // Since a constant pool was just emitted, move the check offset forward by 3054 // the standard interval. 3055 SetNextConstPoolCheckIn(kCheckConstPoolInterval); 3056 } 3057 3058 3059 bool Assembler::ShouldEmitVeneer(int max_reachable_pc, int margin) { 3060 // Account for the branch around the veneers and the guard. 3061 int protection_offset = 2 * kInstructionSize; 3062 return pc_offset() > max_reachable_pc - margin - protection_offset - 3063 static_cast<int>(unresolved_branches_.size() * kMaxVeneerCodeSize); 3064 } 3065 3066 3067 void Assembler::RecordVeneerPool(int location_offset, int size) { 3068 RelocInfo rinfo(isolate(), buffer_ + location_offset, RelocInfo::VENEER_POOL, 3069 static_cast<intptr_t>(size), NULL); 3070 reloc_info_writer.Write(&rinfo); 3071 } 3072 3073 3074 void Assembler::EmitVeneers(bool force_emit, bool need_protection, int margin) { 3075 BlockPoolsScope scope(this); 3076 RecordComment("[ Veneers"); 3077 3078 // The exact size of the veneer pool must be recorded (see the comment at the 3079 // declaration site of RecordConstPool()), but computing the number of 3080 // veneers that will be generated is not obvious. So instead we remember the 3081 // current position and will record the size after the pool has been 3082 // generated. 3083 Label size_check; 3084 bind(&size_check); 3085 int veneer_pool_relocinfo_loc = pc_offset(); 3086 3087 Label end; 3088 if (need_protection) { 3089 b(&end); 3090 } 3091 3092 EmitVeneersGuard(); 3093 3094 Label veneer_size_check; 3095 3096 std::multimap<int, FarBranchInfo>::iterator it, it_to_delete; 3097 3098 it = unresolved_branches_.begin(); 3099 while (it != unresolved_branches_.end()) { 3100 if (force_emit || ShouldEmitVeneer(it->first, margin)) { 3101 Instruction* branch = InstructionAt(it->second.pc_offset_); 3102 Label* label = it->second.label_; 3103 3104 #ifdef DEBUG 3105 bind(&veneer_size_check); 3106 #endif 3107 // Patch the branch to point to the current position, and emit a branch 3108 // to the label. 3109 Instruction* veneer = reinterpret_cast<Instruction*>(pc_); 3110 RemoveBranchFromLabelLinkChain(branch, label, veneer); 3111 branch->SetImmPCOffsetTarget(isolate(), veneer); 3112 b(label); 3113 #ifdef DEBUG 3114 DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <= 3115 static_cast<uint64_t>(kMaxVeneerCodeSize)); 3116 veneer_size_check.Unuse(); 3117 #endif 3118 3119 it_to_delete = it++; 3120 unresolved_branches_.erase(it_to_delete); 3121 } else { 3122 ++it; 3123 } 3124 } 3125 3126 // Record the veneer pool size. 3127 int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check)); 3128 RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size); 3129 3130 if (unresolved_branches_.empty()) { 3131 next_veneer_pool_check_ = kMaxInt; 3132 } else { 3133 next_veneer_pool_check_ = 3134 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 3135 } 3136 3137 bind(&end); 3138 3139 RecordComment("]"); 3140 } 3141 3142 3143 void Assembler::CheckVeneerPool(bool force_emit, bool require_jump, 3144 int margin) { 3145 // There is nothing to do if there are no pending veneer pool entries. 3146 if (unresolved_branches_.empty()) { 3147 DCHECK(next_veneer_pool_check_ == kMaxInt); 3148 return; 3149 } 3150 3151 DCHECK(pc_offset() < unresolved_branches_first_limit()); 3152 3153 // Some short sequence of instruction mustn't be broken up by veneer pool 3154 // emission, such sequences are protected by calls to BlockVeneerPoolFor and 3155 // BlockVeneerPoolScope. 3156 if (is_veneer_pool_blocked()) { 3157 DCHECK(!force_emit); 3158 return; 3159 } 3160 3161 if (!require_jump) { 3162 // Prefer emitting veneers protected by an existing instruction. 3163 margin *= kVeneerNoProtectionFactor; 3164 } 3165 if (force_emit || ShouldEmitVeneers(margin)) { 3166 EmitVeneers(force_emit, require_jump, margin); 3167 } else { 3168 next_veneer_pool_check_ = 3169 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 3170 } 3171 } 3172 3173 3174 int Assembler::buffer_space() const { 3175 return static_cast<int>(reloc_info_writer.pos() - 3176 reinterpret_cast<byte*>(pc_)); 3177 } 3178 3179 3180 void Assembler::RecordConstPool(int size) { 3181 // We only need this for debugger support, to correctly compute offsets in the 3182 // code. 3183 RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size)); 3184 } 3185 3186 3187 void PatchingAssembler::PatchAdrFar(int64_t target_offset) { 3188 // The code at the current instruction should be: 3189 // adr rd, 0 3190 // nop (adr_far) 3191 // nop (adr_far) 3192 // movz scratch, 0 3193 3194 // Verify the expected code. 3195 Instruction* expected_adr = InstructionAt(0); 3196 CHECK(expected_adr->IsAdr() && (expected_adr->ImmPCRel() == 0)); 3197 int rd_code = expected_adr->Rd(); 3198 for (int i = 0; i < kAdrFarPatchableNNops; ++i) { 3199 CHECK(InstructionAt((i + 1) * kInstructionSize)->IsNop(ADR_FAR_NOP)); 3200 } 3201 Instruction* expected_movz = 3202 InstructionAt((kAdrFarPatchableNInstrs - 1) * kInstructionSize); 3203 CHECK(expected_movz->IsMovz() && 3204 (expected_movz->ImmMoveWide() == 0) && 3205 (expected_movz->ShiftMoveWide() == 0)); 3206 int scratch_code = expected_movz->Rd(); 3207 3208 // Patch to load the correct address. 3209 Register rd = Register::XRegFromCode(rd_code); 3210 Register scratch = Register::XRegFromCode(scratch_code); 3211 // Addresses are only 48 bits. 3212 adr(rd, target_offset & 0xFFFF); 3213 movz(scratch, (target_offset >> 16) & 0xFFFF, 16); 3214 movk(scratch, (target_offset >> 32) & 0xFFFF, 32); 3215 DCHECK((target_offset >> 48) == 0); 3216 add(rd, rd, scratch); 3217 } 3218 3219 3220 } // namespace internal 3221 } // namespace v8 3222 3223 #endif // V8_TARGET_ARCH_ARM64 3224