1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // 3 // Redistribution and use in source and binary forms, with or without 4 // modification, are permitted provided that the following conditions are 5 // met: 6 // 7 // * Redistributions of source code must retain the above copyright 8 // notice, this list of conditions and the following disclaimer. 9 // * Redistributions in binary form must reproduce the above 10 // copyright notice, this list of conditions and the following 11 // disclaimer in the documentation and/or other materials provided 12 // with the distribution. 13 // * Neither the name of Google Inc. nor the names of its 14 // contributors may be used to endorse or promote products derived 15 // from this software without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #if V8_TARGET_ARCH_ARM64 30 31 #define ARM64_DEFINE_REG_STATICS 32 #include "src/arm64/assembler-arm64.h" 33 34 #include "src/arm64/assembler-arm64-inl.h" 35 #include "src/arm64/frames-arm64.h" 36 #include "src/base/bits.h" 37 #include "src/base/cpu.h" 38 #include "src/register-configuration.h" 39 40 namespace v8 { 41 namespace internal { 42 43 44 // ----------------------------------------------------------------------------- 45 // CpuFeatures implementation. 46 47 void CpuFeatures::ProbeImpl(bool cross_compile) { 48 // AArch64 has no configuration options, no further probing is required. 49 supported_ = 0; 50 51 // Only use statically determined features for cross compile (snapshot). 52 if (cross_compile) return; 53 54 // We used to probe for coherent cache support, but on older CPUs it 55 // causes crashes (crbug.com/524337), and newer CPUs don't even have 56 // the feature any more. 57 } 58 59 void CpuFeatures::PrintTarget() { } 60 void CpuFeatures::PrintFeatures() {} 61 62 // ----------------------------------------------------------------------------- 63 // CPURegList utilities. 64 65 CPURegister CPURegList::PopLowestIndex() { 66 DCHECK(IsValid()); 67 if (IsEmpty()) { 68 return NoCPUReg; 69 } 70 int index = CountTrailingZeros(list_, kRegListSizeInBits); 71 DCHECK((1 << index) & list_); 72 Remove(index); 73 return CPURegister::Create(index, size_, type_); 74 } 75 76 77 CPURegister CPURegList::PopHighestIndex() { 78 DCHECK(IsValid()); 79 if (IsEmpty()) { 80 return NoCPUReg; 81 } 82 int index = CountLeadingZeros(list_, kRegListSizeInBits); 83 index = kRegListSizeInBits - 1 - index; 84 DCHECK((1 << index) & list_); 85 Remove(index); 86 return CPURegister::Create(index, size_, type_); 87 } 88 89 90 void CPURegList::RemoveCalleeSaved() { 91 if (type() == CPURegister::kRegister) { 92 Remove(GetCalleeSaved(RegisterSizeInBits())); 93 } else if (type() == CPURegister::kFPRegister) { 94 Remove(GetCalleeSavedFP(RegisterSizeInBits())); 95 } else { 96 DCHECK(type() == CPURegister::kNoRegister); 97 DCHECK(IsEmpty()); 98 // The list must already be empty, so do nothing. 99 } 100 } 101 102 103 CPURegList CPURegList::GetCalleeSaved(int size) { 104 return CPURegList(CPURegister::kRegister, size, 19, 29); 105 } 106 107 108 CPURegList CPURegList::GetCalleeSavedFP(int size) { 109 return CPURegList(CPURegister::kFPRegister, size, 8, 15); 110 } 111 112 113 CPURegList CPURegList::GetCallerSaved(int size) { 114 // Registers x0-x18 and lr (x30) are caller-saved. 115 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18); 116 list.Combine(lr); 117 return list; 118 } 119 120 121 CPURegList CPURegList::GetCallerSavedFP(int size) { 122 // Registers d0-d7 and d16-d31 are caller-saved. 123 CPURegList list = CPURegList(CPURegister::kFPRegister, size, 0, 7); 124 list.Combine(CPURegList(CPURegister::kFPRegister, size, 16, 31)); 125 return list; 126 } 127 128 129 // This function defines the list of registers which are associated with a 130 // safepoint slot. Safepoint register slots are saved contiguously on the stack. 131 // MacroAssembler::SafepointRegisterStackIndex handles mapping from register 132 // code to index in the safepoint register slots. Any change here can affect 133 // this mapping. 134 CPURegList CPURegList::GetSafepointSavedRegisters() { 135 CPURegList list = CPURegList::GetCalleeSaved(); 136 list.Combine( 137 CPURegList(CPURegister::kRegister, kXRegSizeInBits, kJSCallerSaved)); 138 139 // Note that unfortunately we can't use symbolic names for registers and have 140 // to directly use register codes. This is because this function is used to 141 // initialize some static variables and we can't rely on register variables 142 // to be initialized due to static initialization order issues in C++. 143 144 // Drop ip0 and ip1 (i.e. x16 and x17), as they should not be expected to be 145 // preserved outside of the macro assembler. 146 list.Remove(16); 147 list.Remove(17); 148 149 // Add x18 to the safepoint list, as although it's not in kJSCallerSaved, it 150 // is a caller-saved register according to the procedure call standard. 151 list.Combine(18); 152 153 // Drop jssp as the stack pointer doesn't need to be included. 154 list.Remove(28); 155 156 // Add the link register (x30) to the safepoint list. 157 list.Combine(30); 158 159 return list; 160 } 161 162 163 // ----------------------------------------------------------------------------- 164 // Implementation of RelocInfo 165 166 const int RelocInfo::kApplyMask = 1 << RelocInfo::INTERNAL_REFERENCE; 167 168 169 bool RelocInfo::IsCodedSpecially() { 170 // The deserializer needs to know whether a pointer is specially coded. Being 171 // specially coded on ARM64 means that it is a movz/movk sequence. We don't 172 // generate those for relocatable pointers. 173 return false; 174 } 175 176 177 bool RelocInfo::IsInConstantPool() { 178 Instruction* instr = reinterpret_cast<Instruction*>(pc_); 179 return instr->IsLdrLiteralX(); 180 } 181 182 Address RelocInfo::wasm_memory_reference() { 183 DCHECK(IsWasmMemoryReference(rmode_)); 184 return Memory::Address_at(Assembler::target_pointer_address_at(pc_)); 185 } 186 187 uint32_t RelocInfo::wasm_memory_size_reference() { 188 DCHECK(IsWasmMemorySizeReference(rmode_)); 189 return Memory::uint32_at(Assembler::target_pointer_address_at(pc_)); 190 } 191 192 Address RelocInfo::wasm_global_reference() { 193 DCHECK(IsWasmGlobalReference(rmode_)); 194 return Memory::Address_at(Assembler::target_pointer_address_at(pc_)); 195 } 196 197 uint32_t RelocInfo::wasm_function_table_size_reference() { 198 DCHECK(IsWasmFunctionTableSizeReference(rmode_)); 199 return Memory::uint32_at(Assembler::target_pointer_address_at(pc_)); 200 } 201 202 void RelocInfo::unchecked_update_wasm_memory_reference( 203 Address address, ICacheFlushMode flush_mode) { 204 Assembler::set_target_address_at(isolate_, pc_, host_, address, flush_mode); 205 } 206 207 void RelocInfo::unchecked_update_wasm_size(uint32_t size, 208 ICacheFlushMode flush_mode) { 209 Memory::uint32_at(Assembler::target_pointer_address_at(pc_)) = size; 210 } 211 212 Register GetAllocatableRegisterThatIsNotOneOf(Register reg1, Register reg2, 213 Register reg3, Register reg4) { 214 CPURegList regs(reg1, reg2, reg3, reg4); 215 const RegisterConfiguration* config = RegisterConfiguration::Crankshaft(); 216 for (int i = 0; i < config->num_allocatable_double_registers(); ++i) { 217 int code = config->GetAllocatableDoubleCode(i); 218 Register candidate = Register::from_code(code); 219 if (regs.IncludesAliasOf(candidate)) continue; 220 return candidate; 221 } 222 UNREACHABLE(); 223 return NoReg; 224 } 225 226 227 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2, 228 const CPURegister& reg3, const CPURegister& reg4, 229 const CPURegister& reg5, const CPURegister& reg6, 230 const CPURegister& reg7, const CPURegister& reg8) { 231 int number_of_valid_regs = 0; 232 int number_of_valid_fpregs = 0; 233 234 RegList unique_regs = 0; 235 RegList unique_fpregs = 0; 236 237 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8}; 238 239 for (unsigned i = 0; i < arraysize(regs); i++) { 240 if (regs[i].IsRegister()) { 241 number_of_valid_regs++; 242 unique_regs |= regs[i].Bit(); 243 } else if (regs[i].IsFPRegister()) { 244 number_of_valid_fpregs++; 245 unique_fpregs |= regs[i].Bit(); 246 } else { 247 DCHECK(!regs[i].IsValid()); 248 } 249 } 250 251 int number_of_unique_regs = 252 CountSetBits(unique_regs, sizeof(unique_regs) * kBitsPerByte); 253 int number_of_unique_fpregs = 254 CountSetBits(unique_fpregs, sizeof(unique_fpregs) * kBitsPerByte); 255 256 DCHECK(number_of_valid_regs >= number_of_unique_regs); 257 DCHECK(number_of_valid_fpregs >= number_of_unique_fpregs); 258 259 return (number_of_valid_regs != number_of_unique_regs) || 260 (number_of_valid_fpregs != number_of_unique_fpregs); 261 } 262 263 264 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2, 265 const CPURegister& reg3, const CPURegister& reg4, 266 const CPURegister& reg5, const CPURegister& reg6, 267 const CPURegister& reg7, const CPURegister& reg8) { 268 DCHECK(reg1.IsValid()); 269 bool match = true; 270 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1); 271 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1); 272 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1); 273 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1); 274 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1); 275 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1); 276 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1); 277 return match; 278 } 279 280 281 void Immediate::InitializeHandle(Handle<Object> handle) { 282 AllowDeferredHandleDereference using_raw_address; 283 284 // Verify all Objects referred by code are NOT in new space. 285 Object* obj = *handle; 286 if (obj->IsHeapObject()) { 287 value_ = reinterpret_cast<intptr_t>(handle.location()); 288 rmode_ = RelocInfo::EMBEDDED_OBJECT; 289 } else { 290 STATIC_ASSERT(sizeof(intptr_t) == sizeof(int64_t)); 291 value_ = reinterpret_cast<intptr_t>(obj); 292 rmode_ = RelocInfo::NONE64; 293 } 294 } 295 296 297 bool Operand::NeedsRelocation(const Assembler* assembler) const { 298 RelocInfo::Mode rmode = immediate_.rmode(); 299 300 if (rmode == RelocInfo::EXTERNAL_REFERENCE) { 301 return assembler->serializer_enabled(); 302 } 303 304 return !RelocInfo::IsNone(rmode); 305 } 306 307 308 // Constant Pool. 309 void ConstPool::RecordEntry(intptr_t data, 310 RelocInfo::Mode mode) { 311 DCHECK(mode != RelocInfo::COMMENT && mode != RelocInfo::CONST_POOL && 312 mode != RelocInfo::VENEER_POOL && 313 mode != RelocInfo::CODE_AGE_SEQUENCE && 314 mode != RelocInfo::DEOPT_SCRIPT_OFFSET && 315 mode != RelocInfo::DEOPT_INLINING_ID && 316 mode != RelocInfo::DEOPT_REASON && mode != RelocInfo::DEOPT_ID); 317 uint64_t raw_data = static_cast<uint64_t>(data); 318 int offset = assm_->pc_offset(); 319 if (IsEmpty()) { 320 first_use_ = offset; 321 } 322 323 std::pair<uint64_t, int> entry = std::make_pair(raw_data, offset); 324 if (CanBeShared(mode)) { 325 shared_entries_.insert(entry); 326 if (shared_entries_.count(entry.first) == 1) { 327 shared_entries_count++; 328 } 329 } else { 330 unique_entries_.push_back(entry); 331 } 332 333 if (EntryCount() > Assembler::kApproxMaxPoolEntryCount) { 334 // Request constant pool emission after the next instruction. 335 assm_->SetNextConstPoolCheckIn(1); 336 } 337 } 338 339 340 int ConstPool::DistanceToFirstUse() { 341 DCHECK(first_use_ >= 0); 342 return assm_->pc_offset() - first_use_; 343 } 344 345 346 int ConstPool::MaxPcOffset() { 347 // There are no pending entries in the pool so we can never get out of 348 // range. 349 if (IsEmpty()) return kMaxInt; 350 351 // Entries are not necessarily emitted in the order they are added so in the 352 // worst case the first constant pool use will be accessing the last entry. 353 return first_use_ + kMaxLoadLiteralRange - WorstCaseSize(); 354 } 355 356 357 int ConstPool::WorstCaseSize() { 358 if (IsEmpty()) return 0; 359 360 // Max size prologue: 361 // b over 362 // ldr xzr, #pool_size 363 // blr xzr 364 // nop 365 // All entries are 64-bit for now. 366 return 4 * kInstructionSize + EntryCount() * kPointerSize; 367 } 368 369 370 int ConstPool::SizeIfEmittedAtCurrentPc(bool require_jump) { 371 if (IsEmpty()) return 0; 372 373 // Prologue is: 374 // b over ;; if require_jump 375 // ldr xzr, #pool_size 376 // blr xzr 377 // nop ;; if not 64-bit aligned 378 int prologue_size = require_jump ? kInstructionSize : 0; 379 prologue_size += 2 * kInstructionSize; 380 prologue_size += IsAligned(assm_->pc_offset() + prologue_size, 8) ? 381 0 : kInstructionSize; 382 383 // All entries are 64-bit for now. 384 return prologue_size + EntryCount() * kPointerSize; 385 } 386 387 388 void ConstPool::Emit(bool require_jump) { 389 DCHECK(!assm_->is_const_pool_blocked()); 390 // Prevent recursive pool emission and protect from veneer pools. 391 Assembler::BlockPoolsScope block_pools(assm_); 392 393 int size = SizeIfEmittedAtCurrentPc(require_jump); 394 Label size_check; 395 assm_->bind(&size_check); 396 397 assm_->RecordConstPool(size); 398 // Emit the constant pool. It is preceded by an optional branch if 399 // require_jump and a header which will: 400 // 1) Encode the size of the constant pool, for use by the disassembler. 401 // 2) Terminate the program, to try to prevent execution from accidentally 402 // flowing into the constant pool. 403 // 3) align the pool entries to 64-bit. 404 // The header is therefore made of up to three arm64 instructions: 405 // ldr xzr, #<size of the constant pool in 32-bit words> 406 // blr xzr 407 // nop 408 // 409 // If executed, the header will likely segfault and lr will point to the 410 // instruction following the offending blr. 411 // TODO(all): Make the alignment part less fragile. Currently code is 412 // allocated as a byte array so there are no guarantees the alignment will 413 // be preserved on compaction. Currently it works as allocation seems to be 414 // 64-bit aligned. 415 416 // Emit branch if required 417 Label after_pool; 418 if (require_jump) { 419 assm_->b(&after_pool); 420 } 421 422 // Emit the header. 423 assm_->RecordComment("[ Constant Pool"); 424 EmitMarker(); 425 EmitGuard(); 426 assm_->Align(8); 427 428 // Emit constant pool entries. 429 // TODO(all): currently each relocated constant is 64 bits, consider adding 430 // support for 32-bit entries. 431 EmitEntries(); 432 assm_->RecordComment("]"); 433 434 if (after_pool.is_linked()) { 435 assm_->bind(&after_pool); 436 } 437 438 DCHECK(assm_->SizeOfCodeGeneratedSince(&size_check) == 439 static_cast<unsigned>(size)); 440 } 441 442 443 void ConstPool::Clear() { 444 shared_entries_.clear(); 445 shared_entries_count = 0; 446 unique_entries_.clear(); 447 first_use_ = -1; 448 } 449 450 451 bool ConstPool::CanBeShared(RelocInfo::Mode mode) { 452 // Constant pool currently does not support 32-bit entries. 453 DCHECK(mode != RelocInfo::NONE32); 454 455 return RelocInfo::IsNone(mode) || 456 (!assm_->serializer_enabled() && 457 (mode >= RelocInfo::FIRST_SHAREABLE_RELOC_MODE)); 458 } 459 460 461 void ConstPool::EmitMarker() { 462 // A constant pool size is expressed in number of 32-bits words. 463 // Currently all entries are 64-bit. 464 // + 1 is for the crash guard. 465 // + 0/1 for alignment. 466 int word_count = EntryCount() * 2 + 1 + 467 (IsAligned(assm_->pc_offset(), 8) ? 0 : 1); 468 assm_->Emit(LDR_x_lit | 469 Assembler::ImmLLiteral(word_count) | 470 Assembler::Rt(xzr)); 471 } 472 473 474 MemOperand::PairResult MemOperand::AreConsistentForPair( 475 const MemOperand& operandA, 476 const MemOperand& operandB, 477 int access_size_log2) { 478 DCHECK(access_size_log2 >= 0); 479 DCHECK(access_size_log2 <= 3); 480 // Step one: check that they share the same base, that the mode is Offset 481 // and that the offset is a multiple of access size. 482 if (!operandA.base().Is(operandB.base()) || 483 (operandA.addrmode() != Offset) || 484 (operandB.addrmode() != Offset) || 485 ((operandA.offset() & ((1 << access_size_log2) - 1)) != 0)) { 486 return kNotPair; 487 } 488 // Step two: check that the offsets are contiguous and that the range 489 // is OK for ldp/stp. 490 if ((operandB.offset() == operandA.offset() + (1 << access_size_log2)) && 491 is_int7(operandA.offset() >> access_size_log2)) { 492 return kPairAB; 493 } 494 if ((operandA.offset() == operandB.offset() + (1 << access_size_log2)) && 495 is_int7(operandB.offset() >> access_size_log2)) { 496 return kPairBA; 497 } 498 return kNotPair; 499 } 500 501 502 void ConstPool::EmitGuard() { 503 #ifdef DEBUG 504 Instruction* instr = reinterpret_cast<Instruction*>(assm_->pc()); 505 DCHECK(instr->preceding()->IsLdrLiteralX() && 506 instr->preceding()->Rt() == xzr.code()); 507 #endif 508 assm_->EmitPoolGuard(); 509 } 510 511 512 void ConstPool::EmitEntries() { 513 DCHECK(IsAligned(assm_->pc_offset(), 8)); 514 515 typedef std::multimap<uint64_t, int>::const_iterator SharedEntriesIterator; 516 SharedEntriesIterator value_it; 517 // Iterate through the keys (constant pool values). 518 for (value_it = shared_entries_.begin(); 519 value_it != shared_entries_.end(); 520 value_it = shared_entries_.upper_bound(value_it->first)) { 521 std::pair<SharedEntriesIterator, SharedEntriesIterator> range; 522 uint64_t data = value_it->first; 523 range = shared_entries_.equal_range(data); 524 SharedEntriesIterator offset_it; 525 // Iterate through the offsets of a given key. 526 for (offset_it = range.first; offset_it != range.second; offset_it++) { 527 Instruction* instr = assm_->InstructionAt(offset_it->second); 528 529 // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0. 530 DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0); 531 instr->SetImmPCOffsetTarget(assm_->isolate(), assm_->pc()); 532 } 533 assm_->dc64(data); 534 } 535 shared_entries_.clear(); 536 shared_entries_count = 0; 537 538 // Emit unique entries. 539 std::vector<std::pair<uint64_t, int> >::const_iterator unique_it; 540 for (unique_it = unique_entries_.begin(); 541 unique_it != unique_entries_.end(); 542 unique_it++) { 543 Instruction* instr = assm_->InstructionAt(unique_it->second); 544 545 // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0. 546 DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0); 547 instr->SetImmPCOffsetTarget(assm_->isolate(), assm_->pc()); 548 assm_->dc64(unique_it->first); 549 } 550 unique_entries_.clear(); 551 first_use_ = -1; 552 } 553 554 555 // Assembler 556 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size) 557 : AssemblerBase(isolate, buffer, buffer_size), 558 constpool_(this), 559 recorded_ast_id_(TypeFeedbackId::None()), 560 unresolved_branches_() { 561 const_pool_blocked_nesting_ = 0; 562 veneer_pool_blocked_nesting_ = 0; 563 Reset(); 564 } 565 566 567 Assembler::~Assembler() { 568 DCHECK(constpool_.IsEmpty()); 569 DCHECK(const_pool_blocked_nesting_ == 0); 570 DCHECK(veneer_pool_blocked_nesting_ == 0); 571 } 572 573 574 void Assembler::Reset() { 575 #ifdef DEBUG 576 DCHECK((pc_ >= buffer_) && (pc_ < buffer_ + buffer_size_)); 577 DCHECK(const_pool_blocked_nesting_ == 0); 578 DCHECK(veneer_pool_blocked_nesting_ == 0); 579 DCHECK(unresolved_branches_.empty()); 580 memset(buffer_, 0, pc_ - buffer_); 581 #endif 582 pc_ = buffer_; 583 reloc_info_writer.Reposition(reinterpret_cast<byte*>(buffer_ + buffer_size_), 584 reinterpret_cast<byte*>(pc_)); 585 constpool_.Clear(); 586 next_constant_pool_check_ = 0; 587 next_veneer_pool_check_ = kMaxInt; 588 no_const_pool_before_ = 0; 589 ClearRecordedAstId(); 590 } 591 592 593 void Assembler::GetCode(CodeDesc* desc) { 594 // Emit constant pool if necessary. 595 CheckConstPool(true, false); 596 DCHECK(constpool_.IsEmpty()); 597 598 // Set up code descriptor. 599 if (desc) { 600 desc->buffer = reinterpret_cast<byte*>(buffer_); 601 desc->buffer_size = buffer_size_; 602 desc->instr_size = pc_offset(); 603 desc->reloc_size = 604 static_cast<int>((reinterpret_cast<byte*>(buffer_) + buffer_size_) - 605 reloc_info_writer.pos()); 606 desc->origin = this; 607 desc->constant_pool_size = 0; 608 desc->unwinding_info_size = 0; 609 desc->unwinding_info = nullptr; 610 } 611 } 612 613 614 void Assembler::Align(int m) { 615 DCHECK(m >= 4 && base::bits::IsPowerOfTwo32(m)); 616 while ((pc_offset() & (m - 1)) != 0) { 617 nop(); 618 } 619 } 620 621 622 void Assembler::CheckLabelLinkChain(Label const * label) { 623 #ifdef DEBUG 624 if (label->is_linked()) { 625 static const int kMaxLinksToCheck = 64; // Avoid O(n2) behaviour. 626 int links_checked = 0; 627 int64_t linkoffset = label->pos(); 628 bool end_of_chain = false; 629 while (!end_of_chain) { 630 if (++links_checked > kMaxLinksToCheck) break; 631 Instruction * link = InstructionAt(linkoffset); 632 int64_t linkpcoffset = link->ImmPCOffset(); 633 int64_t prevlinkoffset = linkoffset + linkpcoffset; 634 635 end_of_chain = (linkoffset == prevlinkoffset); 636 linkoffset = linkoffset + linkpcoffset; 637 } 638 } 639 #endif 640 } 641 642 643 void Assembler::RemoveBranchFromLabelLinkChain(Instruction* branch, 644 Label* label, 645 Instruction* label_veneer) { 646 DCHECK(label->is_linked()); 647 648 CheckLabelLinkChain(label); 649 650 Instruction* link = InstructionAt(label->pos()); 651 Instruction* prev_link = link; 652 Instruction* next_link; 653 bool end_of_chain = false; 654 655 while (link != branch && !end_of_chain) { 656 next_link = link->ImmPCOffsetTarget(); 657 end_of_chain = (link == next_link); 658 prev_link = link; 659 link = next_link; 660 } 661 662 DCHECK(branch == link); 663 next_link = branch->ImmPCOffsetTarget(); 664 665 if (branch == prev_link) { 666 // The branch is the first instruction in the chain. 667 if (branch == next_link) { 668 // It is also the last instruction in the chain, so it is the only branch 669 // currently referring to this label. 670 label->Unuse(); 671 } else { 672 label->link_to( 673 static_cast<int>(reinterpret_cast<byte*>(next_link) - buffer_)); 674 } 675 676 } else if (branch == next_link) { 677 // The branch is the last (but not also the first) instruction in the chain. 678 prev_link->SetImmPCOffsetTarget(isolate(), prev_link); 679 680 } else { 681 // The branch is in the middle of the chain. 682 if (prev_link->IsTargetInImmPCOffsetRange(next_link)) { 683 prev_link->SetImmPCOffsetTarget(isolate(), next_link); 684 } else if (label_veneer != NULL) { 685 // Use the veneer for all previous links in the chain. 686 prev_link->SetImmPCOffsetTarget(isolate(), prev_link); 687 688 end_of_chain = false; 689 link = next_link; 690 while (!end_of_chain) { 691 next_link = link->ImmPCOffsetTarget(); 692 end_of_chain = (link == next_link); 693 link->SetImmPCOffsetTarget(isolate(), label_veneer); 694 link = next_link; 695 } 696 } else { 697 // The assert below will fire. 698 // Some other work could be attempted to fix up the chain, but it would be 699 // rather complicated. If we crash here, we may want to consider using an 700 // other mechanism than a chain of branches. 701 // 702 // Note that this situation currently should not happen, as we always call 703 // this function with a veneer to the target label. 704 // However this could happen with a MacroAssembler in the following state: 705 // [previous code] 706 // B(label); 707 // [20KB code] 708 // Tbz(label); // First tbz. Pointing to unconditional branch. 709 // [20KB code] 710 // Tbz(label); // Second tbz. Pointing to the first tbz. 711 // [more code] 712 // and this function is called to remove the first tbz from the label link 713 // chain. Since tbz has a range of +-32KB, the second tbz cannot point to 714 // the unconditional branch. 715 CHECK(prev_link->IsTargetInImmPCOffsetRange(next_link)); 716 UNREACHABLE(); 717 } 718 } 719 720 CheckLabelLinkChain(label); 721 } 722 723 724 void Assembler::bind(Label* label) { 725 // Bind label to the address at pc_. All instructions (most likely branches) 726 // that are linked to this label will be updated to point to the newly-bound 727 // label. 728 729 DCHECK(!label->is_near_linked()); 730 DCHECK(!label->is_bound()); 731 732 DeleteUnresolvedBranchInfoForLabel(label); 733 734 // If the label is linked, the link chain looks something like this: 735 // 736 // |--I----I-------I-------L 737 // |---------------------->| pc_offset 738 // |-------------->| linkoffset = label->pos() 739 // |<------| link->ImmPCOffset() 740 // |------>| prevlinkoffset = linkoffset + link->ImmPCOffset() 741 // 742 // On each iteration, the last link is updated and then removed from the 743 // chain until only one remains. At that point, the label is bound. 744 // 745 // If the label is not linked, no preparation is required before binding. 746 while (label->is_linked()) { 747 int linkoffset = label->pos(); 748 Instruction* link = InstructionAt(linkoffset); 749 int prevlinkoffset = linkoffset + static_cast<int>(link->ImmPCOffset()); 750 751 CheckLabelLinkChain(label); 752 753 DCHECK(linkoffset >= 0); 754 DCHECK(linkoffset < pc_offset()); 755 DCHECK((linkoffset > prevlinkoffset) || 756 (linkoffset - prevlinkoffset == kStartOfLabelLinkChain)); 757 DCHECK(prevlinkoffset >= 0); 758 759 // Update the link to point to the label. 760 if (link->IsUnresolvedInternalReference()) { 761 // Internal references do not get patched to an instruction but directly 762 // to an address. 763 internal_reference_positions_.push_back(linkoffset); 764 PatchingAssembler patcher(isolate(), link, 2); 765 patcher.dc64(reinterpret_cast<uintptr_t>(pc_)); 766 } else { 767 link->SetImmPCOffsetTarget(isolate(), 768 reinterpret_cast<Instruction*>(pc_)); 769 } 770 771 // Link the label to the previous link in the chain. 772 if (linkoffset - prevlinkoffset == kStartOfLabelLinkChain) { 773 // We hit kStartOfLabelLinkChain, so the chain is fully processed. 774 label->Unuse(); 775 } else { 776 // Update the label for the next iteration. 777 label->link_to(prevlinkoffset); 778 } 779 } 780 label->bind_to(pc_offset()); 781 782 DCHECK(label->is_bound()); 783 DCHECK(!label->is_linked()); 784 } 785 786 787 int Assembler::LinkAndGetByteOffsetTo(Label* label) { 788 DCHECK(sizeof(*pc_) == 1); 789 CheckLabelLinkChain(label); 790 791 int offset; 792 if (label->is_bound()) { 793 // The label is bound, so it does not need to be updated. Referring 794 // instructions must link directly to the label as they will not be 795 // updated. 796 // 797 // In this case, label->pos() returns the offset of the label from the 798 // start of the buffer. 799 // 800 // Note that offset can be zero for self-referential instructions. (This 801 // could be useful for ADR, for example.) 802 offset = label->pos() - pc_offset(); 803 DCHECK(offset <= 0); 804 } else { 805 if (label->is_linked()) { 806 // The label is linked, so the referring instruction should be added onto 807 // the end of the label's link chain. 808 // 809 // In this case, label->pos() returns the offset of the last linked 810 // instruction from the start of the buffer. 811 offset = label->pos() - pc_offset(); 812 DCHECK(offset != kStartOfLabelLinkChain); 813 // Note that the offset here needs to be PC-relative only so that the 814 // first instruction in a buffer can link to an unbound label. Otherwise, 815 // the offset would be 0 for this case, and 0 is reserved for 816 // kStartOfLabelLinkChain. 817 } else { 818 // The label is unused, so it now becomes linked and the referring 819 // instruction is at the start of the new link chain. 820 offset = kStartOfLabelLinkChain; 821 } 822 // The instruction at pc is now the last link in the label's chain. 823 label->link_to(pc_offset()); 824 } 825 826 return offset; 827 } 828 829 830 void Assembler::DeleteUnresolvedBranchInfoForLabelTraverse(Label* label) { 831 DCHECK(label->is_linked()); 832 CheckLabelLinkChain(label); 833 834 int link_offset = label->pos(); 835 int link_pcoffset; 836 bool end_of_chain = false; 837 838 while (!end_of_chain) { 839 Instruction * link = InstructionAt(link_offset); 840 link_pcoffset = static_cast<int>(link->ImmPCOffset()); 841 842 // ADR instructions are not handled by veneers. 843 if (link->IsImmBranch()) { 844 int max_reachable_pc = 845 static_cast<int>(InstructionOffset(link) + 846 Instruction::ImmBranchRange(link->BranchType())); 847 typedef std::multimap<int, FarBranchInfo>::iterator unresolved_info_it; 848 std::pair<unresolved_info_it, unresolved_info_it> range; 849 range = unresolved_branches_.equal_range(max_reachable_pc); 850 unresolved_info_it it; 851 for (it = range.first; it != range.second; ++it) { 852 if (it->second.pc_offset_ == link_offset) { 853 unresolved_branches_.erase(it); 854 break; 855 } 856 } 857 } 858 859 end_of_chain = (link_pcoffset == 0); 860 link_offset = link_offset + link_pcoffset; 861 } 862 } 863 864 865 void Assembler::DeleteUnresolvedBranchInfoForLabel(Label* label) { 866 if (unresolved_branches_.empty()) { 867 DCHECK(next_veneer_pool_check_ == kMaxInt); 868 return; 869 } 870 871 if (label->is_linked()) { 872 // Branches to this label will be resolved when the label is bound, normally 873 // just after all the associated info has been deleted. 874 DeleteUnresolvedBranchInfoForLabelTraverse(label); 875 } 876 if (unresolved_branches_.empty()) { 877 next_veneer_pool_check_ = kMaxInt; 878 } else { 879 next_veneer_pool_check_ = 880 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 881 } 882 } 883 884 885 void Assembler::StartBlockConstPool() { 886 if (const_pool_blocked_nesting_++ == 0) { 887 // Prevent constant pool checks happening by setting the next check to 888 // the biggest possible offset. 889 next_constant_pool_check_ = kMaxInt; 890 } 891 } 892 893 894 void Assembler::EndBlockConstPool() { 895 if (--const_pool_blocked_nesting_ == 0) { 896 // Check the constant pool hasn't been blocked for too long. 897 DCHECK(pc_offset() < constpool_.MaxPcOffset()); 898 // Two cases: 899 // * no_const_pool_before_ >= next_constant_pool_check_ and the emission is 900 // still blocked 901 // * no_const_pool_before_ < next_constant_pool_check_ and the next emit 902 // will trigger a check. 903 next_constant_pool_check_ = no_const_pool_before_; 904 } 905 } 906 907 908 bool Assembler::is_const_pool_blocked() const { 909 return (const_pool_blocked_nesting_ > 0) || 910 (pc_offset() < no_const_pool_before_); 911 } 912 913 914 bool Assembler::IsConstantPoolAt(Instruction* instr) { 915 // The constant pool marker is made of two instructions. These instructions 916 // will never be emitted by the JIT, so checking for the first one is enough: 917 // 0: ldr xzr, #<size of pool> 918 bool result = instr->IsLdrLiteralX() && (instr->Rt() == kZeroRegCode); 919 920 // It is still worth asserting the marker is complete. 921 // 4: blr xzr 922 DCHECK(!result || (instr->following()->IsBranchAndLinkToRegister() && 923 instr->following()->Rn() == kZeroRegCode)); 924 925 return result; 926 } 927 928 929 int Assembler::ConstantPoolSizeAt(Instruction* instr) { 930 #ifdef USE_SIMULATOR 931 // Assembler::debug() embeds constants directly into the instruction stream. 932 // Although this is not a genuine constant pool, treat it like one to avoid 933 // disassembling the constants. 934 if ((instr->Mask(ExceptionMask) == HLT) && 935 (instr->ImmException() == kImmExceptionIsDebug)) { 936 const char* message = 937 reinterpret_cast<const char*>( 938 instr->InstructionAtOffset(kDebugMessageOffset)); 939 int size = static_cast<int>(kDebugMessageOffset + strlen(message) + 1); 940 return RoundUp(size, kInstructionSize) / kInstructionSize; 941 } 942 // Same for printf support, see MacroAssembler::CallPrintf(). 943 if ((instr->Mask(ExceptionMask) == HLT) && 944 (instr->ImmException() == kImmExceptionIsPrintf)) { 945 return kPrintfLength / kInstructionSize; 946 } 947 #endif 948 if (IsConstantPoolAt(instr)) { 949 return instr->ImmLLiteral(); 950 } else { 951 return -1; 952 } 953 } 954 955 956 void Assembler::EmitPoolGuard() { 957 // We must generate only one instruction as this is used in scopes that 958 // control the size of the code generated. 959 Emit(BLR | Rn(xzr)); 960 } 961 962 963 void Assembler::StartBlockVeneerPool() { 964 ++veneer_pool_blocked_nesting_; 965 } 966 967 968 void Assembler::EndBlockVeneerPool() { 969 if (--veneer_pool_blocked_nesting_ == 0) { 970 // Check the veneer pool hasn't been blocked for too long. 971 DCHECK(unresolved_branches_.empty() || 972 (pc_offset() < unresolved_branches_first_limit())); 973 } 974 } 975 976 977 void Assembler::br(const Register& xn) { 978 DCHECK(xn.Is64Bits()); 979 Emit(BR | Rn(xn)); 980 } 981 982 983 void Assembler::blr(const Register& xn) { 984 DCHECK(xn.Is64Bits()); 985 // The pattern 'blr xzr' is used as a guard to detect when execution falls 986 // through the constant pool. It should not be emitted. 987 DCHECK(!xn.Is(xzr)); 988 Emit(BLR | Rn(xn)); 989 } 990 991 992 void Assembler::ret(const Register& xn) { 993 DCHECK(xn.Is64Bits()); 994 Emit(RET | Rn(xn)); 995 } 996 997 998 void Assembler::b(int imm26) { 999 Emit(B | ImmUncondBranch(imm26)); 1000 } 1001 1002 1003 void Assembler::b(Label* label) { 1004 b(LinkAndGetInstructionOffsetTo(label)); 1005 } 1006 1007 1008 void Assembler::b(int imm19, Condition cond) { 1009 Emit(B_cond | ImmCondBranch(imm19) | cond); 1010 } 1011 1012 1013 void Assembler::b(Label* label, Condition cond) { 1014 b(LinkAndGetInstructionOffsetTo(label), cond); 1015 } 1016 1017 1018 void Assembler::bl(int imm26) { 1019 Emit(BL | ImmUncondBranch(imm26)); 1020 } 1021 1022 1023 void Assembler::bl(Label* label) { 1024 bl(LinkAndGetInstructionOffsetTo(label)); 1025 } 1026 1027 1028 void Assembler::cbz(const Register& rt, 1029 int imm19) { 1030 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt)); 1031 } 1032 1033 1034 void Assembler::cbz(const Register& rt, 1035 Label* label) { 1036 cbz(rt, LinkAndGetInstructionOffsetTo(label)); 1037 } 1038 1039 1040 void Assembler::cbnz(const Register& rt, 1041 int imm19) { 1042 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt)); 1043 } 1044 1045 1046 void Assembler::cbnz(const Register& rt, 1047 Label* label) { 1048 cbnz(rt, LinkAndGetInstructionOffsetTo(label)); 1049 } 1050 1051 1052 void Assembler::tbz(const Register& rt, 1053 unsigned bit_pos, 1054 int imm14) { 1055 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits))); 1056 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 1057 } 1058 1059 1060 void Assembler::tbz(const Register& rt, 1061 unsigned bit_pos, 1062 Label* label) { 1063 tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 1064 } 1065 1066 1067 void Assembler::tbnz(const Register& rt, 1068 unsigned bit_pos, 1069 int imm14) { 1070 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits))); 1071 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 1072 } 1073 1074 1075 void Assembler::tbnz(const Register& rt, 1076 unsigned bit_pos, 1077 Label* label) { 1078 tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 1079 } 1080 1081 1082 void Assembler::adr(const Register& rd, int imm21) { 1083 DCHECK(rd.Is64Bits()); 1084 Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd)); 1085 } 1086 1087 1088 void Assembler::adr(const Register& rd, Label* label) { 1089 adr(rd, LinkAndGetByteOffsetTo(label)); 1090 } 1091 1092 1093 void Assembler::add(const Register& rd, 1094 const Register& rn, 1095 const Operand& operand) { 1096 AddSub(rd, rn, operand, LeaveFlags, ADD); 1097 } 1098 1099 1100 void Assembler::adds(const Register& rd, 1101 const Register& rn, 1102 const Operand& operand) { 1103 AddSub(rd, rn, operand, SetFlags, ADD); 1104 } 1105 1106 1107 void Assembler::cmn(const Register& rn, 1108 const Operand& operand) { 1109 Register zr = AppropriateZeroRegFor(rn); 1110 adds(zr, rn, operand); 1111 } 1112 1113 1114 void Assembler::sub(const Register& rd, 1115 const Register& rn, 1116 const Operand& operand) { 1117 AddSub(rd, rn, operand, LeaveFlags, SUB); 1118 } 1119 1120 1121 void Assembler::subs(const Register& rd, 1122 const Register& rn, 1123 const Operand& operand) { 1124 AddSub(rd, rn, operand, SetFlags, SUB); 1125 } 1126 1127 1128 void Assembler::cmp(const Register& rn, const Operand& operand) { 1129 Register zr = AppropriateZeroRegFor(rn); 1130 subs(zr, rn, operand); 1131 } 1132 1133 1134 void Assembler::neg(const Register& rd, const Operand& operand) { 1135 Register zr = AppropriateZeroRegFor(rd); 1136 sub(rd, zr, operand); 1137 } 1138 1139 1140 void Assembler::negs(const Register& rd, const Operand& operand) { 1141 Register zr = AppropriateZeroRegFor(rd); 1142 subs(rd, zr, operand); 1143 } 1144 1145 1146 void Assembler::adc(const Register& rd, 1147 const Register& rn, 1148 const Operand& operand) { 1149 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC); 1150 } 1151 1152 1153 void Assembler::adcs(const Register& rd, 1154 const Register& rn, 1155 const Operand& operand) { 1156 AddSubWithCarry(rd, rn, operand, SetFlags, ADC); 1157 } 1158 1159 1160 void Assembler::sbc(const Register& rd, 1161 const Register& rn, 1162 const Operand& operand) { 1163 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC); 1164 } 1165 1166 1167 void Assembler::sbcs(const Register& rd, 1168 const Register& rn, 1169 const Operand& operand) { 1170 AddSubWithCarry(rd, rn, operand, SetFlags, SBC); 1171 } 1172 1173 1174 void Assembler::ngc(const Register& rd, const Operand& operand) { 1175 Register zr = AppropriateZeroRegFor(rd); 1176 sbc(rd, zr, operand); 1177 } 1178 1179 1180 void Assembler::ngcs(const Register& rd, const Operand& operand) { 1181 Register zr = AppropriateZeroRegFor(rd); 1182 sbcs(rd, zr, operand); 1183 } 1184 1185 1186 // Logical instructions. 1187 void Assembler::and_(const Register& rd, 1188 const Register& rn, 1189 const Operand& operand) { 1190 Logical(rd, rn, operand, AND); 1191 } 1192 1193 1194 void Assembler::ands(const Register& rd, 1195 const Register& rn, 1196 const Operand& operand) { 1197 Logical(rd, rn, operand, ANDS); 1198 } 1199 1200 1201 void Assembler::tst(const Register& rn, 1202 const Operand& operand) { 1203 ands(AppropriateZeroRegFor(rn), rn, operand); 1204 } 1205 1206 1207 void Assembler::bic(const Register& rd, 1208 const Register& rn, 1209 const Operand& operand) { 1210 Logical(rd, rn, operand, BIC); 1211 } 1212 1213 1214 void Assembler::bics(const Register& rd, 1215 const Register& rn, 1216 const Operand& operand) { 1217 Logical(rd, rn, operand, BICS); 1218 } 1219 1220 1221 void Assembler::orr(const Register& rd, 1222 const Register& rn, 1223 const Operand& operand) { 1224 Logical(rd, rn, operand, ORR); 1225 } 1226 1227 1228 void Assembler::orn(const Register& rd, 1229 const Register& rn, 1230 const Operand& operand) { 1231 Logical(rd, rn, operand, ORN); 1232 } 1233 1234 1235 void Assembler::eor(const Register& rd, 1236 const Register& rn, 1237 const Operand& operand) { 1238 Logical(rd, rn, operand, EOR); 1239 } 1240 1241 1242 void Assembler::eon(const Register& rd, 1243 const Register& rn, 1244 const Operand& operand) { 1245 Logical(rd, rn, operand, EON); 1246 } 1247 1248 1249 void Assembler::lslv(const Register& rd, 1250 const Register& rn, 1251 const Register& rm) { 1252 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1253 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1254 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd)); 1255 } 1256 1257 1258 void Assembler::lsrv(const Register& rd, 1259 const Register& rn, 1260 const Register& rm) { 1261 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1262 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1263 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd)); 1264 } 1265 1266 1267 void Assembler::asrv(const Register& rd, 1268 const Register& rn, 1269 const Register& rm) { 1270 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1271 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1272 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd)); 1273 } 1274 1275 1276 void Assembler::rorv(const Register& rd, 1277 const Register& rn, 1278 const Register& rm) { 1279 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1280 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1281 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd)); 1282 } 1283 1284 1285 // Bitfield operations. 1286 void Assembler::bfm(const Register& rd, const Register& rn, int immr, 1287 int imms) { 1288 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1289 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1290 Emit(SF(rd) | BFM | N | 1291 ImmR(immr, rd.SizeInBits()) | 1292 ImmS(imms, rn.SizeInBits()) | 1293 Rn(rn) | Rd(rd)); 1294 } 1295 1296 1297 void Assembler::sbfm(const Register& rd, const Register& rn, int immr, 1298 int imms) { 1299 DCHECK(rd.Is64Bits() || rn.Is32Bits()); 1300 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1301 Emit(SF(rd) | SBFM | N | 1302 ImmR(immr, rd.SizeInBits()) | 1303 ImmS(imms, rn.SizeInBits()) | 1304 Rn(rn) | Rd(rd)); 1305 } 1306 1307 1308 void Assembler::ubfm(const Register& rd, const Register& rn, int immr, 1309 int imms) { 1310 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1311 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1312 Emit(SF(rd) | UBFM | N | 1313 ImmR(immr, rd.SizeInBits()) | 1314 ImmS(imms, rn.SizeInBits()) | 1315 Rn(rn) | Rd(rd)); 1316 } 1317 1318 1319 void Assembler::extr(const Register& rd, const Register& rn, const Register& rm, 1320 int lsb) { 1321 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1322 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1323 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1324 Emit(SF(rd) | EXTR | N | Rm(rm) | 1325 ImmS(lsb, rn.SizeInBits()) | Rn(rn) | Rd(rd)); 1326 } 1327 1328 1329 void Assembler::csel(const Register& rd, 1330 const Register& rn, 1331 const Register& rm, 1332 Condition cond) { 1333 ConditionalSelect(rd, rn, rm, cond, CSEL); 1334 } 1335 1336 1337 void Assembler::csinc(const Register& rd, 1338 const Register& rn, 1339 const Register& rm, 1340 Condition cond) { 1341 ConditionalSelect(rd, rn, rm, cond, CSINC); 1342 } 1343 1344 1345 void Assembler::csinv(const Register& rd, 1346 const Register& rn, 1347 const Register& rm, 1348 Condition cond) { 1349 ConditionalSelect(rd, rn, rm, cond, CSINV); 1350 } 1351 1352 1353 void Assembler::csneg(const Register& rd, 1354 const Register& rn, 1355 const Register& rm, 1356 Condition cond) { 1357 ConditionalSelect(rd, rn, rm, cond, CSNEG); 1358 } 1359 1360 1361 void Assembler::cset(const Register &rd, Condition cond) { 1362 DCHECK((cond != al) && (cond != nv)); 1363 Register zr = AppropriateZeroRegFor(rd); 1364 csinc(rd, zr, zr, NegateCondition(cond)); 1365 } 1366 1367 1368 void Assembler::csetm(const Register &rd, Condition cond) { 1369 DCHECK((cond != al) && (cond != nv)); 1370 Register zr = AppropriateZeroRegFor(rd); 1371 csinv(rd, zr, zr, NegateCondition(cond)); 1372 } 1373 1374 1375 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) { 1376 DCHECK((cond != al) && (cond != nv)); 1377 csinc(rd, rn, rn, NegateCondition(cond)); 1378 } 1379 1380 1381 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) { 1382 DCHECK((cond != al) && (cond != nv)); 1383 csinv(rd, rn, rn, NegateCondition(cond)); 1384 } 1385 1386 1387 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) { 1388 DCHECK((cond != al) && (cond != nv)); 1389 csneg(rd, rn, rn, NegateCondition(cond)); 1390 } 1391 1392 1393 void Assembler::ConditionalSelect(const Register& rd, 1394 const Register& rn, 1395 const Register& rm, 1396 Condition cond, 1397 ConditionalSelectOp op) { 1398 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1399 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1400 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd)); 1401 } 1402 1403 1404 void Assembler::ccmn(const Register& rn, 1405 const Operand& operand, 1406 StatusFlags nzcv, 1407 Condition cond) { 1408 ConditionalCompare(rn, operand, nzcv, cond, CCMN); 1409 } 1410 1411 1412 void Assembler::ccmp(const Register& rn, 1413 const Operand& operand, 1414 StatusFlags nzcv, 1415 Condition cond) { 1416 ConditionalCompare(rn, operand, nzcv, cond, CCMP); 1417 } 1418 1419 1420 void Assembler::DataProcessing3Source(const Register& rd, 1421 const Register& rn, 1422 const Register& rm, 1423 const Register& ra, 1424 DataProcessing3SourceOp op) { 1425 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd)); 1426 } 1427 1428 1429 void Assembler::mul(const Register& rd, 1430 const Register& rn, 1431 const Register& rm) { 1432 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1433 Register zr = AppropriateZeroRegFor(rn); 1434 DataProcessing3Source(rd, rn, rm, zr, MADD); 1435 } 1436 1437 1438 void Assembler::madd(const Register& rd, 1439 const Register& rn, 1440 const Register& rm, 1441 const Register& ra) { 1442 DCHECK(AreSameSizeAndType(rd, rn, rm, ra)); 1443 DataProcessing3Source(rd, rn, rm, ra, MADD); 1444 } 1445 1446 1447 void Assembler::mneg(const Register& rd, 1448 const Register& rn, 1449 const Register& rm) { 1450 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1451 Register zr = AppropriateZeroRegFor(rn); 1452 DataProcessing3Source(rd, rn, rm, zr, MSUB); 1453 } 1454 1455 1456 void Assembler::msub(const Register& rd, 1457 const Register& rn, 1458 const Register& rm, 1459 const Register& ra) { 1460 DCHECK(AreSameSizeAndType(rd, rn, rm, ra)); 1461 DataProcessing3Source(rd, rn, rm, ra, MSUB); 1462 } 1463 1464 1465 void Assembler::smaddl(const Register& rd, 1466 const Register& rn, 1467 const Register& rm, 1468 const Register& ra) { 1469 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1470 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1471 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x); 1472 } 1473 1474 1475 void Assembler::smsubl(const Register& rd, 1476 const Register& rn, 1477 const Register& rm, 1478 const Register& ra) { 1479 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1480 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1481 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x); 1482 } 1483 1484 1485 void Assembler::umaddl(const Register& rd, 1486 const Register& rn, 1487 const Register& rm, 1488 const Register& ra) { 1489 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1490 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1491 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x); 1492 } 1493 1494 1495 void Assembler::umsubl(const Register& rd, 1496 const Register& rn, 1497 const Register& rm, 1498 const Register& ra) { 1499 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1500 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1501 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x); 1502 } 1503 1504 1505 void Assembler::smull(const Register& rd, 1506 const Register& rn, 1507 const Register& rm) { 1508 DCHECK(rd.Is64Bits()); 1509 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1510 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x); 1511 } 1512 1513 1514 void Assembler::smulh(const Register& rd, 1515 const Register& rn, 1516 const Register& rm) { 1517 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1518 DataProcessing3Source(rd, rn, rm, xzr, SMULH_x); 1519 } 1520 1521 1522 void Assembler::sdiv(const Register& rd, 1523 const Register& rn, 1524 const Register& rm) { 1525 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1526 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1527 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1528 } 1529 1530 1531 void Assembler::udiv(const Register& rd, 1532 const Register& rn, 1533 const Register& rm) { 1534 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1535 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1536 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1537 } 1538 1539 1540 void Assembler::rbit(const Register& rd, 1541 const Register& rn) { 1542 DataProcessing1Source(rd, rn, RBIT); 1543 } 1544 1545 1546 void Assembler::rev16(const Register& rd, 1547 const Register& rn) { 1548 DataProcessing1Source(rd, rn, REV16); 1549 } 1550 1551 1552 void Assembler::rev32(const Register& rd, 1553 const Register& rn) { 1554 DCHECK(rd.Is64Bits()); 1555 DataProcessing1Source(rd, rn, REV); 1556 } 1557 1558 1559 void Assembler::rev(const Register& rd, 1560 const Register& rn) { 1561 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w); 1562 } 1563 1564 1565 void Assembler::clz(const Register& rd, 1566 const Register& rn) { 1567 DataProcessing1Source(rd, rn, CLZ); 1568 } 1569 1570 1571 void Assembler::cls(const Register& rd, 1572 const Register& rn) { 1573 DataProcessing1Source(rd, rn, CLS); 1574 } 1575 1576 1577 void Assembler::ldp(const CPURegister& rt, 1578 const CPURegister& rt2, 1579 const MemOperand& src) { 1580 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2)); 1581 } 1582 1583 1584 void Assembler::stp(const CPURegister& rt, 1585 const CPURegister& rt2, 1586 const MemOperand& dst) { 1587 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2)); 1588 } 1589 1590 1591 void Assembler::ldpsw(const Register& rt, 1592 const Register& rt2, 1593 const MemOperand& src) { 1594 DCHECK(rt.Is64Bits()); 1595 LoadStorePair(rt, rt2, src, LDPSW_x); 1596 } 1597 1598 1599 void Assembler::LoadStorePair(const CPURegister& rt, 1600 const CPURegister& rt2, 1601 const MemOperand& addr, 1602 LoadStorePairOp op) { 1603 // 'rt' and 'rt2' can only be aliased for stores. 1604 DCHECK(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2)); 1605 DCHECK(AreSameSizeAndType(rt, rt2)); 1606 DCHECK(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op))); 1607 int offset = static_cast<int>(addr.offset()); 1608 1609 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | 1610 ImmLSPair(offset, CalcLSPairDataSize(op)); 1611 1612 Instr addrmodeop; 1613 if (addr.IsImmediateOffset()) { 1614 addrmodeop = LoadStorePairOffsetFixed; 1615 } else { 1616 // Pre-index and post-index modes. 1617 DCHECK(!rt.Is(addr.base())); 1618 DCHECK(!rt2.Is(addr.base())); 1619 DCHECK(addr.offset() != 0); 1620 if (addr.IsPreIndex()) { 1621 addrmodeop = LoadStorePairPreIndexFixed; 1622 } else { 1623 DCHECK(addr.IsPostIndex()); 1624 addrmodeop = LoadStorePairPostIndexFixed; 1625 } 1626 } 1627 Emit(addrmodeop | memop); 1628 } 1629 1630 1631 // Memory instructions. 1632 void Assembler::ldrb(const Register& rt, const MemOperand& src) { 1633 LoadStore(rt, src, LDRB_w); 1634 } 1635 1636 1637 void Assembler::strb(const Register& rt, const MemOperand& dst) { 1638 LoadStore(rt, dst, STRB_w); 1639 } 1640 1641 1642 void Assembler::ldrsb(const Register& rt, const MemOperand& src) { 1643 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w); 1644 } 1645 1646 1647 void Assembler::ldrh(const Register& rt, const MemOperand& src) { 1648 LoadStore(rt, src, LDRH_w); 1649 } 1650 1651 1652 void Assembler::strh(const Register& rt, const MemOperand& dst) { 1653 LoadStore(rt, dst, STRH_w); 1654 } 1655 1656 1657 void Assembler::ldrsh(const Register& rt, const MemOperand& src) { 1658 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w); 1659 } 1660 1661 1662 void Assembler::ldr(const CPURegister& rt, const MemOperand& src) { 1663 LoadStore(rt, src, LoadOpFor(rt)); 1664 } 1665 1666 1667 void Assembler::str(const CPURegister& rt, const MemOperand& src) { 1668 LoadStore(rt, src, StoreOpFor(rt)); 1669 } 1670 1671 1672 void Assembler::ldrsw(const Register& rt, const MemOperand& src) { 1673 DCHECK(rt.Is64Bits()); 1674 LoadStore(rt, src, LDRSW_x); 1675 } 1676 1677 1678 void Assembler::ldr_pcrel(const CPURegister& rt, int imm19) { 1679 // The pattern 'ldr xzr, #offset' is used to indicate the beginning of a 1680 // constant pool. It should not be emitted. 1681 DCHECK(!rt.IsZero()); 1682 Emit(LoadLiteralOpFor(rt) | ImmLLiteral(imm19) | Rt(rt)); 1683 } 1684 1685 1686 void Assembler::ldr(const CPURegister& rt, const Immediate& imm) { 1687 // Currently we only support 64-bit literals. 1688 DCHECK(rt.Is64Bits()); 1689 1690 RecordRelocInfo(imm.rmode(), imm.value()); 1691 BlockConstPoolFor(1); 1692 // The load will be patched when the constpool is emitted, patching code 1693 // expect a load literal with offset 0. 1694 ldr_pcrel(rt, 0); 1695 } 1696 1697 void Assembler::ldar(const Register& rt, const Register& rn) { 1698 DCHECK(rn.Is64Bits()); 1699 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x; 1700 Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1701 } 1702 1703 void Assembler::ldaxr(const Register& rt, const Register& rn) { 1704 DCHECK(rn.Is64Bits()); 1705 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x; 1706 Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1707 } 1708 1709 void Assembler::stlr(const Register& rt, const Register& rn) { 1710 DCHECK(rn.Is64Bits()); 1711 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x; 1712 Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1713 } 1714 1715 void Assembler::stlxr(const Register& rs, const Register& rt, 1716 const Register& rn) { 1717 DCHECK(rs.Is32Bits()); 1718 DCHECK(rn.Is64Bits()); 1719 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x; 1720 Emit(op | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt)); 1721 } 1722 1723 void Assembler::ldarb(const Register& rt, const Register& rn) { 1724 DCHECK(rt.Is32Bits()); 1725 DCHECK(rn.Is64Bits()); 1726 Emit(LDAR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1727 } 1728 1729 void Assembler::ldaxrb(const Register& rt, const Register& rn) { 1730 DCHECK(rt.Is32Bits()); 1731 DCHECK(rn.Is64Bits()); 1732 Emit(LDAXR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1733 } 1734 1735 void Assembler::stlrb(const Register& rt, const Register& rn) { 1736 DCHECK(rt.Is32Bits()); 1737 DCHECK(rn.Is64Bits()); 1738 Emit(STLR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1739 } 1740 1741 void Assembler::stlxrb(const Register& rs, const Register& rt, 1742 const Register& rn) { 1743 DCHECK(rs.Is32Bits()); 1744 DCHECK(rt.Is32Bits()); 1745 DCHECK(rn.Is64Bits()); 1746 Emit(STLXR_b | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt)); 1747 } 1748 1749 void Assembler::ldarh(const Register& rt, const Register& rn) { 1750 DCHECK(rt.Is32Bits()); 1751 DCHECK(rn.Is64Bits()); 1752 Emit(LDAR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1753 } 1754 1755 void Assembler::ldaxrh(const Register& rt, const Register& rn) { 1756 DCHECK(rt.Is32Bits()); 1757 DCHECK(rn.Is64Bits()); 1758 Emit(LDAXR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1759 } 1760 1761 void Assembler::stlrh(const Register& rt, const Register& rn) { 1762 DCHECK(rt.Is32Bits()); 1763 DCHECK(rn.Is64Bits()); 1764 Emit(STLR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt)); 1765 } 1766 1767 void Assembler::stlxrh(const Register& rs, const Register& rt, 1768 const Register& rn) { 1769 DCHECK(rs.Is32Bits()); 1770 DCHECK(rt.Is32Bits()); 1771 DCHECK(rn.Is64Bits()); 1772 Emit(STLXR_h | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt)); 1773 } 1774 1775 void Assembler::mov(const Register& rd, const Register& rm) { 1776 // Moves involving the stack pointer are encoded as add immediate with 1777 // second operand of zero. Otherwise, orr with first operand zr is 1778 // used. 1779 if (rd.IsSP() || rm.IsSP()) { 1780 add(rd, rm, 0); 1781 } else { 1782 orr(rd, AppropriateZeroRegFor(rd), rm); 1783 } 1784 } 1785 1786 1787 void Assembler::mvn(const Register& rd, const Operand& operand) { 1788 orn(rd, AppropriateZeroRegFor(rd), operand); 1789 } 1790 1791 1792 void Assembler::mrs(const Register& rt, SystemRegister sysreg) { 1793 DCHECK(rt.Is64Bits()); 1794 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt)); 1795 } 1796 1797 1798 void Assembler::msr(SystemRegister sysreg, const Register& rt) { 1799 DCHECK(rt.Is64Bits()); 1800 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg)); 1801 } 1802 1803 1804 void Assembler::hint(SystemHint code) { 1805 Emit(HINT | ImmHint(code) | Rt(xzr)); 1806 } 1807 1808 1809 void Assembler::dmb(BarrierDomain domain, BarrierType type) { 1810 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 1811 } 1812 1813 1814 void Assembler::dsb(BarrierDomain domain, BarrierType type) { 1815 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 1816 } 1817 1818 1819 void Assembler::isb() { 1820 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll)); 1821 } 1822 1823 1824 void Assembler::fmov(FPRegister fd, double imm) { 1825 DCHECK(fd.Is64Bits()); 1826 DCHECK(IsImmFP64(imm)); 1827 Emit(FMOV_d_imm | Rd(fd) | ImmFP64(imm)); 1828 } 1829 1830 1831 void Assembler::fmov(FPRegister fd, float imm) { 1832 DCHECK(fd.Is32Bits()); 1833 DCHECK(IsImmFP32(imm)); 1834 Emit(FMOV_s_imm | Rd(fd) | ImmFP32(imm)); 1835 } 1836 1837 1838 void Assembler::fmov(Register rd, FPRegister fn) { 1839 DCHECK(rd.SizeInBits() == fn.SizeInBits()); 1840 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd; 1841 Emit(op | Rd(rd) | Rn(fn)); 1842 } 1843 1844 1845 void Assembler::fmov(FPRegister fd, Register rn) { 1846 DCHECK(fd.SizeInBits() == rn.SizeInBits()); 1847 FPIntegerConvertOp op = fd.Is32Bits() ? FMOV_sw : FMOV_dx; 1848 Emit(op | Rd(fd) | Rn(rn)); 1849 } 1850 1851 1852 void Assembler::fmov(FPRegister fd, FPRegister fn) { 1853 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1854 Emit(FPType(fd) | FMOV | Rd(fd) | Rn(fn)); 1855 } 1856 1857 1858 void Assembler::fadd(const FPRegister& fd, 1859 const FPRegister& fn, 1860 const FPRegister& fm) { 1861 FPDataProcessing2Source(fd, fn, fm, FADD); 1862 } 1863 1864 1865 void Assembler::fsub(const FPRegister& fd, 1866 const FPRegister& fn, 1867 const FPRegister& fm) { 1868 FPDataProcessing2Source(fd, fn, fm, FSUB); 1869 } 1870 1871 1872 void Assembler::fmul(const FPRegister& fd, 1873 const FPRegister& fn, 1874 const FPRegister& fm) { 1875 FPDataProcessing2Source(fd, fn, fm, FMUL); 1876 } 1877 1878 1879 void Assembler::fmadd(const FPRegister& fd, 1880 const FPRegister& fn, 1881 const FPRegister& fm, 1882 const FPRegister& fa) { 1883 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMADD_s : FMADD_d); 1884 } 1885 1886 1887 void Assembler::fmsub(const FPRegister& fd, 1888 const FPRegister& fn, 1889 const FPRegister& fm, 1890 const FPRegister& fa) { 1891 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMSUB_s : FMSUB_d); 1892 } 1893 1894 1895 void Assembler::fnmadd(const FPRegister& fd, 1896 const FPRegister& fn, 1897 const FPRegister& fm, 1898 const FPRegister& fa) { 1899 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMADD_s : FNMADD_d); 1900 } 1901 1902 1903 void Assembler::fnmsub(const FPRegister& fd, 1904 const FPRegister& fn, 1905 const FPRegister& fm, 1906 const FPRegister& fa) { 1907 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMSUB_s : FNMSUB_d); 1908 } 1909 1910 1911 void Assembler::fdiv(const FPRegister& fd, 1912 const FPRegister& fn, 1913 const FPRegister& fm) { 1914 FPDataProcessing2Source(fd, fn, fm, FDIV); 1915 } 1916 1917 1918 void Assembler::fmax(const FPRegister& fd, 1919 const FPRegister& fn, 1920 const FPRegister& fm) { 1921 FPDataProcessing2Source(fd, fn, fm, FMAX); 1922 } 1923 1924 1925 void Assembler::fmaxnm(const FPRegister& fd, 1926 const FPRegister& fn, 1927 const FPRegister& fm) { 1928 FPDataProcessing2Source(fd, fn, fm, FMAXNM); 1929 } 1930 1931 1932 void Assembler::fmin(const FPRegister& fd, 1933 const FPRegister& fn, 1934 const FPRegister& fm) { 1935 FPDataProcessing2Source(fd, fn, fm, FMIN); 1936 } 1937 1938 1939 void Assembler::fminnm(const FPRegister& fd, 1940 const FPRegister& fn, 1941 const FPRegister& fm) { 1942 FPDataProcessing2Source(fd, fn, fm, FMINNM); 1943 } 1944 1945 1946 void Assembler::fabs(const FPRegister& fd, 1947 const FPRegister& fn) { 1948 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1949 FPDataProcessing1Source(fd, fn, FABS); 1950 } 1951 1952 1953 void Assembler::fneg(const FPRegister& fd, 1954 const FPRegister& fn) { 1955 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1956 FPDataProcessing1Source(fd, fn, FNEG); 1957 } 1958 1959 1960 void Assembler::fsqrt(const FPRegister& fd, 1961 const FPRegister& fn) { 1962 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1963 FPDataProcessing1Source(fd, fn, FSQRT); 1964 } 1965 1966 1967 void Assembler::frinta(const FPRegister& fd, 1968 const FPRegister& fn) { 1969 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1970 FPDataProcessing1Source(fd, fn, FRINTA); 1971 } 1972 1973 1974 void Assembler::frintm(const FPRegister& fd, 1975 const FPRegister& fn) { 1976 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1977 FPDataProcessing1Source(fd, fn, FRINTM); 1978 } 1979 1980 1981 void Assembler::frintn(const FPRegister& fd, 1982 const FPRegister& fn) { 1983 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1984 FPDataProcessing1Source(fd, fn, FRINTN); 1985 } 1986 1987 1988 void Assembler::frintp(const FPRegister& fd, const FPRegister& fn) { 1989 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1990 FPDataProcessing1Source(fd, fn, FRINTP); 1991 } 1992 1993 1994 void Assembler::frintz(const FPRegister& fd, 1995 const FPRegister& fn) { 1996 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 1997 FPDataProcessing1Source(fd, fn, FRINTZ); 1998 } 1999 2000 2001 void Assembler::fcmp(const FPRegister& fn, 2002 const FPRegister& fm) { 2003 DCHECK(fn.SizeInBits() == fm.SizeInBits()); 2004 Emit(FPType(fn) | FCMP | Rm(fm) | Rn(fn)); 2005 } 2006 2007 2008 void Assembler::fcmp(const FPRegister& fn, 2009 double value) { 2010 USE(value); 2011 // Although the fcmp instruction can strictly only take an immediate value of 2012 // +0.0, we don't need to check for -0.0 because the sign of 0.0 doesn't 2013 // affect the result of the comparison. 2014 DCHECK(value == 0.0); 2015 Emit(FPType(fn) | FCMP_zero | Rn(fn)); 2016 } 2017 2018 2019 void Assembler::fccmp(const FPRegister& fn, 2020 const FPRegister& fm, 2021 StatusFlags nzcv, 2022 Condition cond) { 2023 DCHECK(fn.SizeInBits() == fm.SizeInBits()); 2024 Emit(FPType(fn) | FCCMP | Rm(fm) | Cond(cond) | Rn(fn) | Nzcv(nzcv)); 2025 } 2026 2027 2028 void Assembler::fcsel(const FPRegister& fd, 2029 const FPRegister& fn, 2030 const FPRegister& fm, 2031 Condition cond) { 2032 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 2033 DCHECK(fd.SizeInBits() == fm.SizeInBits()); 2034 Emit(FPType(fd) | FCSEL | Rm(fm) | Cond(cond) | Rn(fn) | Rd(fd)); 2035 } 2036 2037 2038 void Assembler::FPConvertToInt(const Register& rd, 2039 const FPRegister& fn, 2040 FPIntegerConvertOp op) { 2041 Emit(SF(rd) | FPType(fn) | op | Rn(fn) | Rd(rd)); 2042 } 2043 2044 2045 void Assembler::fcvt(const FPRegister& fd, 2046 const FPRegister& fn) { 2047 if (fd.Is64Bits()) { 2048 // Convert float to double. 2049 DCHECK(fn.Is32Bits()); 2050 FPDataProcessing1Source(fd, fn, FCVT_ds); 2051 } else { 2052 // Convert double to float. 2053 DCHECK(fn.Is64Bits()); 2054 FPDataProcessing1Source(fd, fn, FCVT_sd); 2055 } 2056 } 2057 2058 2059 void Assembler::fcvtau(const Register& rd, const FPRegister& fn) { 2060 FPConvertToInt(rd, fn, FCVTAU); 2061 } 2062 2063 2064 void Assembler::fcvtas(const Register& rd, const FPRegister& fn) { 2065 FPConvertToInt(rd, fn, FCVTAS); 2066 } 2067 2068 2069 void Assembler::fcvtmu(const Register& rd, const FPRegister& fn) { 2070 FPConvertToInt(rd, fn, FCVTMU); 2071 } 2072 2073 2074 void Assembler::fcvtms(const Register& rd, const FPRegister& fn) { 2075 FPConvertToInt(rd, fn, FCVTMS); 2076 } 2077 2078 2079 void Assembler::fcvtnu(const Register& rd, const FPRegister& fn) { 2080 FPConvertToInt(rd, fn, FCVTNU); 2081 } 2082 2083 2084 void Assembler::fcvtns(const Register& rd, const FPRegister& fn) { 2085 FPConvertToInt(rd, fn, FCVTNS); 2086 } 2087 2088 2089 void Assembler::fcvtzu(const Register& rd, const FPRegister& fn) { 2090 FPConvertToInt(rd, fn, FCVTZU); 2091 } 2092 2093 2094 void Assembler::fcvtzs(const Register& rd, const FPRegister& fn) { 2095 FPConvertToInt(rd, fn, FCVTZS); 2096 } 2097 2098 2099 void Assembler::scvtf(const FPRegister& fd, 2100 const Register& rn, 2101 unsigned fbits) { 2102 if (fbits == 0) { 2103 Emit(SF(rn) | FPType(fd) | SCVTF | Rn(rn) | Rd(fd)); 2104 } else { 2105 Emit(SF(rn) | FPType(fd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 2106 Rd(fd)); 2107 } 2108 } 2109 2110 2111 void Assembler::ucvtf(const FPRegister& fd, 2112 const Register& rn, 2113 unsigned fbits) { 2114 if (fbits == 0) { 2115 Emit(SF(rn) | FPType(fd) | UCVTF | Rn(rn) | Rd(fd)); 2116 } else { 2117 Emit(SF(rn) | FPType(fd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 2118 Rd(fd)); 2119 } 2120 } 2121 2122 2123 void Assembler::dcptr(Label* label) { 2124 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 2125 if (label->is_bound()) { 2126 // The label is bound, so it does not need to be updated and the internal 2127 // reference should be emitted. 2128 // 2129 // In this case, label->pos() returns the offset of the label from the 2130 // start of the buffer. 2131 internal_reference_positions_.push_back(pc_offset()); 2132 dc64(reinterpret_cast<uintptr_t>(buffer_ + label->pos())); 2133 } else { 2134 int32_t offset; 2135 if (label->is_linked()) { 2136 // The label is linked, so the internal reference should be added 2137 // onto the end of the label's link chain. 2138 // 2139 // In this case, label->pos() returns the offset of the last linked 2140 // instruction from the start of the buffer. 2141 offset = label->pos() - pc_offset(); 2142 DCHECK(offset != kStartOfLabelLinkChain); 2143 } else { 2144 // The label is unused, so it now becomes linked and the internal 2145 // reference is at the start of the new link chain. 2146 offset = kStartOfLabelLinkChain; 2147 } 2148 // The instruction at pc is now the last link in the label's chain. 2149 label->link_to(pc_offset()); 2150 2151 // Traditionally the offset to the previous instruction in the chain is 2152 // encoded in the instruction payload (e.g. branch range) but internal 2153 // references are not instructions so while unbound they are encoded as 2154 // two consecutive brk instructions. The two 16-bit immediates are used 2155 // to encode the offset. 2156 offset >>= kInstructionSizeLog2; 2157 DCHECK(is_int32(offset)); 2158 uint32_t high16 = unsigned_bitextract_32(31, 16, offset); 2159 uint32_t low16 = unsigned_bitextract_32(15, 0, offset); 2160 2161 brk(high16); 2162 brk(low16); 2163 } 2164 } 2165 2166 2167 // Note: 2168 // Below, a difference in case for the same letter indicates a 2169 // negated bit. 2170 // If b is 1, then B is 0. 2171 Instr Assembler::ImmFP32(float imm) { 2172 DCHECK(IsImmFP32(imm)); 2173 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000 2174 uint32_t bits = float_to_rawbits(imm); 2175 // bit7: a000.0000 2176 uint32_t bit7 = ((bits >> 31) & 0x1) << 7; 2177 // bit6: 0b00.0000 2178 uint32_t bit6 = ((bits >> 29) & 0x1) << 6; 2179 // bit5_to_0: 00cd.efgh 2180 uint32_t bit5_to_0 = (bits >> 19) & 0x3f; 2181 2182 return (bit7 | bit6 | bit5_to_0) << ImmFP_offset; 2183 } 2184 2185 2186 Instr Assembler::ImmFP64(double imm) { 2187 DCHECK(IsImmFP64(imm)); 2188 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 2189 // 0000.0000.0000.0000.0000.0000.0000.0000 2190 uint64_t bits = double_to_rawbits(imm); 2191 // bit7: a000.0000 2192 uint64_t bit7 = ((bits >> 63) & 0x1) << 7; 2193 // bit6: 0b00.0000 2194 uint64_t bit6 = ((bits >> 61) & 0x1) << 6; 2195 // bit5_to_0: 00cd.efgh 2196 uint64_t bit5_to_0 = (bits >> 48) & 0x3f; 2197 2198 return static_cast<Instr>((bit7 | bit6 | bit5_to_0) << ImmFP_offset); 2199 } 2200 2201 2202 // Code generation helpers. 2203 void Assembler::MoveWide(const Register& rd, 2204 uint64_t imm, 2205 int shift, 2206 MoveWideImmediateOp mov_op) { 2207 // Ignore the top 32 bits of an immediate if we're moving to a W register. 2208 if (rd.Is32Bits()) { 2209 // Check that the top 32 bits are zero (a positive 32-bit number) or top 2210 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits). 2211 DCHECK(((imm >> kWRegSizeInBits) == 0) || 2212 ((imm >> (kWRegSizeInBits - 1)) == 0x1ffffffff)); 2213 imm &= kWRegMask; 2214 } 2215 2216 if (shift >= 0) { 2217 // Explicit shift specified. 2218 DCHECK((shift == 0) || (shift == 16) || (shift == 32) || (shift == 48)); 2219 DCHECK(rd.Is64Bits() || (shift == 0) || (shift == 16)); 2220 shift /= 16; 2221 } else { 2222 // Calculate a new immediate and shift combination to encode the immediate 2223 // argument. 2224 shift = 0; 2225 if ((imm & ~0xffffUL) == 0) { 2226 // Nothing to do. 2227 } else if ((imm & ~(0xffffUL << 16)) == 0) { 2228 imm >>= 16; 2229 shift = 1; 2230 } else if ((imm & ~(0xffffUL << 32)) == 0) { 2231 DCHECK(rd.Is64Bits()); 2232 imm >>= 32; 2233 shift = 2; 2234 } else if ((imm & ~(0xffffUL << 48)) == 0) { 2235 DCHECK(rd.Is64Bits()); 2236 imm >>= 48; 2237 shift = 3; 2238 } 2239 } 2240 2241 DCHECK(is_uint16(imm)); 2242 2243 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | 2244 ImmMoveWide(static_cast<int>(imm)) | ShiftMoveWide(shift)); 2245 } 2246 2247 2248 void Assembler::AddSub(const Register& rd, 2249 const Register& rn, 2250 const Operand& operand, 2251 FlagsUpdate S, 2252 AddSubOp op) { 2253 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2254 DCHECK(!operand.NeedsRelocation(this)); 2255 if (operand.IsImmediate()) { 2256 int64_t immediate = operand.ImmediateValue(); 2257 DCHECK(IsImmAddSub(immediate)); 2258 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 2259 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) | 2260 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn)); 2261 } else if (operand.IsShiftedRegister()) { 2262 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits()); 2263 DCHECK(operand.shift() != ROR); 2264 2265 // For instructions of the form: 2266 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ] 2267 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ] 2268 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ] 2269 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ] 2270 // or their 64-bit register equivalents, convert the operand from shifted to 2271 // extended register mode, and emit an add/sub extended instruction. 2272 if (rn.IsSP() || rd.IsSP()) { 2273 DCHECK(!(rd.IsSP() && (S == SetFlags))); 2274 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S, 2275 AddSubExtendedFixed | op); 2276 } else { 2277 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op); 2278 } 2279 } else { 2280 DCHECK(operand.IsExtendedRegister()); 2281 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op); 2282 } 2283 } 2284 2285 2286 void Assembler::AddSubWithCarry(const Register& rd, 2287 const Register& rn, 2288 const Operand& operand, 2289 FlagsUpdate S, 2290 AddSubWithCarryOp op) { 2291 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2292 DCHECK(rd.SizeInBits() == operand.reg().SizeInBits()); 2293 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 2294 DCHECK(!operand.NeedsRelocation(this)); 2295 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd)); 2296 } 2297 2298 2299 void Assembler::hlt(int code) { 2300 DCHECK(is_uint16(code)); 2301 Emit(HLT | ImmException(code)); 2302 } 2303 2304 2305 void Assembler::brk(int code) { 2306 DCHECK(is_uint16(code)); 2307 Emit(BRK | ImmException(code)); 2308 } 2309 2310 2311 void Assembler::EmitStringData(const char* string) { 2312 size_t len = strlen(string) + 1; 2313 DCHECK(RoundUp(len, kInstructionSize) <= static_cast<size_t>(kGap)); 2314 EmitData(string, static_cast<int>(len)); 2315 // Pad with NULL characters until pc_ is aligned. 2316 const char pad[] = {'\0', '\0', '\0', '\0'}; 2317 STATIC_ASSERT(sizeof(pad) == kInstructionSize); 2318 EmitData(pad, RoundUp(pc_offset(), kInstructionSize) - pc_offset()); 2319 } 2320 2321 2322 void Assembler::debug(const char* message, uint32_t code, Instr params) { 2323 #ifdef USE_SIMULATOR 2324 // Don't generate simulator specific code if we are building a snapshot, which 2325 // might be run on real hardware. 2326 if (!serializer_enabled()) { 2327 // The arguments to the debug marker need to be contiguous in memory, so 2328 // make sure we don't try to emit pools. 2329 BlockPoolsScope scope(this); 2330 2331 Label start; 2332 bind(&start); 2333 2334 // Refer to instructions-arm64.h for a description of the marker and its 2335 // arguments. 2336 hlt(kImmExceptionIsDebug); 2337 DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugCodeOffset); 2338 dc32(code); 2339 DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugParamsOffset); 2340 dc32(params); 2341 DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugMessageOffset); 2342 EmitStringData(message); 2343 hlt(kImmExceptionIsUnreachable); 2344 2345 return; 2346 } 2347 // Fall through if Serializer is enabled. 2348 #endif 2349 2350 if (params & BREAK) { 2351 hlt(kImmExceptionIsDebug); 2352 } 2353 } 2354 2355 2356 void Assembler::Logical(const Register& rd, 2357 const Register& rn, 2358 const Operand& operand, 2359 LogicalOp op) { 2360 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2361 DCHECK(!operand.NeedsRelocation(this)); 2362 if (operand.IsImmediate()) { 2363 int64_t immediate = operand.ImmediateValue(); 2364 unsigned reg_size = rd.SizeInBits(); 2365 2366 DCHECK(immediate != 0); 2367 DCHECK(immediate != -1); 2368 DCHECK(rd.Is64Bits() || is_uint32(immediate)); 2369 2370 // If the operation is NOT, invert the operation and immediate. 2371 if ((op & NOT) == NOT) { 2372 op = static_cast<LogicalOp>(op & ~NOT); 2373 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask); 2374 } 2375 2376 unsigned n, imm_s, imm_r; 2377 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 2378 // Immediate can be encoded in the instruction. 2379 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 2380 } else { 2381 // This case is handled in the macro assembler. 2382 UNREACHABLE(); 2383 } 2384 } else { 2385 DCHECK(operand.IsShiftedRegister()); 2386 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits()); 2387 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed); 2388 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op); 2389 } 2390 } 2391 2392 2393 void Assembler::LogicalImmediate(const Register& rd, 2394 const Register& rn, 2395 unsigned n, 2396 unsigned imm_s, 2397 unsigned imm_r, 2398 LogicalOp op) { 2399 unsigned reg_size = rd.SizeInBits(); 2400 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd); 2401 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) | 2402 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg | 2403 Rn(rn)); 2404 } 2405 2406 2407 void Assembler::ConditionalCompare(const Register& rn, 2408 const Operand& operand, 2409 StatusFlags nzcv, 2410 Condition cond, 2411 ConditionalCompareOp op) { 2412 Instr ccmpop; 2413 DCHECK(!operand.NeedsRelocation(this)); 2414 if (operand.IsImmediate()) { 2415 int64_t immediate = operand.ImmediateValue(); 2416 DCHECK(IsImmConditionalCompare(immediate)); 2417 ccmpop = ConditionalCompareImmediateFixed | op | 2418 ImmCondCmp(static_cast<unsigned>(immediate)); 2419 } else { 2420 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 2421 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg()); 2422 } 2423 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv)); 2424 } 2425 2426 2427 void Assembler::DataProcessing1Source(const Register& rd, 2428 const Register& rn, 2429 DataProcessing1SourceOp op) { 2430 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 2431 Emit(SF(rn) | op | Rn(rn) | Rd(rd)); 2432 } 2433 2434 2435 void Assembler::FPDataProcessing1Source(const FPRegister& fd, 2436 const FPRegister& fn, 2437 FPDataProcessing1SourceOp op) { 2438 Emit(FPType(fn) | op | Rn(fn) | Rd(fd)); 2439 } 2440 2441 2442 void Assembler::FPDataProcessing2Source(const FPRegister& fd, 2443 const FPRegister& fn, 2444 const FPRegister& fm, 2445 FPDataProcessing2SourceOp op) { 2446 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 2447 DCHECK(fd.SizeInBits() == fm.SizeInBits()); 2448 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd)); 2449 } 2450 2451 2452 void Assembler::FPDataProcessing3Source(const FPRegister& fd, 2453 const FPRegister& fn, 2454 const FPRegister& fm, 2455 const FPRegister& fa, 2456 FPDataProcessing3SourceOp op) { 2457 DCHECK(AreSameSizeAndType(fd, fn, fm, fa)); 2458 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd) | Ra(fa)); 2459 } 2460 2461 2462 void Assembler::EmitShift(const Register& rd, 2463 const Register& rn, 2464 Shift shift, 2465 unsigned shift_amount) { 2466 switch (shift) { 2467 case LSL: 2468 lsl(rd, rn, shift_amount); 2469 break; 2470 case LSR: 2471 lsr(rd, rn, shift_amount); 2472 break; 2473 case ASR: 2474 asr(rd, rn, shift_amount); 2475 break; 2476 case ROR: 2477 ror(rd, rn, shift_amount); 2478 break; 2479 default: 2480 UNREACHABLE(); 2481 } 2482 } 2483 2484 2485 void Assembler::EmitExtendShift(const Register& rd, 2486 const Register& rn, 2487 Extend extend, 2488 unsigned left_shift) { 2489 DCHECK(rd.SizeInBits() >= rn.SizeInBits()); 2490 unsigned reg_size = rd.SizeInBits(); 2491 // Use the correct size of register. 2492 Register rn_ = Register::Create(rn.code(), rd.SizeInBits()); 2493 // Bits extracted are high_bit:0. 2494 unsigned high_bit = (8 << (extend & 0x3)) - 1; 2495 // Number of bits left in the result that are not introduced by the shift. 2496 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1); 2497 2498 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) { 2499 switch (extend) { 2500 case UXTB: 2501 case UXTH: 2502 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break; 2503 case SXTB: 2504 case SXTH: 2505 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break; 2506 case UXTX: 2507 case SXTX: { 2508 DCHECK(rn.SizeInBits() == kXRegSizeInBits); 2509 // Nothing to extend. Just shift. 2510 lsl(rd, rn_, left_shift); 2511 break; 2512 } 2513 default: UNREACHABLE(); 2514 } 2515 } else { 2516 // No need to extend as the extended bits would be shifted away. 2517 lsl(rd, rn_, left_shift); 2518 } 2519 } 2520 2521 2522 void Assembler::DataProcShiftedRegister(const Register& rd, 2523 const Register& rn, 2524 const Operand& operand, 2525 FlagsUpdate S, 2526 Instr op) { 2527 DCHECK(operand.IsShiftedRegister()); 2528 DCHECK(rn.Is64Bits() || (rn.Is32Bits() && is_uint5(operand.shift_amount()))); 2529 DCHECK(!operand.NeedsRelocation(this)); 2530 Emit(SF(rd) | op | Flags(S) | 2531 ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) | 2532 Rm(operand.reg()) | Rn(rn) | Rd(rd)); 2533 } 2534 2535 2536 void Assembler::DataProcExtendedRegister(const Register& rd, 2537 const Register& rn, 2538 const Operand& operand, 2539 FlagsUpdate S, 2540 Instr op) { 2541 DCHECK(!operand.NeedsRelocation(this)); 2542 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 2543 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | 2544 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) | 2545 dest_reg | RnSP(rn)); 2546 } 2547 2548 2549 bool Assembler::IsImmAddSub(int64_t immediate) { 2550 return is_uint12(immediate) || 2551 (is_uint12(immediate >> 12) && ((immediate & 0xfff) == 0)); 2552 } 2553 2554 void Assembler::LoadStore(const CPURegister& rt, 2555 const MemOperand& addr, 2556 LoadStoreOp op) { 2557 Instr memop = op | Rt(rt) | RnSP(addr.base()); 2558 2559 if (addr.IsImmediateOffset()) { 2560 LSDataSize size = CalcLSDataSize(op); 2561 if (IsImmLSScaled(addr.offset(), size)) { 2562 int offset = static_cast<int>(addr.offset()); 2563 // Use the scaled addressing mode. 2564 Emit(LoadStoreUnsignedOffsetFixed | memop | 2565 ImmLSUnsigned(offset >> size)); 2566 } else if (IsImmLSUnscaled(addr.offset())) { 2567 int offset = static_cast<int>(addr.offset()); 2568 // Use the unscaled addressing mode. 2569 Emit(LoadStoreUnscaledOffsetFixed | memop | ImmLS(offset)); 2570 } else { 2571 // This case is handled in the macro assembler. 2572 UNREACHABLE(); 2573 } 2574 } else if (addr.IsRegisterOffset()) { 2575 Extend ext = addr.extend(); 2576 Shift shift = addr.shift(); 2577 unsigned shift_amount = addr.shift_amount(); 2578 2579 // LSL is encoded in the option field as UXTX. 2580 if (shift == LSL) { 2581 ext = UXTX; 2582 } 2583 2584 // Shifts are encoded in one bit, indicating a left shift by the memory 2585 // access size. 2586 DCHECK((shift_amount == 0) || 2587 (shift_amount == static_cast<unsigned>(CalcLSDataSize(op)))); 2588 Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) | 2589 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0)); 2590 } else { 2591 // Pre-index and post-index modes. 2592 DCHECK(!rt.Is(addr.base())); 2593 if (IsImmLSUnscaled(addr.offset())) { 2594 int offset = static_cast<int>(addr.offset()); 2595 if (addr.IsPreIndex()) { 2596 Emit(LoadStorePreIndexFixed | memop | ImmLS(offset)); 2597 } else { 2598 DCHECK(addr.IsPostIndex()); 2599 Emit(LoadStorePostIndexFixed | memop | ImmLS(offset)); 2600 } 2601 } else { 2602 // This case is handled in the macro assembler. 2603 UNREACHABLE(); 2604 } 2605 } 2606 } 2607 2608 2609 bool Assembler::IsImmLSUnscaled(int64_t offset) { 2610 return is_int9(offset); 2611 } 2612 2613 2614 bool Assembler::IsImmLSScaled(int64_t offset, LSDataSize size) { 2615 bool offset_is_size_multiple = (((offset >> size) << size) == offset); 2616 return offset_is_size_multiple && is_uint12(offset >> size); 2617 } 2618 2619 2620 bool Assembler::IsImmLSPair(int64_t offset, LSDataSize size) { 2621 bool offset_is_size_multiple = (((offset >> size) << size) == offset); 2622 return offset_is_size_multiple && is_int7(offset >> size); 2623 } 2624 2625 2626 bool Assembler::IsImmLLiteral(int64_t offset) { 2627 int inst_size = static_cast<int>(kInstructionSizeLog2); 2628 bool offset_is_inst_multiple = 2629 (((offset >> inst_size) << inst_size) == offset); 2630 return offset_is_inst_multiple && is_intn(offset, ImmLLiteral_width); 2631 } 2632 2633 2634 // Test if a given value can be encoded in the immediate field of a logical 2635 // instruction. 2636 // If it can be encoded, the function returns true, and values pointed to by n, 2637 // imm_s and imm_r are updated with immediates encoded in the format required 2638 // by the corresponding fields in the logical instruction. 2639 // If it can not be encoded, the function returns false, and the values pointed 2640 // to by n, imm_s and imm_r are undefined. 2641 bool Assembler::IsImmLogical(uint64_t value, 2642 unsigned width, 2643 unsigned* n, 2644 unsigned* imm_s, 2645 unsigned* imm_r) { 2646 DCHECK((n != NULL) && (imm_s != NULL) && (imm_r != NULL)); 2647 DCHECK((width == kWRegSizeInBits) || (width == kXRegSizeInBits)); 2648 2649 bool negate = false; 2650 2651 // Logical immediates are encoded using parameters n, imm_s and imm_r using 2652 // the following table: 2653 // 2654 // N imms immr size S R 2655 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr) 2656 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr) 2657 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr) 2658 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr) 2659 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr) 2660 // 0 11110s xxxxxr 2 UInt(s) UInt(r) 2661 // (s bits must not be all set) 2662 // 2663 // A pattern is constructed of size bits, where the least significant S+1 bits 2664 // are set. The pattern is rotated right by R, and repeated across a 32 or 2665 // 64-bit value, depending on destination register width. 2666 // 2667 // Put another way: the basic format of a logical immediate is a single 2668 // contiguous stretch of 1 bits, repeated across the whole word at intervals 2669 // given by a power of 2. To identify them quickly, we first locate the 2670 // lowest stretch of 1 bits, then the next 1 bit above that; that combination 2671 // is different for every logical immediate, so it gives us all the 2672 // information we need to identify the only logical immediate that our input 2673 // could be, and then we simply check if that's the value we actually have. 2674 // 2675 // (The rotation parameter does give the possibility of the stretch of 1 bits 2676 // going 'round the end' of the word. To deal with that, we observe that in 2677 // any situation where that happens the bitwise NOT of the value is also a 2678 // valid logical immediate. So we simply invert the input whenever its low bit 2679 // is set, and then we know that the rotated case can't arise.) 2680 2681 if (value & 1) { 2682 // If the low bit is 1, negate the value, and set a flag to remember that we 2683 // did (so that we can adjust the return values appropriately). 2684 negate = true; 2685 value = ~value; 2686 } 2687 2688 if (width == kWRegSizeInBits) { 2689 // To handle 32-bit logical immediates, the very easiest thing is to repeat 2690 // the input value twice to make a 64-bit word. The correct encoding of that 2691 // as a logical immediate will also be the correct encoding of the 32-bit 2692 // value. 2693 2694 // The most-significant 32 bits may not be zero (ie. negate is true) so 2695 // shift the value left before duplicating it. 2696 value <<= kWRegSizeInBits; 2697 value |= value >> kWRegSizeInBits; 2698 } 2699 2700 // The basic analysis idea: imagine our input word looks like this. 2701 // 2702 // 0011111000111110001111100011111000111110001111100011111000111110 2703 // c b a 2704 // |<--d-->| 2705 // 2706 // We find the lowest set bit (as an actual power-of-2 value, not its index) 2707 // and call it a. Then we add a to our original number, which wipes out the 2708 // bottommost stretch of set bits and replaces it with a 1 carried into the 2709 // next zero bit. Then we look for the new lowest set bit, which is in 2710 // position b, and subtract it, so now our number is just like the original 2711 // but with the lowest stretch of set bits completely gone. Now we find the 2712 // lowest set bit again, which is position c in the diagram above. Then we'll 2713 // measure the distance d between bit positions a and c (using CLZ), and that 2714 // tells us that the only valid logical immediate that could possibly be equal 2715 // to this number is the one in which a stretch of bits running from a to just 2716 // below b is replicated every d bits. 2717 uint64_t a = LargestPowerOf2Divisor(value); 2718 uint64_t value_plus_a = value + a; 2719 uint64_t b = LargestPowerOf2Divisor(value_plus_a); 2720 uint64_t value_plus_a_minus_b = value_plus_a - b; 2721 uint64_t c = LargestPowerOf2Divisor(value_plus_a_minus_b); 2722 2723 int d, clz_a, out_n; 2724 uint64_t mask; 2725 2726 if (c != 0) { 2727 // The general case, in which there is more than one stretch of set bits. 2728 // Compute the repeat distance d, and set up a bitmask covering the basic 2729 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all 2730 // of these cases the N bit of the output will be zero. 2731 clz_a = CountLeadingZeros(a, kXRegSizeInBits); 2732 int clz_c = CountLeadingZeros(c, kXRegSizeInBits); 2733 d = clz_a - clz_c; 2734 mask = ((V8_UINT64_C(1) << d) - 1); 2735 out_n = 0; 2736 } else { 2737 // Handle degenerate cases. 2738 // 2739 // If any of those 'find lowest set bit' operations didn't find a set bit at 2740 // all, then the word will have been zero thereafter, so in particular the 2741 // last lowest_set_bit operation will have returned zero. So we can test for 2742 // all the special case conditions in one go by seeing if c is zero. 2743 if (a == 0) { 2744 // The input was zero (or all 1 bits, which will come to here too after we 2745 // inverted it at the start of the function), for which we just return 2746 // false. 2747 return false; 2748 } else { 2749 // Otherwise, if c was zero but a was not, then there's just one stretch 2750 // of set bits in our word, meaning that we have the trivial case of 2751 // d == 64 and only one 'repetition'. Set up all the same variables as in 2752 // the general case above, and set the N bit in the output. 2753 clz_a = CountLeadingZeros(a, kXRegSizeInBits); 2754 d = 64; 2755 mask = ~V8_UINT64_C(0); 2756 out_n = 1; 2757 } 2758 } 2759 2760 // If the repeat period d is not a power of two, it can't be encoded. 2761 if (!IS_POWER_OF_TWO(d)) { 2762 return false; 2763 } 2764 2765 if (((b - a) & ~mask) != 0) { 2766 // If the bit stretch (b - a) does not fit within the mask derived from the 2767 // repeat period, then fail. 2768 return false; 2769 } 2770 2771 // The only possible option is b - a repeated every d bits. Now we're going to 2772 // actually construct the valid logical immediate derived from that 2773 // specification, and see if it equals our original input. 2774 // 2775 // To repeat a value every d bits, we multiply it by a number of the form 2776 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can 2777 // be derived using a table lookup on CLZ(d). 2778 static const uint64_t multipliers[] = { 2779 0x0000000000000001UL, 2780 0x0000000100000001UL, 2781 0x0001000100010001UL, 2782 0x0101010101010101UL, 2783 0x1111111111111111UL, 2784 0x5555555555555555UL, 2785 }; 2786 int multiplier_idx = CountLeadingZeros(d, kXRegSizeInBits) - 57; 2787 // Ensure that the index to the multipliers array is within bounds. 2788 DCHECK((multiplier_idx >= 0) && 2789 (static_cast<size_t>(multiplier_idx) < arraysize(multipliers))); 2790 uint64_t multiplier = multipliers[multiplier_idx]; 2791 uint64_t candidate = (b - a) * multiplier; 2792 2793 if (value != candidate) { 2794 // The candidate pattern doesn't match our input value, so fail. 2795 return false; 2796 } 2797 2798 // We have a match! This is a valid logical immediate, so now we have to 2799 // construct the bits and pieces of the instruction encoding that generates 2800 // it. 2801 2802 // Count the set bits in our basic stretch. The special case of clz(0) == -1 2803 // makes the answer come out right for stretches that reach the very top of 2804 // the word (e.g. numbers like 0xffffc00000000000). 2805 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSizeInBits); 2806 int s = clz_a - clz_b; 2807 2808 // Decide how many bits to rotate right by, to put the low bit of that basic 2809 // stretch in position a. 2810 int r; 2811 if (negate) { 2812 // If we inverted the input right at the start of this function, here's 2813 // where we compensate: the number of set bits becomes the number of clear 2814 // bits, and the rotation count is based on position b rather than position 2815 // a (since b is the location of the 'lowest' 1 bit after inversion). 2816 s = d - s; 2817 r = (clz_b + 1) & (d - 1); 2818 } else { 2819 r = (clz_a + 1) & (d - 1); 2820 } 2821 2822 // Now we're done, except for having to encode the S output in such a way that 2823 // it gives both the number of set bits and the length of the repeated 2824 // segment. The s field is encoded like this: 2825 // 2826 // imms size S 2827 // ssssss 64 UInt(ssssss) 2828 // 0sssss 32 UInt(sssss) 2829 // 10ssss 16 UInt(ssss) 2830 // 110sss 8 UInt(sss) 2831 // 1110ss 4 UInt(ss) 2832 // 11110s 2 UInt(s) 2833 // 2834 // So we 'or' (-d << 1) with our computed s to form imms. 2835 *n = out_n; 2836 *imm_s = ((-d << 1) | (s - 1)) & 0x3f; 2837 *imm_r = r; 2838 2839 return true; 2840 } 2841 2842 2843 bool Assembler::IsImmConditionalCompare(int64_t immediate) { 2844 return is_uint5(immediate); 2845 } 2846 2847 2848 bool Assembler::IsImmFP32(float imm) { 2849 // Valid values will have the form: 2850 // aBbb.bbbc.defg.h000.0000.0000.0000.0000 2851 uint32_t bits = float_to_rawbits(imm); 2852 // bits[19..0] are cleared. 2853 if ((bits & 0x7ffff) != 0) { 2854 return false; 2855 } 2856 2857 // bits[29..25] are all set or all cleared. 2858 uint32_t b_pattern = (bits >> 16) & 0x3e00; 2859 if (b_pattern != 0 && b_pattern != 0x3e00) { 2860 return false; 2861 } 2862 2863 // bit[30] and bit[29] are opposite. 2864 if (((bits ^ (bits << 1)) & 0x40000000) == 0) { 2865 return false; 2866 } 2867 2868 return true; 2869 } 2870 2871 2872 bool Assembler::IsImmFP64(double imm) { 2873 // Valid values will have the form: 2874 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 2875 // 0000.0000.0000.0000.0000.0000.0000.0000 2876 uint64_t bits = double_to_rawbits(imm); 2877 // bits[47..0] are cleared. 2878 if ((bits & 0xffffffffffffL) != 0) { 2879 return false; 2880 } 2881 2882 // bits[61..54] are all set or all cleared. 2883 uint32_t b_pattern = (bits >> 48) & 0x3fc0; 2884 if (b_pattern != 0 && b_pattern != 0x3fc0) { 2885 return false; 2886 } 2887 2888 // bit[62] and bit[61] are opposite. 2889 if (((bits ^ (bits << 1)) & 0x4000000000000000L) == 0) { 2890 return false; 2891 } 2892 2893 return true; 2894 } 2895 2896 2897 void Assembler::GrowBuffer() { 2898 if (!own_buffer_) FATAL("external code buffer is too small"); 2899 2900 // Compute new buffer size. 2901 CodeDesc desc; // the new buffer 2902 if (buffer_size_ < 1 * MB) { 2903 desc.buffer_size = 2 * buffer_size_; 2904 } else { 2905 desc.buffer_size = buffer_size_ + 1 * MB; 2906 } 2907 CHECK_GT(desc.buffer_size, 0); // No overflow. 2908 2909 byte* buffer = reinterpret_cast<byte*>(buffer_); 2910 2911 // Set up new buffer. 2912 desc.buffer = NewArray<byte>(desc.buffer_size); 2913 desc.origin = this; 2914 2915 desc.instr_size = pc_offset(); 2916 desc.reloc_size = 2917 static_cast<int>((buffer + buffer_size_) - reloc_info_writer.pos()); 2918 2919 // Copy the data. 2920 intptr_t pc_delta = desc.buffer - buffer; 2921 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 2922 (buffer + buffer_size_); 2923 memmove(desc.buffer, buffer, desc.instr_size); 2924 memmove(reloc_info_writer.pos() + rc_delta, 2925 reloc_info_writer.pos(), desc.reloc_size); 2926 2927 // Switch buffers. 2928 DeleteArray(buffer_); 2929 buffer_ = desc.buffer; 2930 buffer_size_ = desc.buffer_size; 2931 pc_ = reinterpret_cast<byte*>(pc_) + pc_delta; 2932 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 2933 reloc_info_writer.last_pc() + pc_delta); 2934 2935 // None of our relocation types are pc relative pointing outside the code 2936 // buffer nor pc absolute pointing inside the code buffer, so there is no need 2937 // to relocate any emitted relocation entries. 2938 2939 // Relocate internal references. 2940 for (auto pos : internal_reference_positions_) { 2941 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 2942 *p += pc_delta; 2943 } 2944 2945 // Pending relocation entries are also relative, no need to relocate. 2946 } 2947 2948 2949 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { 2950 // We do not try to reuse pool constants. 2951 RelocInfo rinfo(isolate(), reinterpret_cast<byte*>(pc_), rmode, data, NULL); 2952 if (((rmode >= RelocInfo::COMMENT) && 2953 (rmode <= RelocInfo::DEBUG_BREAK_SLOT_AT_TAIL_CALL)) || 2954 (rmode == RelocInfo::INTERNAL_REFERENCE) || 2955 (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) || 2956 (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) || 2957 (rmode == RelocInfo::DEOPT_INLINING_ID) || 2958 (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID)) { 2959 // Adjust code for new modes. 2960 DCHECK(RelocInfo::IsDebugBreakSlot(rmode) || RelocInfo::IsComment(rmode) || 2961 RelocInfo::IsDeoptReason(rmode) || RelocInfo::IsDeoptId(rmode) || 2962 RelocInfo::IsDeoptPosition(rmode) || 2963 RelocInfo::IsInternalReference(rmode) || 2964 RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode)); 2965 // These modes do not need an entry in the constant pool. 2966 } else { 2967 constpool_.RecordEntry(data, rmode); 2968 // Make sure the constant pool is not emitted in place of the next 2969 // instruction for which we just recorded relocation info. 2970 BlockConstPoolFor(1); 2971 } 2972 2973 if (!RelocInfo::IsNone(rmode)) { 2974 // Don't record external references unless the heap will be serialized. 2975 if (rmode == RelocInfo::EXTERNAL_REFERENCE && 2976 !serializer_enabled() && !emit_debug_code()) { 2977 return; 2978 } 2979 DCHECK(buffer_space() >= kMaxRelocSize); // too late to grow buffer here 2980 if (rmode == RelocInfo::CODE_TARGET_WITH_ID) { 2981 RelocInfo reloc_info_with_ast_id(isolate(), reinterpret_cast<byte*>(pc_), 2982 rmode, RecordedAstId().ToInt(), NULL); 2983 ClearRecordedAstId(); 2984 reloc_info_writer.Write(&reloc_info_with_ast_id); 2985 } else { 2986 reloc_info_writer.Write(&rinfo); 2987 } 2988 } 2989 } 2990 2991 2992 void Assembler::BlockConstPoolFor(int instructions) { 2993 int pc_limit = pc_offset() + instructions * kInstructionSize; 2994 if (no_const_pool_before_ < pc_limit) { 2995 no_const_pool_before_ = pc_limit; 2996 // Make sure the pool won't be blocked for too long. 2997 DCHECK(pc_limit < constpool_.MaxPcOffset()); 2998 } 2999 3000 if (next_constant_pool_check_ < no_const_pool_before_) { 3001 next_constant_pool_check_ = no_const_pool_before_; 3002 } 3003 } 3004 3005 3006 void Assembler::CheckConstPool(bool force_emit, bool require_jump) { 3007 // Some short sequence of instruction mustn't be broken up by constant pool 3008 // emission, such sequences are protected by calls to BlockConstPoolFor and 3009 // BlockConstPoolScope. 3010 if (is_const_pool_blocked()) { 3011 // Something is wrong if emission is forced and blocked at the same time. 3012 DCHECK(!force_emit); 3013 return; 3014 } 3015 3016 // There is nothing to do if there are no pending constant pool entries. 3017 if (constpool_.IsEmpty()) { 3018 // Calculate the offset of the next check. 3019 SetNextConstPoolCheckIn(kCheckConstPoolInterval); 3020 return; 3021 } 3022 3023 // We emit a constant pool when: 3024 // * requested to do so by parameter force_emit (e.g. after each function). 3025 // * the distance to the first instruction accessing the constant pool is 3026 // kApproxMaxDistToConstPool or more. 3027 // * the number of entries in the pool is kApproxMaxPoolEntryCount or more. 3028 int dist = constpool_.DistanceToFirstUse(); 3029 int count = constpool_.EntryCount(); 3030 if (!force_emit && 3031 (dist < kApproxMaxDistToConstPool) && 3032 (count < kApproxMaxPoolEntryCount)) { 3033 return; 3034 } 3035 3036 3037 // Emit veneers for branches that would go out of range during emission of the 3038 // constant pool. 3039 int worst_case_size = constpool_.WorstCaseSize(); 3040 CheckVeneerPool(false, require_jump, 3041 kVeneerDistanceMargin + worst_case_size); 3042 3043 // Check that the code buffer is large enough before emitting the constant 3044 // pool (this includes the gap to the relocation information). 3045 int needed_space = worst_case_size + kGap + 1 * kInstructionSize; 3046 while (buffer_space() <= needed_space) { 3047 GrowBuffer(); 3048 } 3049 3050 Label size_check; 3051 bind(&size_check); 3052 constpool_.Emit(require_jump); 3053 DCHECK(SizeOfCodeGeneratedSince(&size_check) <= 3054 static_cast<unsigned>(worst_case_size)); 3055 3056 // Since a constant pool was just emitted, move the check offset forward by 3057 // the standard interval. 3058 SetNextConstPoolCheckIn(kCheckConstPoolInterval); 3059 } 3060 3061 3062 bool Assembler::ShouldEmitVeneer(int max_reachable_pc, int margin) { 3063 // Account for the branch around the veneers and the guard. 3064 int protection_offset = 2 * kInstructionSize; 3065 return pc_offset() > max_reachable_pc - margin - protection_offset - 3066 static_cast<int>(unresolved_branches_.size() * kMaxVeneerCodeSize); 3067 } 3068 3069 3070 void Assembler::RecordVeneerPool(int location_offset, int size) { 3071 RelocInfo rinfo(isolate(), buffer_ + location_offset, RelocInfo::VENEER_POOL, 3072 static_cast<intptr_t>(size), NULL); 3073 reloc_info_writer.Write(&rinfo); 3074 } 3075 3076 3077 void Assembler::EmitVeneers(bool force_emit, bool need_protection, int margin) { 3078 BlockPoolsScope scope(this); 3079 RecordComment("[ Veneers"); 3080 3081 // The exact size of the veneer pool must be recorded (see the comment at the 3082 // declaration site of RecordConstPool()), but computing the number of 3083 // veneers that will be generated is not obvious. So instead we remember the 3084 // current position and will record the size after the pool has been 3085 // generated. 3086 Label size_check; 3087 bind(&size_check); 3088 int veneer_pool_relocinfo_loc = pc_offset(); 3089 3090 Label end; 3091 if (need_protection) { 3092 b(&end); 3093 } 3094 3095 EmitVeneersGuard(); 3096 3097 Label veneer_size_check; 3098 3099 std::multimap<int, FarBranchInfo>::iterator it, it_to_delete; 3100 3101 it = unresolved_branches_.begin(); 3102 while (it != unresolved_branches_.end()) { 3103 if (force_emit || ShouldEmitVeneer(it->first, margin)) { 3104 Instruction* branch = InstructionAt(it->second.pc_offset_); 3105 Label* label = it->second.label_; 3106 3107 #ifdef DEBUG 3108 bind(&veneer_size_check); 3109 #endif 3110 // Patch the branch to point to the current position, and emit a branch 3111 // to the label. 3112 Instruction* veneer = reinterpret_cast<Instruction*>(pc_); 3113 RemoveBranchFromLabelLinkChain(branch, label, veneer); 3114 branch->SetImmPCOffsetTarget(isolate(), veneer); 3115 b(label); 3116 #ifdef DEBUG 3117 DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <= 3118 static_cast<uint64_t>(kMaxVeneerCodeSize)); 3119 veneer_size_check.Unuse(); 3120 #endif 3121 3122 it_to_delete = it++; 3123 unresolved_branches_.erase(it_to_delete); 3124 } else { 3125 ++it; 3126 } 3127 } 3128 3129 // Record the veneer pool size. 3130 int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check)); 3131 RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size); 3132 3133 if (unresolved_branches_.empty()) { 3134 next_veneer_pool_check_ = kMaxInt; 3135 } else { 3136 next_veneer_pool_check_ = 3137 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 3138 } 3139 3140 bind(&end); 3141 3142 RecordComment("]"); 3143 } 3144 3145 3146 void Assembler::CheckVeneerPool(bool force_emit, bool require_jump, 3147 int margin) { 3148 // There is nothing to do if there are no pending veneer pool entries. 3149 if (unresolved_branches_.empty()) { 3150 DCHECK(next_veneer_pool_check_ == kMaxInt); 3151 return; 3152 } 3153 3154 DCHECK(pc_offset() < unresolved_branches_first_limit()); 3155 3156 // Some short sequence of instruction mustn't be broken up by veneer pool 3157 // emission, such sequences are protected by calls to BlockVeneerPoolFor and 3158 // BlockVeneerPoolScope. 3159 if (is_veneer_pool_blocked()) { 3160 DCHECK(!force_emit); 3161 return; 3162 } 3163 3164 if (!require_jump) { 3165 // Prefer emitting veneers protected by an existing instruction. 3166 margin *= kVeneerNoProtectionFactor; 3167 } 3168 if (force_emit || ShouldEmitVeneers(margin)) { 3169 EmitVeneers(force_emit, require_jump, margin); 3170 } else { 3171 next_veneer_pool_check_ = 3172 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 3173 } 3174 } 3175 3176 3177 int Assembler::buffer_space() const { 3178 return static_cast<int>(reloc_info_writer.pos() - 3179 reinterpret_cast<byte*>(pc_)); 3180 } 3181 3182 3183 void Assembler::RecordConstPool(int size) { 3184 // We only need this for debugger support, to correctly compute offsets in the 3185 // code. 3186 RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size)); 3187 } 3188 3189 3190 void PatchingAssembler::PatchAdrFar(int64_t target_offset) { 3191 // The code at the current instruction should be: 3192 // adr rd, 0 3193 // nop (adr_far) 3194 // nop (adr_far) 3195 // movz scratch, 0 3196 3197 // Verify the expected code. 3198 Instruction* expected_adr = InstructionAt(0); 3199 CHECK(expected_adr->IsAdr() && (expected_adr->ImmPCRel() == 0)); 3200 int rd_code = expected_adr->Rd(); 3201 for (int i = 0; i < kAdrFarPatchableNNops; ++i) { 3202 CHECK(InstructionAt((i + 1) * kInstructionSize)->IsNop(ADR_FAR_NOP)); 3203 } 3204 Instruction* expected_movz = 3205 InstructionAt((kAdrFarPatchableNInstrs - 1) * kInstructionSize); 3206 CHECK(expected_movz->IsMovz() && 3207 (expected_movz->ImmMoveWide() == 0) && 3208 (expected_movz->ShiftMoveWide() == 0)); 3209 int scratch_code = expected_movz->Rd(); 3210 3211 // Patch to load the correct address. 3212 Register rd = Register::XRegFromCode(rd_code); 3213 Register scratch = Register::XRegFromCode(scratch_code); 3214 // Addresses are only 48 bits. 3215 adr(rd, target_offset & 0xFFFF); 3216 movz(scratch, (target_offset >> 16) & 0xFFFF, 16); 3217 movk(scratch, (target_offset >> 32) & 0xFFFF, 32); 3218 DCHECK((target_offset >> 48) == 0); 3219 add(rd, rd, scratch); 3220 } 3221 3222 3223 } // namespace internal 3224 } // namespace v8 3225 3226 #endif // V8_TARGET_ARCH_ARM64 3227