1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // 3 // Redistribution and use in source and binary forms, with or without 4 // modification, are permitted provided that the following conditions are 5 // met: 6 // 7 // * Redistributions of source code must retain the above copyright 8 // notice, this list of conditions and the following disclaimer. 9 // * Redistributions in binary form must reproduce the above 10 // copyright notice, this list of conditions and the following 11 // disclaimer in the documentation and/or other materials provided 12 // with the distribution. 13 // * Neither the name of Google Inc. nor the names of its 14 // contributors may be used to endorse or promote products derived 15 // from this software without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #if V8_TARGET_ARCH_ARM64 30 31 #include "src/arm64/assembler-arm64.h" 32 33 #include "src/arm64/assembler-arm64-inl.h" 34 #include "src/base/bits.h" 35 #include "src/base/cpu.h" 36 #include "src/code-stubs.h" 37 #include "src/frame-constants.h" 38 #include "src/register-configuration.h" 39 40 namespace v8 { 41 namespace internal { 42 43 // ----------------------------------------------------------------------------- 44 // CpuFeatures implementation. 45 46 void CpuFeatures::ProbeImpl(bool cross_compile) { 47 // AArch64 has no configuration options, no further probing is required. 48 supported_ = 0; 49 50 // Only use statically determined features for cross compile (snapshot). 51 if (cross_compile) return; 52 53 // We used to probe for coherent cache support, but on older CPUs it 54 // causes crashes (crbug.com/524337), and newer CPUs don't even have 55 // the feature any more. 56 } 57 58 void CpuFeatures::PrintTarget() { } 59 void CpuFeatures::PrintFeatures() {} 60 61 // ----------------------------------------------------------------------------- 62 // CPURegList utilities. 63 64 CPURegister CPURegList::PopLowestIndex() { 65 DCHECK(IsValid()); 66 if (IsEmpty()) { 67 return NoCPUReg; 68 } 69 int index = CountTrailingZeros(list_, kRegListSizeInBits); 70 DCHECK((1 << index) & list_); 71 Remove(index); 72 return CPURegister::Create(index, size_, type_); 73 } 74 75 76 CPURegister CPURegList::PopHighestIndex() { 77 DCHECK(IsValid()); 78 if (IsEmpty()) { 79 return NoCPUReg; 80 } 81 int index = CountLeadingZeros(list_, kRegListSizeInBits); 82 index = kRegListSizeInBits - 1 - index; 83 DCHECK((1 << index) & list_); 84 Remove(index); 85 return CPURegister::Create(index, size_, type_); 86 } 87 88 89 void CPURegList::RemoveCalleeSaved() { 90 if (type() == CPURegister::kRegister) { 91 Remove(GetCalleeSaved(RegisterSizeInBits())); 92 } else if (type() == CPURegister::kVRegister) { 93 Remove(GetCalleeSavedV(RegisterSizeInBits())); 94 } else { 95 DCHECK_EQ(type(), CPURegister::kNoRegister); 96 DCHECK(IsEmpty()); 97 // The list must already be empty, so do nothing. 98 } 99 } 100 101 102 CPURegList CPURegList::GetCalleeSaved(int size) { 103 return CPURegList(CPURegister::kRegister, size, 19, 29); 104 } 105 106 CPURegList CPURegList::GetCalleeSavedV(int size) { 107 return CPURegList(CPURegister::kVRegister, size, 8, 15); 108 } 109 110 111 CPURegList CPURegList::GetCallerSaved(int size) { 112 // Registers x0-x18 and lr (x30) are caller-saved. 113 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18); 114 list.Combine(lr); 115 return list; 116 } 117 118 CPURegList CPURegList::GetCallerSavedV(int size) { 119 // Registers d0-d7 and d16-d31 are caller-saved. 120 CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7); 121 list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31)); 122 return list; 123 } 124 125 126 // This function defines the list of registers which are associated with a 127 // safepoint slot. Safepoint register slots are saved contiguously on the stack. 128 // MacroAssembler::SafepointRegisterStackIndex handles mapping from register 129 // code to index in the safepoint register slots. Any change here can affect 130 // this mapping. 131 CPURegList CPURegList::GetSafepointSavedRegisters() { 132 CPURegList list = CPURegList::GetCalleeSaved(); 133 list.Combine( 134 CPURegList(CPURegister::kRegister, kXRegSizeInBits, kJSCallerSaved)); 135 136 // Note that unfortunately we can't use symbolic names for registers and have 137 // to directly use register codes. This is because this function is used to 138 // initialize some static variables and we can't rely on register variables 139 // to be initialized due to static initialization order issues in C++. 140 141 // Drop ip0 and ip1 (i.e. x16 and x17), as they should not be expected to be 142 // preserved outside of the macro assembler. 143 list.Remove(16); 144 list.Remove(17); 145 146 // Add x18 to the safepoint list, as although it's not in kJSCallerSaved, it 147 // is a caller-saved register according to the procedure call standard. 148 list.Combine(18); 149 150 // Add the link register (x30) to the safepoint list. 151 list.Combine(30); 152 153 return list; 154 } 155 156 157 // ----------------------------------------------------------------------------- 158 // Implementation of RelocInfo 159 160 const int RelocInfo::kApplyMask = 161 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) | 162 RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) | 163 RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE); 164 165 bool RelocInfo::IsCodedSpecially() { 166 // The deserializer needs to know whether a pointer is specially coded. Being 167 // specially coded on ARM64 means that it is an immediate branch. 168 Instruction* instr = reinterpret_cast<Instruction*>(pc_); 169 if (instr->IsLdrLiteralX()) { 170 return false; 171 } else { 172 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); 173 return true; 174 } 175 } 176 177 178 bool RelocInfo::IsInConstantPool() { 179 Instruction* instr = reinterpret_cast<Instruction*>(pc_); 180 return instr->IsLdrLiteralX(); 181 } 182 183 int RelocInfo::GetDeoptimizationId(Isolate* isolate, DeoptimizeKind kind) { 184 DCHECK(IsRuntimeEntry(rmode_)); 185 Instruction* movz_instr = reinterpret_cast<Instruction*>(pc_)->preceding(); 186 DCHECK(movz_instr->IsMovz()); 187 uint64_t imm = static_cast<uint64_t>(movz_instr->ImmMoveWide()) 188 << (16 * movz_instr->ShiftMoveWide()); 189 DCHECK_LE(imm, INT_MAX); 190 191 return static_cast<int>(imm); 192 } 193 194 void RelocInfo::set_js_to_wasm_address(Address address, 195 ICacheFlushMode icache_flush_mode) { 196 DCHECK_EQ(rmode_, JS_TO_WASM_CALL); 197 Assembler::set_target_address_at(pc_, constant_pool_, address, 198 icache_flush_mode); 199 } 200 201 Address RelocInfo::js_to_wasm_address() const { 202 DCHECK_EQ(rmode_, JS_TO_WASM_CALL); 203 return Assembler::target_address_at(pc_, constant_pool_); 204 } 205 206 uint32_t RelocInfo::wasm_call_tag() const { 207 DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL); 208 Instruction* instr = reinterpret_cast<Instruction*>(pc_); 209 if (instr->IsLdrLiteralX()) { 210 return static_cast<uint32_t>( 211 Memory<Address>(Assembler::target_pointer_address_at(pc_))); 212 } else { 213 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); 214 return static_cast<uint32_t>(instr->ImmPCOffset() / kInstrSize); 215 } 216 } 217 218 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2, 219 const CPURegister& reg3, const CPURegister& reg4, 220 const CPURegister& reg5, const CPURegister& reg6, 221 const CPURegister& reg7, const CPURegister& reg8) { 222 int number_of_valid_regs = 0; 223 int number_of_valid_fpregs = 0; 224 225 RegList unique_regs = 0; 226 RegList unique_fpregs = 0; 227 228 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8}; 229 230 for (unsigned i = 0; i < arraysize(regs); i++) { 231 if (regs[i].IsRegister()) { 232 number_of_valid_regs++; 233 unique_regs |= regs[i].bit(); 234 } else if (regs[i].IsVRegister()) { 235 number_of_valid_fpregs++; 236 unique_fpregs |= regs[i].bit(); 237 } else { 238 DCHECK(!regs[i].IsValid()); 239 } 240 } 241 242 int number_of_unique_regs = 243 CountSetBits(unique_regs, sizeof(unique_regs) * kBitsPerByte); 244 int number_of_unique_fpregs = 245 CountSetBits(unique_fpregs, sizeof(unique_fpregs) * kBitsPerByte); 246 247 DCHECK(number_of_valid_regs >= number_of_unique_regs); 248 DCHECK(number_of_valid_fpregs >= number_of_unique_fpregs); 249 250 return (number_of_valid_regs != number_of_unique_regs) || 251 (number_of_valid_fpregs != number_of_unique_fpregs); 252 } 253 254 255 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2, 256 const CPURegister& reg3, const CPURegister& reg4, 257 const CPURegister& reg5, const CPURegister& reg6, 258 const CPURegister& reg7, const CPURegister& reg8) { 259 DCHECK(reg1.IsValid()); 260 bool match = true; 261 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1); 262 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1); 263 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1); 264 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1); 265 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1); 266 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1); 267 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1); 268 return match; 269 } 270 271 bool AreSameFormat(const VRegister& reg1, const VRegister& reg2, 272 const VRegister& reg3, const VRegister& reg4) { 273 DCHECK(reg1.IsValid()); 274 return (!reg2.IsValid() || reg2.IsSameFormat(reg1)) && 275 (!reg3.IsValid() || reg3.IsSameFormat(reg1)) && 276 (!reg4.IsValid() || reg4.IsSameFormat(reg1)); 277 } 278 279 bool AreConsecutive(const VRegister& reg1, const VRegister& reg2, 280 const VRegister& reg3, const VRegister& reg4) { 281 DCHECK(reg1.IsValid()); 282 if (!reg2.IsValid()) { 283 DCHECK(!reg3.IsValid() && !reg4.IsValid()); 284 return true; 285 } else if (reg2.code() != ((reg1.code() + 1) % kNumberOfVRegisters)) { 286 return false; 287 } 288 289 if (!reg3.IsValid()) { 290 DCHECK(!reg4.IsValid()); 291 return true; 292 } else if (reg3.code() != ((reg2.code() + 1) % kNumberOfVRegisters)) { 293 return false; 294 } 295 296 if (!reg4.IsValid()) { 297 return true; 298 } else if (reg4.code() != ((reg3.code() + 1) % kNumberOfVRegisters)) { 299 return false; 300 } 301 302 return true; 303 } 304 305 void Immediate::InitializeHandle(Handle<HeapObject> handle) { 306 value_ = static_cast<intptr_t>(handle.address()); 307 rmode_ = RelocInfo::EMBEDDED_OBJECT; 308 } 309 310 311 bool Operand::NeedsRelocation(const Assembler* assembler) const { 312 RelocInfo::Mode rmode = immediate_.rmode(); 313 314 if (RelocInfo::IsOnlyForSerializer(rmode)) { 315 return assembler->options().record_reloc_info_for_serialization; 316 } 317 318 return !RelocInfo::IsNone(rmode); 319 } 320 321 bool ConstPool::AddSharedEntry(SharedEntryMap& entry_map, uint64_t data, 322 int offset) { 323 auto existing = entry_map.find(data); 324 if (existing == entry_map.end()) { 325 entry_map[data] = static_cast<int>(entries_.size()); 326 entries_.push_back(std::make_pair(data, std::vector<int>(1, offset))); 327 return true; 328 } 329 int index = existing->second; 330 entries_[index].second.push_back(offset); 331 return false; 332 } 333 334 // Constant Pool. 335 bool ConstPool::RecordEntry(intptr_t data, RelocInfo::Mode mode) { 336 DCHECK(mode != RelocInfo::COMMENT && mode != RelocInfo::CONST_POOL && 337 mode != RelocInfo::VENEER_POOL && 338 mode != RelocInfo::DEOPT_SCRIPT_OFFSET && 339 mode != RelocInfo::DEOPT_INLINING_ID && 340 mode != RelocInfo::DEOPT_REASON && mode != RelocInfo::DEOPT_ID); 341 342 bool write_reloc_info = true; 343 344 uint64_t raw_data = static_cast<uint64_t>(data); 345 int offset = assm_->pc_offset(); 346 if (IsEmpty()) { 347 first_use_ = offset; 348 } 349 350 if (RelocInfo::IsShareableRelocMode(mode)) { 351 write_reloc_info = AddSharedEntry(shared_entries_, raw_data, offset); 352 } else if (mode == RelocInfo::CODE_TARGET && raw_data != 0) { 353 // A zero data value is a placeholder and must not be shared. 354 write_reloc_info = AddSharedEntry(handle_to_index_map_, raw_data, offset); 355 } else { 356 entries_.push_back(std::make_pair(raw_data, std::vector<int>(1, offset))); 357 } 358 359 if (EntryCount() > Assembler::kApproxMaxPoolEntryCount) { 360 // Request constant pool emission after the next instruction. 361 assm_->SetNextConstPoolCheckIn(1); 362 } 363 364 return write_reloc_info; 365 } 366 367 368 int ConstPool::DistanceToFirstUse() { 369 DCHECK_GE(first_use_, 0); 370 return assm_->pc_offset() - first_use_; 371 } 372 373 374 int ConstPool::MaxPcOffset() { 375 // There are no pending entries in the pool so we can never get out of 376 // range. 377 if (IsEmpty()) return kMaxInt; 378 379 // Entries are not necessarily emitted in the order they are added so in the 380 // worst case the first constant pool use will be accessing the last entry. 381 return first_use_ + kMaxLoadLiteralRange - WorstCaseSize(); 382 } 383 384 385 int ConstPool::WorstCaseSize() { 386 if (IsEmpty()) return 0; 387 388 // Max size prologue: 389 // b over 390 // ldr xzr, #pool_size 391 // blr xzr 392 // nop 393 // All entries are 64-bit for now. 394 return 4 * kInstrSize + EntryCount() * kPointerSize; 395 } 396 397 398 int ConstPool::SizeIfEmittedAtCurrentPc(bool require_jump) { 399 if (IsEmpty()) return 0; 400 401 // Prologue is: 402 // b over ;; if require_jump 403 // ldr xzr, #pool_size 404 // blr xzr 405 // nop ;; if not 64-bit aligned 406 int prologue_size = require_jump ? kInstrSize : 0; 407 prologue_size += 2 * kInstrSize; 408 prologue_size += 409 IsAligned(assm_->pc_offset() + prologue_size, 8) ? 0 : kInstrSize; 410 411 // All entries are 64-bit for now. 412 return prologue_size + EntryCount() * kPointerSize; 413 } 414 415 416 void ConstPool::Emit(bool require_jump) { 417 DCHECK(!assm_->is_const_pool_blocked()); 418 // Prevent recursive pool emission and protect from veneer pools. 419 Assembler::BlockPoolsScope block_pools(assm_); 420 421 int size = SizeIfEmittedAtCurrentPc(require_jump); 422 Label size_check; 423 assm_->bind(&size_check); 424 425 assm_->RecordConstPool(size); 426 // Emit the constant pool. It is preceded by an optional branch if 427 // require_jump and a header which will: 428 // 1) Encode the size of the constant pool, for use by the disassembler. 429 // 2) Terminate the program, to try to prevent execution from accidentally 430 // flowing into the constant pool. 431 // 3) align the pool entries to 64-bit. 432 // The header is therefore made of up to three arm64 instructions: 433 // ldr xzr, #<size of the constant pool in 32-bit words> 434 // blr xzr 435 // nop 436 // 437 // If executed, the header will likely segfault and lr will point to the 438 // instruction following the offending blr. 439 // TODO(all): Make the alignment part less fragile. Currently code is 440 // allocated as a byte array so there are no guarantees the alignment will 441 // be preserved on compaction. Currently it works as allocation seems to be 442 // 64-bit aligned. 443 444 // Emit branch if required 445 Label after_pool; 446 if (require_jump) { 447 assm_->b(&after_pool); 448 } 449 450 // Emit the header. 451 assm_->RecordComment("[ Constant Pool"); 452 EmitMarker(); 453 EmitGuard(); 454 assm_->Align(8); 455 456 // Emit constant pool entries. 457 // TODO(all): currently each relocated constant is 64 bits, consider adding 458 // support for 32-bit entries. 459 EmitEntries(); 460 assm_->RecordComment("]"); 461 462 if (after_pool.is_linked()) { 463 assm_->bind(&after_pool); 464 } 465 466 DCHECK(assm_->SizeOfCodeGeneratedSince(&size_check) == 467 static_cast<unsigned>(size)); 468 } 469 470 471 void ConstPool::Clear() { 472 shared_entries_.clear(); 473 handle_to_index_map_.clear(); 474 entries_.clear(); 475 first_use_ = -1; 476 } 477 478 479 void ConstPool::EmitMarker() { 480 // A constant pool size is expressed in number of 32-bits words. 481 // Currently all entries are 64-bit. 482 // + 1 is for the crash guard. 483 // + 0/1 for alignment. 484 int word_count = EntryCount() * 2 + 1 + 485 (IsAligned(assm_->pc_offset(), 8) ? 0 : 1); 486 assm_->Emit(LDR_x_lit | 487 Assembler::ImmLLiteral(word_count) | 488 Assembler::Rt(xzr)); 489 } 490 491 492 MemOperand::PairResult MemOperand::AreConsistentForPair( 493 const MemOperand& operandA, 494 const MemOperand& operandB, 495 int access_size_log2) { 496 DCHECK_GE(access_size_log2, 0); 497 DCHECK_LE(access_size_log2, 3); 498 // Step one: check that they share the same base, that the mode is Offset 499 // and that the offset is a multiple of access size. 500 if (!operandA.base().Is(operandB.base()) || 501 (operandA.addrmode() != Offset) || 502 (operandB.addrmode() != Offset) || 503 ((operandA.offset() & ((1 << access_size_log2) - 1)) != 0)) { 504 return kNotPair; 505 } 506 // Step two: check that the offsets are contiguous and that the range 507 // is OK for ldp/stp. 508 if ((operandB.offset() == operandA.offset() + (1 << access_size_log2)) && 509 is_int7(operandA.offset() >> access_size_log2)) { 510 return kPairAB; 511 } 512 if ((operandA.offset() == operandB.offset() + (1 << access_size_log2)) && 513 is_int7(operandB.offset() >> access_size_log2)) { 514 return kPairBA; 515 } 516 return kNotPair; 517 } 518 519 520 void ConstPool::EmitGuard() { 521 #ifdef DEBUG 522 Instruction* instr = reinterpret_cast<Instruction*>(assm_->pc()); 523 DCHECK(instr->preceding()->IsLdrLiteralX() && 524 instr->preceding()->Rt() == xzr.code()); 525 #endif 526 assm_->EmitPoolGuard(); 527 } 528 529 530 void ConstPool::EmitEntries() { 531 DCHECK(IsAligned(assm_->pc_offset(), 8)); 532 533 // Emit entries. 534 for (const auto& entry : entries_) { 535 for (const auto& pc : entry.second) { 536 Instruction* instr = assm_->InstructionAt(pc); 537 538 // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0. 539 DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0); 540 instr->SetImmPCOffsetTarget(assm_->options(), assm_->pc()); 541 } 542 543 assm_->dc64(entry.first); 544 } 545 Clear(); 546 } 547 548 549 // Assembler 550 Assembler::Assembler(const AssemblerOptions& options, void* buffer, 551 int buffer_size) 552 : AssemblerBase(options, buffer, buffer_size), 553 constpool_(this), 554 unresolved_branches_() { 555 const_pool_blocked_nesting_ = 0; 556 veneer_pool_blocked_nesting_ = 0; 557 Reset(); 558 } 559 560 561 Assembler::~Assembler() { 562 DCHECK(constpool_.IsEmpty()); 563 DCHECK_EQ(const_pool_blocked_nesting_, 0); 564 DCHECK_EQ(veneer_pool_blocked_nesting_, 0); 565 } 566 567 568 void Assembler::Reset() { 569 #ifdef DEBUG 570 DCHECK((pc_ >= buffer_) && (pc_ < buffer_ + buffer_size_)); 571 DCHECK_EQ(const_pool_blocked_nesting_, 0); 572 DCHECK_EQ(veneer_pool_blocked_nesting_, 0); 573 DCHECK(unresolved_branches_.empty()); 574 memset(buffer_, 0, pc_ - buffer_); 575 #endif 576 pc_ = buffer_; 577 ReserveCodeTargetSpace(64); 578 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_); 579 constpool_.Clear(); 580 next_constant_pool_check_ = 0; 581 next_veneer_pool_check_ = kMaxInt; 582 no_const_pool_before_ = 0; 583 } 584 585 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) { 586 for (auto& request : heap_object_requests_) { 587 Address pc = reinterpret_cast<Address>(buffer_) + request.offset(); 588 switch (request.kind()) { 589 case HeapObjectRequest::kHeapNumber: { 590 Handle<HeapObject> object = 591 isolate->factory()->NewHeapNumber(request.heap_number(), TENURED); 592 set_target_address_at(pc, 0 /* unused */, object.address()); 593 break; 594 } 595 case HeapObjectRequest::kCodeStub: { 596 request.code_stub()->set_isolate(isolate); 597 Instruction* instr = reinterpret_cast<Instruction*>(pc); 598 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); 599 DCHECK_EQ(instr->ImmPCOffset() % kInstrSize, 0); 600 UpdateCodeTarget(instr->ImmPCOffset() >> kInstrSizeLog2, 601 request.code_stub()->GetCode()); 602 break; 603 } 604 } 605 } 606 } 607 608 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc) { 609 // Emit constant pool if necessary. 610 CheckConstPool(true, false); 611 DCHECK(constpool_.IsEmpty()); 612 613 AllocateAndInstallRequestedHeapObjects(isolate); 614 615 // Set up code descriptor. 616 if (desc) { 617 desc->buffer = reinterpret_cast<byte*>(buffer_); 618 desc->buffer_size = buffer_size_; 619 desc->instr_size = pc_offset(); 620 desc->reloc_size = 621 static_cast<int>((reinterpret_cast<byte*>(buffer_) + buffer_size_) - 622 reloc_info_writer.pos()); 623 desc->origin = this; 624 desc->constant_pool_size = 0; 625 desc->unwinding_info_size = 0; 626 desc->unwinding_info = nullptr; 627 } 628 } 629 630 631 void Assembler::Align(int m) { 632 DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m)); 633 while ((pc_offset() & (m - 1)) != 0) { 634 nop(); 635 } 636 } 637 638 639 void Assembler::CheckLabelLinkChain(Label const * label) { 640 #ifdef DEBUG 641 if (label->is_linked()) { 642 static const int kMaxLinksToCheck = 64; // Avoid O(n2) behaviour. 643 int links_checked = 0; 644 int64_t linkoffset = label->pos(); 645 bool end_of_chain = false; 646 while (!end_of_chain) { 647 if (++links_checked > kMaxLinksToCheck) break; 648 Instruction * link = InstructionAt(linkoffset); 649 int64_t linkpcoffset = link->ImmPCOffset(); 650 int64_t prevlinkoffset = linkoffset + linkpcoffset; 651 652 end_of_chain = (linkoffset == prevlinkoffset); 653 linkoffset = linkoffset + linkpcoffset; 654 } 655 } 656 #endif 657 } 658 659 660 void Assembler::RemoveBranchFromLabelLinkChain(Instruction* branch, 661 Label* label, 662 Instruction* label_veneer) { 663 DCHECK(label->is_linked()); 664 665 CheckLabelLinkChain(label); 666 667 Instruction* link = InstructionAt(label->pos()); 668 Instruction* prev_link = link; 669 Instruction* next_link; 670 bool end_of_chain = false; 671 672 while (link != branch && !end_of_chain) { 673 next_link = link->ImmPCOffsetTarget(); 674 end_of_chain = (link == next_link); 675 prev_link = link; 676 link = next_link; 677 } 678 679 DCHECK(branch == link); 680 next_link = branch->ImmPCOffsetTarget(); 681 682 if (branch == prev_link) { 683 // The branch is the first instruction in the chain. 684 if (branch == next_link) { 685 // It is also the last instruction in the chain, so it is the only branch 686 // currently referring to this label. 687 label->Unuse(); 688 } else { 689 label->link_to( 690 static_cast<int>(reinterpret_cast<byte*>(next_link) - buffer_)); 691 } 692 693 } else if (branch == next_link) { 694 // The branch is the last (but not also the first) instruction in the chain. 695 prev_link->SetImmPCOffsetTarget(options(), prev_link); 696 697 } else { 698 // The branch is in the middle of the chain. 699 if (prev_link->IsTargetInImmPCOffsetRange(next_link)) { 700 prev_link->SetImmPCOffsetTarget(options(), next_link); 701 } else if (label_veneer != nullptr) { 702 // Use the veneer for all previous links in the chain. 703 prev_link->SetImmPCOffsetTarget(options(), prev_link); 704 705 end_of_chain = false; 706 link = next_link; 707 while (!end_of_chain) { 708 next_link = link->ImmPCOffsetTarget(); 709 end_of_chain = (link == next_link); 710 link->SetImmPCOffsetTarget(options(), label_veneer); 711 link = next_link; 712 } 713 } else { 714 // The assert below will fire. 715 // Some other work could be attempted to fix up the chain, but it would be 716 // rather complicated. If we crash here, we may want to consider using an 717 // other mechanism than a chain of branches. 718 // 719 // Note that this situation currently should not happen, as we always call 720 // this function with a veneer to the target label. 721 // However this could happen with a MacroAssembler in the following state: 722 // [previous code] 723 // B(label); 724 // [20KB code] 725 // Tbz(label); // First tbz. Pointing to unconditional branch. 726 // [20KB code] 727 // Tbz(label); // Second tbz. Pointing to the first tbz. 728 // [more code] 729 // and this function is called to remove the first tbz from the label link 730 // chain. Since tbz has a range of +-32KB, the second tbz cannot point to 731 // the unconditional branch. 732 CHECK(prev_link->IsTargetInImmPCOffsetRange(next_link)); 733 UNREACHABLE(); 734 } 735 } 736 737 CheckLabelLinkChain(label); 738 } 739 740 741 void Assembler::bind(Label* label) { 742 // Bind label to the address at pc_. All instructions (most likely branches) 743 // that are linked to this label will be updated to point to the newly-bound 744 // label. 745 746 DCHECK(!label->is_near_linked()); 747 DCHECK(!label->is_bound()); 748 749 DeleteUnresolvedBranchInfoForLabel(label); 750 751 // If the label is linked, the link chain looks something like this: 752 // 753 // |--I----I-------I-------L 754 // |---------------------->| pc_offset 755 // |-------------->| linkoffset = label->pos() 756 // |<------| link->ImmPCOffset() 757 // |------>| prevlinkoffset = linkoffset + link->ImmPCOffset() 758 // 759 // On each iteration, the last link is updated and then removed from the 760 // chain until only one remains. At that point, the label is bound. 761 // 762 // If the label is not linked, no preparation is required before binding. 763 while (label->is_linked()) { 764 int linkoffset = label->pos(); 765 Instruction* link = InstructionAt(linkoffset); 766 int prevlinkoffset = linkoffset + static_cast<int>(link->ImmPCOffset()); 767 768 CheckLabelLinkChain(label); 769 770 DCHECK_GE(linkoffset, 0); 771 DCHECK(linkoffset < pc_offset()); 772 DCHECK((linkoffset > prevlinkoffset) || 773 (linkoffset - prevlinkoffset == kStartOfLabelLinkChain)); 774 DCHECK_GE(prevlinkoffset, 0); 775 776 // Update the link to point to the label. 777 if (link->IsUnresolvedInternalReference()) { 778 // Internal references do not get patched to an instruction but directly 779 // to an address. 780 internal_reference_positions_.push_back(linkoffset); 781 PatchingAssembler patcher(options(), reinterpret_cast<byte*>(link), 2); 782 patcher.dc64(reinterpret_cast<uintptr_t>(pc_)); 783 } else { 784 link->SetImmPCOffsetTarget(options(), 785 reinterpret_cast<Instruction*>(pc_)); 786 } 787 788 // Link the label to the previous link in the chain. 789 if (linkoffset - prevlinkoffset == kStartOfLabelLinkChain) { 790 // We hit kStartOfLabelLinkChain, so the chain is fully processed. 791 label->Unuse(); 792 } else { 793 // Update the label for the next iteration. 794 label->link_to(prevlinkoffset); 795 } 796 } 797 label->bind_to(pc_offset()); 798 799 DCHECK(label->is_bound()); 800 DCHECK(!label->is_linked()); 801 } 802 803 804 int Assembler::LinkAndGetByteOffsetTo(Label* label) { 805 DCHECK_EQ(sizeof(*pc_), 1); 806 CheckLabelLinkChain(label); 807 808 int offset; 809 if (label->is_bound()) { 810 // The label is bound, so it does not need to be updated. Referring 811 // instructions must link directly to the label as they will not be 812 // updated. 813 // 814 // In this case, label->pos() returns the offset of the label from the 815 // start of the buffer. 816 // 817 // Note that offset can be zero for self-referential instructions. (This 818 // could be useful for ADR, for example.) 819 offset = label->pos() - pc_offset(); 820 DCHECK_LE(offset, 0); 821 } else { 822 if (label->is_linked()) { 823 // The label is linked, so the referring instruction should be added onto 824 // the end of the label's link chain. 825 // 826 // In this case, label->pos() returns the offset of the last linked 827 // instruction from the start of the buffer. 828 offset = label->pos() - pc_offset(); 829 DCHECK_NE(offset, kStartOfLabelLinkChain); 830 // Note that the offset here needs to be PC-relative only so that the 831 // first instruction in a buffer can link to an unbound label. Otherwise, 832 // the offset would be 0 for this case, and 0 is reserved for 833 // kStartOfLabelLinkChain. 834 } else { 835 // The label is unused, so it now becomes linked and the referring 836 // instruction is at the start of the new link chain. 837 offset = kStartOfLabelLinkChain; 838 } 839 // The instruction at pc is now the last link in the label's chain. 840 label->link_to(pc_offset()); 841 } 842 843 return offset; 844 } 845 846 847 void Assembler::DeleteUnresolvedBranchInfoForLabelTraverse(Label* label) { 848 DCHECK(label->is_linked()); 849 CheckLabelLinkChain(label); 850 851 int link_offset = label->pos(); 852 int link_pcoffset; 853 bool end_of_chain = false; 854 855 while (!end_of_chain) { 856 Instruction * link = InstructionAt(link_offset); 857 link_pcoffset = static_cast<int>(link->ImmPCOffset()); 858 859 // ADR instructions are not handled by veneers. 860 if (link->IsImmBranch()) { 861 int max_reachable_pc = 862 static_cast<int>(InstructionOffset(link) + 863 Instruction::ImmBranchRange(link->BranchType())); 864 typedef std::multimap<int, FarBranchInfo>::iterator unresolved_info_it; 865 std::pair<unresolved_info_it, unresolved_info_it> range; 866 range = unresolved_branches_.equal_range(max_reachable_pc); 867 unresolved_info_it it; 868 for (it = range.first; it != range.second; ++it) { 869 if (it->second.pc_offset_ == link_offset) { 870 unresolved_branches_.erase(it); 871 break; 872 } 873 } 874 } 875 876 end_of_chain = (link_pcoffset == 0); 877 link_offset = link_offset + link_pcoffset; 878 } 879 } 880 881 882 void Assembler::DeleteUnresolvedBranchInfoForLabel(Label* label) { 883 if (unresolved_branches_.empty()) { 884 DCHECK_EQ(next_veneer_pool_check_, kMaxInt); 885 return; 886 } 887 888 if (label->is_linked()) { 889 // Branches to this label will be resolved when the label is bound, normally 890 // just after all the associated info has been deleted. 891 DeleteUnresolvedBranchInfoForLabelTraverse(label); 892 } 893 if (unresolved_branches_.empty()) { 894 next_veneer_pool_check_ = kMaxInt; 895 } else { 896 next_veneer_pool_check_ = 897 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 898 } 899 } 900 901 902 void Assembler::StartBlockConstPool() { 903 if (const_pool_blocked_nesting_++ == 0) { 904 // Prevent constant pool checks happening by setting the next check to 905 // the biggest possible offset. 906 next_constant_pool_check_ = kMaxInt; 907 } 908 } 909 910 911 void Assembler::EndBlockConstPool() { 912 if (--const_pool_blocked_nesting_ == 0) { 913 // Check the constant pool hasn't been blocked for too long. 914 DCHECK(pc_offset() < constpool_.MaxPcOffset()); 915 // Two cases: 916 // * no_const_pool_before_ >= next_constant_pool_check_ and the emission is 917 // still blocked 918 // * no_const_pool_before_ < next_constant_pool_check_ and the next emit 919 // will trigger a check. 920 next_constant_pool_check_ = no_const_pool_before_; 921 } 922 } 923 924 925 bool Assembler::is_const_pool_blocked() const { 926 return (const_pool_blocked_nesting_ > 0) || 927 (pc_offset() < no_const_pool_before_); 928 } 929 930 931 bool Assembler::IsConstantPoolAt(Instruction* instr) { 932 // The constant pool marker is made of two instructions. These instructions 933 // will never be emitted by the JIT, so checking for the first one is enough: 934 // 0: ldr xzr, #<size of pool> 935 bool result = instr->IsLdrLiteralX() && (instr->Rt() == kZeroRegCode); 936 937 // It is still worth asserting the marker is complete. 938 // 4: blr xzr 939 DCHECK(!result || (instr->following()->IsBranchAndLinkToRegister() && 940 instr->following()->Rn() == kZeroRegCode)); 941 942 return result; 943 } 944 945 946 int Assembler::ConstantPoolSizeAt(Instruction* instr) { 947 #ifdef USE_SIMULATOR 948 // Assembler::debug() embeds constants directly into the instruction stream. 949 // Although this is not a genuine constant pool, treat it like one to avoid 950 // disassembling the constants. 951 if ((instr->Mask(ExceptionMask) == HLT) && 952 (instr->ImmException() == kImmExceptionIsDebug)) { 953 const char* message = 954 reinterpret_cast<const char*>( 955 instr->InstructionAtOffset(kDebugMessageOffset)); 956 int size = static_cast<int>(kDebugMessageOffset + strlen(message) + 1); 957 return RoundUp(size, kInstrSize) / kInstrSize; 958 } 959 // Same for printf support, see MacroAssembler::CallPrintf(). 960 if ((instr->Mask(ExceptionMask) == HLT) && 961 (instr->ImmException() == kImmExceptionIsPrintf)) { 962 return kPrintfLength / kInstrSize; 963 } 964 #endif 965 if (IsConstantPoolAt(instr)) { 966 return instr->ImmLLiteral(); 967 } else { 968 return -1; 969 } 970 } 971 972 973 void Assembler::EmitPoolGuard() { 974 // We must generate only one instruction as this is used in scopes that 975 // control the size of the code generated. 976 Emit(BLR | Rn(xzr)); 977 } 978 979 980 void Assembler::StartBlockVeneerPool() { 981 ++veneer_pool_blocked_nesting_; 982 } 983 984 985 void Assembler::EndBlockVeneerPool() { 986 if (--veneer_pool_blocked_nesting_ == 0) { 987 // Check the veneer pool hasn't been blocked for too long. 988 DCHECK(unresolved_branches_.empty() || 989 (pc_offset() < unresolved_branches_first_limit())); 990 } 991 } 992 993 994 void Assembler::br(const Register& xn) { 995 DCHECK(xn.Is64Bits()); 996 Emit(BR | Rn(xn)); 997 } 998 999 1000 void Assembler::blr(const Register& xn) { 1001 DCHECK(xn.Is64Bits()); 1002 // The pattern 'blr xzr' is used as a guard to detect when execution falls 1003 // through the constant pool. It should not be emitted. 1004 DCHECK(!xn.Is(xzr)); 1005 Emit(BLR | Rn(xn)); 1006 } 1007 1008 1009 void Assembler::ret(const Register& xn) { 1010 DCHECK(xn.Is64Bits()); 1011 Emit(RET | Rn(xn)); 1012 } 1013 1014 1015 void Assembler::b(int imm26) { 1016 Emit(B | ImmUncondBranch(imm26)); 1017 } 1018 1019 1020 void Assembler::b(Label* label) { 1021 b(LinkAndGetInstructionOffsetTo(label)); 1022 } 1023 1024 1025 void Assembler::b(int imm19, Condition cond) { 1026 Emit(B_cond | ImmCondBranch(imm19) | cond); 1027 } 1028 1029 1030 void Assembler::b(Label* label, Condition cond) { 1031 b(LinkAndGetInstructionOffsetTo(label), cond); 1032 } 1033 1034 1035 void Assembler::bl(int imm26) { 1036 Emit(BL | ImmUncondBranch(imm26)); 1037 } 1038 1039 1040 void Assembler::bl(Label* label) { 1041 bl(LinkAndGetInstructionOffsetTo(label)); 1042 } 1043 1044 1045 void Assembler::cbz(const Register& rt, 1046 int imm19) { 1047 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt)); 1048 } 1049 1050 1051 void Assembler::cbz(const Register& rt, 1052 Label* label) { 1053 cbz(rt, LinkAndGetInstructionOffsetTo(label)); 1054 } 1055 1056 1057 void Assembler::cbnz(const Register& rt, 1058 int imm19) { 1059 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt)); 1060 } 1061 1062 1063 void Assembler::cbnz(const Register& rt, 1064 Label* label) { 1065 cbnz(rt, LinkAndGetInstructionOffsetTo(label)); 1066 } 1067 1068 1069 void Assembler::tbz(const Register& rt, 1070 unsigned bit_pos, 1071 int imm14) { 1072 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits))); 1073 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 1074 } 1075 1076 1077 void Assembler::tbz(const Register& rt, 1078 unsigned bit_pos, 1079 Label* label) { 1080 tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 1081 } 1082 1083 1084 void Assembler::tbnz(const Register& rt, 1085 unsigned bit_pos, 1086 int imm14) { 1087 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits))); 1088 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 1089 } 1090 1091 1092 void Assembler::tbnz(const Register& rt, 1093 unsigned bit_pos, 1094 Label* label) { 1095 tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 1096 } 1097 1098 1099 void Assembler::adr(const Register& rd, int imm21) { 1100 DCHECK(rd.Is64Bits()); 1101 Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd)); 1102 } 1103 1104 1105 void Assembler::adr(const Register& rd, Label* label) { 1106 adr(rd, LinkAndGetByteOffsetTo(label)); 1107 } 1108 1109 1110 void Assembler::add(const Register& rd, 1111 const Register& rn, 1112 const Operand& operand) { 1113 AddSub(rd, rn, operand, LeaveFlags, ADD); 1114 } 1115 1116 1117 void Assembler::adds(const Register& rd, 1118 const Register& rn, 1119 const Operand& operand) { 1120 AddSub(rd, rn, operand, SetFlags, ADD); 1121 } 1122 1123 1124 void Assembler::cmn(const Register& rn, 1125 const Operand& operand) { 1126 Register zr = AppropriateZeroRegFor(rn); 1127 adds(zr, rn, operand); 1128 } 1129 1130 1131 void Assembler::sub(const Register& rd, 1132 const Register& rn, 1133 const Operand& operand) { 1134 AddSub(rd, rn, operand, LeaveFlags, SUB); 1135 } 1136 1137 1138 void Assembler::subs(const Register& rd, 1139 const Register& rn, 1140 const Operand& operand) { 1141 AddSub(rd, rn, operand, SetFlags, SUB); 1142 } 1143 1144 1145 void Assembler::cmp(const Register& rn, const Operand& operand) { 1146 Register zr = AppropriateZeroRegFor(rn); 1147 subs(zr, rn, operand); 1148 } 1149 1150 1151 void Assembler::neg(const Register& rd, const Operand& operand) { 1152 Register zr = AppropriateZeroRegFor(rd); 1153 sub(rd, zr, operand); 1154 } 1155 1156 1157 void Assembler::negs(const Register& rd, const Operand& operand) { 1158 Register zr = AppropriateZeroRegFor(rd); 1159 subs(rd, zr, operand); 1160 } 1161 1162 1163 void Assembler::adc(const Register& rd, 1164 const Register& rn, 1165 const Operand& operand) { 1166 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC); 1167 } 1168 1169 1170 void Assembler::adcs(const Register& rd, 1171 const Register& rn, 1172 const Operand& operand) { 1173 AddSubWithCarry(rd, rn, operand, SetFlags, ADC); 1174 } 1175 1176 1177 void Assembler::sbc(const Register& rd, 1178 const Register& rn, 1179 const Operand& operand) { 1180 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC); 1181 } 1182 1183 1184 void Assembler::sbcs(const Register& rd, 1185 const Register& rn, 1186 const Operand& operand) { 1187 AddSubWithCarry(rd, rn, operand, SetFlags, SBC); 1188 } 1189 1190 1191 void Assembler::ngc(const Register& rd, const Operand& operand) { 1192 Register zr = AppropriateZeroRegFor(rd); 1193 sbc(rd, zr, operand); 1194 } 1195 1196 1197 void Assembler::ngcs(const Register& rd, const Operand& operand) { 1198 Register zr = AppropriateZeroRegFor(rd); 1199 sbcs(rd, zr, operand); 1200 } 1201 1202 1203 // Logical instructions. 1204 void Assembler::and_(const Register& rd, 1205 const Register& rn, 1206 const Operand& operand) { 1207 Logical(rd, rn, operand, AND); 1208 } 1209 1210 1211 void Assembler::ands(const Register& rd, 1212 const Register& rn, 1213 const Operand& operand) { 1214 Logical(rd, rn, operand, ANDS); 1215 } 1216 1217 1218 void Assembler::tst(const Register& rn, 1219 const Operand& operand) { 1220 ands(AppropriateZeroRegFor(rn), rn, operand); 1221 } 1222 1223 1224 void Assembler::bic(const Register& rd, 1225 const Register& rn, 1226 const Operand& operand) { 1227 Logical(rd, rn, operand, BIC); 1228 } 1229 1230 1231 void Assembler::bics(const Register& rd, 1232 const Register& rn, 1233 const Operand& operand) { 1234 Logical(rd, rn, operand, BICS); 1235 } 1236 1237 1238 void Assembler::orr(const Register& rd, 1239 const Register& rn, 1240 const Operand& operand) { 1241 Logical(rd, rn, operand, ORR); 1242 } 1243 1244 1245 void Assembler::orn(const Register& rd, 1246 const Register& rn, 1247 const Operand& operand) { 1248 Logical(rd, rn, operand, ORN); 1249 } 1250 1251 1252 void Assembler::eor(const Register& rd, 1253 const Register& rn, 1254 const Operand& operand) { 1255 Logical(rd, rn, operand, EOR); 1256 } 1257 1258 1259 void Assembler::eon(const Register& rd, 1260 const Register& rn, 1261 const Operand& operand) { 1262 Logical(rd, rn, operand, EON); 1263 } 1264 1265 1266 void Assembler::lslv(const Register& rd, 1267 const Register& rn, 1268 const Register& rm) { 1269 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1270 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1271 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd)); 1272 } 1273 1274 1275 void Assembler::lsrv(const Register& rd, 1276 const Register& rn, 1277 const Register& rm) { 1278 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1279 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1280 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd)); 1281 } 1282 1283 1284 void Assembler::asrv(const Register& rd, 1285 const Register& rn, 1286 const Register& rm) { 1287 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1288 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1289 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd)); 1290 } 1291 1292 1293 void Assembler::rorv(const Register& rd, 1294 const Register& rn, 1295 const Register& rm) { 1296 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1297 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1298 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd)); 1299 } 1300 1301 1302 // Bitfield operations. 1303 void Assembler::bfm(const Register& rd, const Register& rn, int immr, 1304 int imms) { 1305 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1306 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1307 Emit(SF(rd) | BFM | N | 1308 ImmR(immr, rd.SizeInBits()) | 1309 ImmS(imms, rn.SizeInBits()) | 1310 Rn(rn) | Rd(rd)); 1311 } 1312 1313 1314 void Assembler::sbfm(const Register& rd, const Register& rn, int immr, 1315 int imms) { 1316 DCHECK(rd.Is64Bits() || rn.Is32Bits()); 1317 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1318 Emit(SF(rd) | SBFM | N | 1319 ImmR(immr, rd.SizeInBits()) | 1320 ImmS(imms, rn.SizeInBits()) | 1321 Rn(rn) | Rd(rd)); 1322 } 1323 1324 1325 void Assembler::ubfm(const Register& rd, const Register& rn, int immr, 1326 int imms) { 1327 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1328 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1329 Emit(SF(rd) | UBFM | N | 1330 ImmR(immr, rd.SizeInBits()) | 1331 ImmS(imms, rn.SizeInBits()) | 1332 Rn(rn) | Rd(rd)); 1333 } 1334 1335 1336 void Assembler::extr(const Register& rd, const Register& rn, const Register& rm, 1337 int lsb) { 1338 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1339 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1340 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1341 Emit(SF(rd) | EXTR | N | Rm(rm) | 1342 ImmS(lsb, rn.SizeInBits()) | Rn(rn) | Rd(rd)); 1343 } 1344 1345 1346 void Assembler::csel(const Register& rd, 1347 const Register& rn, 1348 const Register& rm, 1349 Condition cond) { 1350 ConditionalSelect(rd, rn, rm, cond, CSEL); 1351 } 1352 1353 1354 void Assembler::csinc(const Register& rd, 1355 const Register& rn, 1356 const Register& rm, 1357 Condition cond) { 1358 ConditionalSelect(rd, rn, rm, cond, CSINC); 1359 } 1360 1361 1362 void Assembler::csinv(const Register& rd, 1363 const Register& rn, 1364 const Register& rm, 1365 Condition cond) { 1366 ConditionalSelect(rd, rn, rm, cond, CSINV); 1367 } 1368 1369 1370 void Assembler::csneg(const Register& rd, 1371 const Register& rn, 1372 const Register& rm, 1373 Condition cond) { 1374 ConditionalSelect(rd, rn, rm, cond, CSNEG); 1375 } 1376 1377 1378 void Assembler::cset(const Register &rd, Condition cond) { 1379 DCHECK((cond != al) && (cond != nv)); 1380 Register zr = AppropriateZeroRegFor(rd); 1381 csinc(rd, zr, zr, NegateCondition(cond)); 1382 } 1383 1384 1385 void Assembler::csetm(const Register &rd, Condition cond) { 1386 DCHECK((cond != al) && (cond != nv)); 1387 Register zr = AppropriateZeroRegFor(rd); 1388 csinv(rd, zr, zr, NegateCondition(cond)); 1389 } 1390 1391 1392 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) { 1393 DCHECK((cond != al) && (cond != nv)); 1394 csinc(rd, rn, rn, NegateCondition(cond)); 1395 } 1396 1397 1398 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) { 1399 DCHECK((cond != al) && (cond != nv)); 1400 csinv(rd, rn, rn, NegateCondition(cond)); 1401 } 1402 1403 1404 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) { 1405 DCHECK((cond != al) && (cond != nv)); 1406 csneg(rd, rn, rn, NegateCondition(cond)); 1407 } 1408 1409 1410 void Assembler::ConditionalSelect(const Register& rd, 1411 const Register& rn, 1412 const Register& rm, 1413 Condition cond, 1414 ConditionalSelectOp op) { 1415 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1416 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1417 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd)); 1418 } 1419 1420 1421 void Assembler::ccmn(const Register& rn, 1422 const Operand& operand, 1423 StatusFlags nzcv, 1424 Condition cond) { 1425 ConditionalCompare(rn, operand, nzcv, cond, CCMN); 1426 } 1427 1428 1429 void Assembler::ccmp(const Register& rn, 1430 const Operand& operand, 1431 StatusFlags nzcv, 1432 Condition cond) { 1433 ConditionalCompare(rn, operand, nzcv, cond, CCMP); 1434 } 1435 1436 1437 void Assembler::DataProcessing3Source(const Register& rd, 1438 const Register& rn, 1439 const Register& rm, 1440 const Register& ra, 1441 DataProcessing3SourceOp op) { 1442 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd)); 1443 } 1444 1445 1446 void Assembler::mul(const Register& rd, 1447 const Register& rn, 1448 const Register& rm) { 1449 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1450 Register zr = AppropriateZeroRegFor(rn); 1451 DataProcessing3Source(rd, rn, rm, zr, MADD); 1452 } 1453 1454 1455 void Assembler::madd(const Register& rd, 1456 const Register& rn, 1457 const Register& rm, 1458 const Register& ra) { 1459 DCHECK(AreSameSizeAndType(rd, rn, rm, ra)); 1460 DataProcessing3Source(rd, rn, rm, ra, MADD); 1461 } 1462 1463 1464 void Assembler::mneg(const Register& rd, 1465 const Register& rn, 1466 const Register& rm) { 1467 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1468 Register zr = AppropriateZeroRegFor(rn); 1469 DataProcessing3Source(rd, rn, rm, zr, MSUB); 1470 } 1471 1472 1473 void Assembler::msub(const Register& rd, 1474 const Register& rn, 1475 const Register& rm, 1476 const Register& ra) { 1477 DCHECK(AreSameSizeAndType(rd, rn, rm, ra)); 1478 DataProcessing3Source(rd, rn, rm, ra, MSUB); 1479 } 1480 1481 1482 void Assembler::smaddl(const Register& rd, 1483 const Register& rn, 1484 const Register& rm, 1485 const Register& ra) { 1486 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1487 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1488 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x); 1489 } 1490 1491 1492 void Assembler::smsubl(const Register& rd, 1493 const Register& rn, 1494 const Register& rm, 1495 const Register& ra) { 1496 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1497 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1498 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x); 1499 } 1500 1501 1502 void Assembler::umaddl(const Register& rd, 1503 const Register& rn, 1504 const Register& rm, 1505 const Register& ra) { 1506 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1507 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1508 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x); 1509 } 1510 1511 1512 void Assembler::umsubl(const Register& rd, 1513 const Register& rn, 1514 const Register& rm, 1515 const Register& ra) { 1516 DCHECK(rd.Is64Bits() && ra.Is64Bits()); 1517 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1518 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x); 1519 } 1520 1521 1522 void Assembler::smull(const Register& rd, 1523 const Register& rn, 1524 const Register& rm) { 1525 DCHECK(rd.Is64Bits()); 1526 DCHECK(rn.Is32Bits() && rm.Is32Bits()); 1527 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x); 1528 } 1529 1530 1531 void Assembler::smulh(const Register& rd, 1532 const Register& rn, 1533 const Register& rm) { 1534 DCHECK(AreSameSizeAndType(rd, rn, rm)); 1535 DataProcessing3Source(rd, rn, rm, xzr, SMULH_x); 1536 } 1537 1538 1539 void Assembler::sdiv(const Register& rd, 1540 const Register& rn, 1541 const Register& rm) { 1542 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1543 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1544 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1545 } 1546 1547 1548 void Assembler::udiv(const Register& rd, 1549 const Register& rn, 1550 const Register& rm) { 1551 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 1552 DCHECK(rd.SizeInBits() == rm.SizeInBits()); 1553 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1554 } 1555 1556 1557 void Assembler::rbit(const Register& rd, 1558 const Register& rn) { 1559 DataProcessing1Source(rd, rn, RBIT); 1560 } 1561 1562 1563 void Assembler::rev16(const Register& rd, 1564 const Register& rn) { 1565 DataProcessing1Source(rd, rn, REV16); 1566 } 1567 1568 1569 void Assembler::rev32(const Register& rd, 1570 const Register& rn) { 1571 DCHECK(rd.Is64Bits()); 1572 DataProcessing1Source(rd, rn, REV); 1573 } 1574 1575 1576 void Assembler::rev(const Register& rd, 1577 const Register& rn) { 1578 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w); 1579 } 1580 1581 1582 void Assembler::clz(const Register& rd, 1583 const Register& rn) { 1584 DataProcessing1Source(rd, rn, CLZ); 1585 } 1586 1587 1588 void Assembler::cls(const Register& rd, 1589 const Register& rn) { 1590 DataProcessing1Source(rd, rn, CLS); 1591 } 1592 1593 1594 void Assembler::ldp(const CPURegister& rt, 1595 const CPURegister& rt2, 1596 const MemOperand& src) { 1597 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2)); 1598 } 1599 1600 1601 void Assembler::stp(const CPURegister& rt, 1602 const CPURegister& rt2, 1603 const MemOperand& dst) { 1604 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2)); 1605 } 1606 1607 1608 void Assembler::ldpsw(const Register& rt, 1609 const Register& rt2, 1610 const MemOperand& src) { 1611 DCHECK(rt.Is64Bits()); 1612 LoadStorePair(rt, rt2, src, LDPSW_x); 1613 } 1614 1615 1616 void Assembler::LoadStorePair(const CPURegister& rt, 1617 const CPURegister& rt2, 1618 const MemOperand& addr, 1619 LoadStorePairOp op) { 1620 // 'rt' and 'rt2' can only be aliased for stores. 1621 DCHECK(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2)); 1622 DCHECK(AreSameSizeAndType(rt, rt2)); 1623 DCHECK(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op))); 1624 int offset = static_cast<int>(addr.offset()); 1625 1626 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | 1627 ImmLSPair(offset, CalcLSPairDataSize(op)); 1628 1629 Instr addrmodeop; 1630 if (addr.IsImmediateOffset()) { 1631 addrmodeop = LoadStorePairOffsetFixed; 1632 } else { 1633 // Pre-index and post-index modes. 1634 DCHECK(!rt.Is(addr.base())); 1635 DCHECK(!rt2.Is(addr.base())); 1636 DCHECK_NE(addr.offset(), 0); 1637 if (addr.IsPreIndex()) { 1638 addrmodeop = LoadStorePairPreIndexFixed; 1639 } else { 1640 DCHECK(addr.IsPostIndex()); 1641 addrmodeop = LoadStorePairPostIndexFixed; 1642 } 1643 } 1644 Emit(addrmodeop | memop); 1645 } 1646 1647 1648 // Memory instructions. 1649 void Assembler::ldrb(const Register& rt, const MemOperand& src) { 1650 LoadStore(rt, src, LDRB_w); 1651 } 1652 1653 1654 void Assembler::strb(const Register& rt, const MemOperand& dst) { 1655 LoadStore(rt, dst, STRB_w); 1656 } 1657 1658 1659 void Assembler::ldrsb(const Register& rt, const MemOperand& src) { 1660 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w); 1661 } 1662 1663 1664 void Assembler::ldrh(const Register& rt, const MemOperand& src) { 1665 LoadStore(rt, src, LDRH_w); 1666 } 1667 1668 1669 void Assembler::strh(const Register& rt, const MemOperand& dst) { 1670 LoadStore(rt, dst, STRH_w); 1671 } 1672 1673 1674 void Assembler::ldrsh(const Register& rt, const MemOperand& src) { 1675 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w); 1676 } 1677 1678 1679 void Assembler::ldr(const CPURegister& rt, const MemOperand& src) { 1680 LoadStore(rt, src, LoadOpFor(rt)); 1681 } 1682 1683 1684 void Assembler::str(const CPURegister& rt, const MemOperand& src) { 1685 LoadStore(rt, src, StoreOpFor(rt)); 1686 } 1687 1688 1689 void Assembler::ldrsw(const Register& rt, const MemOperand& src) { 1690 DCHECK(rt.Is64Bits()); 1691 LoadStore(rt, src, LDRSW_x); 1692 } 1693 1694 1695 void Assembler::ldr_pcrel(const CPURegister& rt, int imm19) { 1696 // The pattern 'ldr xzr, #offset' is used to indicate the beginning of a 1697 // constant pool. It should not be emitted. 1698 DCHECK(!rt.IsZero()); 1699 Emit(LoadLiteralOpFor(rt) | ImmLLiteral(imm19) | Rt(rt)); 1700 } 1701 1702 Operand Operand::EmbeddedNumber(double number) { 1703 int32_t smi; 1704 if (DoubleToSmiInteger(number, &smi)) { 1705 return Operand(Immediate(Smi::FromInt(smi))); 1706 } 1707 Operand result(0, RelocInfo::EMBEDDED_OBJECT); 1708 result.heap_object_request_.emplace(number); 1709 DCHECK(result.IsHeapObjectRequest()); 1710 return result; 1711 } 1712 1713 Operand Operand::EmbeddedCode(CodeStub* stub) { 1714 Operand result(0, RelocInfo::CODE_TARGET); 1715 result.heap_object_request_.emplace(stub); 1716 DCHECK(result.IsHeapObjectRequest()); 1717 return result; 1718 } 1719 1720 void Assembler::ldr(const CPURegister& rt, const Operand& operand) { 1721 if (operand.IsHeapObjectRequest()) { 1722 RequestHeapObject(operand.heap_object_request()); 1723 ldr(rt, operand.immediate_for_heap_object_request()); 1724 } else { 1725 ldr(rt, operand.immediate()); 1726 } 1727 } 1728 1729 void Assembler::ldr(const CPURegister& rt, const Immediate& imm) { 1730 // Currently we only support 64-bit literals. 1731 DCHECK(rt.Is64Bits()); 1732 1733 RecordRelocInfo(imm.rmode(), imm.value()); 1734 BlockConstPoolFor(1); 1735 // The load will be patched when the constpool is emitted, patching code 1736 // expect a load literal with offset 0. 1737 ldr_pcrel(rt, 0); 1738 } 1739 1740 void Assembler::ldar(const Register& rt, const Register& rn) { 1741 DCHECK(rn.Is64Bits()); 1742 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x; 1743 Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1744 } 1745 1746 void Assembler::ldaxr(const Register& rt, const Register& rn) { 1747 DCHECK(rn.Is64Bits()); 1748 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x; 1749 Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1750 } 1751 1752 void Assembler::stlr(const Register& rt, const Register& rn) { 1753 DCHECK(rn.Is64Bits()); 1754 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x; 1755 Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1756 } 1757 1758 void Assembler::stlxr(const Register& rs, const Register& rt, 1759 const Register& rn) { 1760 DCHECK(rn.Is64Bits()); 1761 DCHECK(!rs.Is(rt) && !rs.Is(rn)); 1762 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x; 1763 Emit(op | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1764 } 1765 1766 void Assembler::ldarb(const Register& rt, const Register& rn) { 1767 DCHECK(rt.Is32Bits()); 1768 DCHECK(rn.Is64Bits()); 1769 Emit(LDAR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1770 } 1771 1772 void Assembler::ldaxrb(const Register& rt, const Register& rn) { 1773 DCHECK(rt.Is32Bits()); 1774 DCHECK(rn.Is64Bits()); 1775 Emit(LDAXR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1776 } 1777 1778 void Assembler::stlrb(const Register& rt, const Register& rn) { 1779 DCHECK(rt.Is32Bits()); 1780 DCHECK(rn.Is64Bits()); 1781 Emit(STLR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1782 } 1783 1784 void Assembler::stlxrb(const Register& rs, const Register& rt, 1785 const Register& rn) { 1786 DCHECK(rs.Is32Bits()); 1787 DCHECK(rt.Is32Bits()); 1788 DCHECK(rn.Is64Bits()); 1789 DCHECK(!rs.Is(rt) && !rs.Is(rn)); 1790 Emit(STLXR_b | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1791 } 1792 1793 void Assembler::ldarh(const Register& rt, const Register& rn) { 1794 DCHECK(rt.Is32Bits()); 1795 DCHECK(rn.Is64Bits()); 1796 Emit(LDAR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1797 } 1798 1799 void Assembler::ldaxrh(const Register& rt, const Register& rn) { 1800 DCHECK(rt.Is32Bits()); 1801 DCHECK(rn.Is64Bits()); 1802 Emit(LDAXR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1803 } 1804 1805 void Assembler::stlrh(const Register& rt, const Register& rn) { 1806 DCHECK(rt.Is32Bits()); 1807 DCHECK(rn.Is64Bits()); 1808 Emit(STLR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1809 } 1810 1811 void Assembler::stlxrh(const Register& rs, const Register& rt, 1812 const Register& rn) { 1813 DCHECK(rs.Is32Bits()); 1814 DCHECK(rt.Is32Bits()); 1815 DCHECK(rn.Is64Bits()); 1816 DCHECK(!rs.Is(rt) && !rs.Is(rn)); 1817 Emit(STLXR_h | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt)); 1818 } 1819 1820 void Assembler::NEON3DifferentL(const VRegister& vd, const VRegister& vn, 1821 const VRegister& vm, NEON3DifferentOp vop) { 1822 DCHECK(AreSameFormat(vn, vm)); 1823 DCHECK((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) || 1824 (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) || 1825 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) || 1826 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D())); 1827 Instr format, op = vop; 1828 if (vd.IsScalar()) { 1829 op |= NEON_Q | NEONScalar; 1830 format = SFormat(vn); 1831 } else { 1832 format = VFormat(vn); 1833 } 1834 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd)); 1835 } 1836 1837 void Assembler::NEON3DifferentW(const VRegister& vd, const VRegister& vn, 1838 const VRegister& vm, NEON3DifferentOp vop) { 1839 DCHECK(AreSameFormat(vd, vn)); 1840 DCHECK((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) || 1841 (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) || 1842 (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D())); 1843 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd)); 1844 } 1845 1846 void Assembler::NEON3DifferentHN(const VRegister& vd, const VRegister& vn, 1847 const VRegister& vm, NEON3DifferentOp vop) { 1848 DCHECK(AreSameFormat(vm, vn)); 1849 DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) || 1850 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) || 1851 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D())); 1852 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd)); 1853 } 1854 1855 #define NEON_3DIFF_LONG_LIST(V) \ 1856 V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \ 1857 V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \ 1858 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \ 1859 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \ 1860 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \ 1861 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \ 1862 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \ 1863 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \ 1864 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \ 1865 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \ 1866 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \ 1867 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \ 1868 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \ 1869 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \ 1870 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \ 1871 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \ 1872 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \ 1873 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \ 1874 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \ 1875 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \ 1876 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \ 1877 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \ 1878 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \ 1879 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \ 1880 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \ 1881 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \ 1882 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \ 1883 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \ 1884 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \ 1885 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \ 1886 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \ 1887 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \ 1888 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \ 1889 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \ 1890 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \ 1891 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) 1892 1893 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 1894 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 1895 const VRegister& vm) { \ 1896 DCHECK(AS); \ 1897 NEON3DifferentL(vd, vn, vm, OP); \ 1898 } 1899 NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC) 1900 #undef DEFINE_ASM_FUNC 1901 1902 #define NEON_3DIFF_HN_LIST(V) \ 1903 V(addhn, NEON_ADDHN, vd.IsD()) \ 1904 V(addhn2, NEON_ADDHN2, vd.IsQ()) \ 1905 V(raddhn, NEON_RADDHN, vd.IsD()) \ 1906 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \ 1907 V(subhn, NEON_SUBHN, vd.IsD()) \ 1908 V(subhn2, NEON_SUBHN2, vd.IsQ()) \ 1909 V(rsubhn, NEON_RSUBHN, vd.IsD()) \ 1910 V(rsubhn2, NEON_RSUBHN2, vd.IsQ()) 1911 1912 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 1913 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 1914 const VRegister& vm) { \ 1915 DCHECK(AS); \ 1916 NEON3DifferentHN(vd, vn, vm, OP); \ 1917 } 1918 NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC) 1919 #undef DEFINE_ASM_FUNC 1920 1921 void Assembler::NEONPerm(const VRegister& vd, const VRegister& vn, 1922 const VRegister& vm, NEONPermOp op) { 1923 DCHECK(AreSameFormat(vd, vn, vm)); 1924 DCHECK(!vd.Is1D()); 1925 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd)); 1926 } 1927 1928 void Assembler::trn1(const VRegister& vd, const VRegister& vn, 1929 const VRegister& vm) { 1930 NEONPerm(vd, vn, vm, NEON_TRN1); 1931 } 1932 1933 void Assembler::trn2(const VRegister& vd, const VRegister& vn, 1934 const VRegister& vm) { 1935 NEONPerm(vd, vn, vm, NEON_TRN2); 1936 } 1937 1938 void Assembler::uzp1(const VRegister& vd, const VRegister& vn, 1939 const VRegister& vm) { 1940 NEONPerm(vd, vn, vm, NEON_UZP1); 1941 } 1942 1943 void Assembler::uzp2(const VRegister& vd, const VRegister& vn, 1944 const VRegister& vm) { 1945 NEONPerm(vd, vn, vm, NEON_UZP2); 1946 } 1947 1948 void Assembler::zip1(const VRegister& vd, const VRegister& vn, 1949 const VRegister& vm) { 1950 NEONPerm(vd, vn, vm, NEON_ZIP1); 1951 } 1952 1953 void Assembler::zip2(const VRegister& vd, const VRegister& vn, 1954 const VRegister& vm) { 1955 NEONPerm(vd, vn, vm, NEON_ZIP2); 1956 } 1957 1958 void Assembler::NEONShiftImmediate(const VRegister& vd, const VRegister& vn, 1959 NEONShiftImmediateOp op, int immh_immb) { 1960 DCHECK(AreSameFormat(vd, vn)); 1961 Instr q, scalar; 1962 if (vn.IsScalar()) { 1963 q = NEON_Q; 1964 scalar = NEONScalar; 1965 } else { 1966 q = vd.IsD() ? 0 : NEON_Q; 1967 scalar = 0; 1968 } 1969 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd)); 1970 } 1971 1972 void Assembler::NEONShiftLeftImmediate(const VRegister& vd, const VRegister& vn, 1973 int shift, NEONShiftImmediateOp op) { 1974 int laneSizeInBits = vn.LaneSizeInBits(); 1975 DCHECK((shift >= 0) && (shift < laneSizeInBits)); 1976 NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16); 1977 } 1978 1979 void Assembler::NEONShiftRightImmediate(const VRegister& vd, 1980 const VRegister& vn, int shift, 1981 NEONShiftImmediateOp op) { 1982 int laneSizeInBits = vn.LaneSizeInBits(); 1983 DCHECK((shift >= 1) && (shift <= laneSizeInBits)); 1984 NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16); 1985 } 1986 1987 void Assembler::NEONShiftImmediateL(const VRegister& vd, const VRegister& vn, 1988 int shift, NEONShiftImmediateOp op) { 1989 int laneSizeInBits = vn.LaneSizeInBits(); 1990 DCHECK((shift >= 0) && (shift < laneSizeInBits)); 1991 int immh_immb = (laneSizeInBits + shift) << 16; 1992 1993 DCHECK((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) || 1994 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) || 1995 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D())); 1996 Instr q; 1997 q = vn.IsD() ? 0 : NEON_Q; 1998 Emit(q | op | immh_immb | Rn(vn) | Rd(vd)); 1999 } 2000 2001 void Assembler::NEONShiftImmediateN(const VRegister& vd, const VRegister& vn, 2002 int shift, NEONShiftImmediateOp op) { 2003 Instr q, scalar; 2004 int laneSizeInBits = vd.LaneSizeInBits(); 2005 DCHECK((shift >= 1) && (shift <= laneSizeInBits)); 2006 int immh_immb = (2 * laneSizeInBits - shift) << 16; 2007 2008 if (vn.IsScalar()) { 2009 DCHECK((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) || 2010 (vd.Is1S() && vn.Is1D())); 2011 q = NEON_Q; 2012 scalar = NEONScalar; 2013 } else { 2014 DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) || 2015 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) || 2016 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D())); 2017 scalar = 0; 2018 q = vd.IsD() ? 0 : NEON_Q; 2019 } 2020 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd)); 2021 } 2022 2023 void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) { 2024 DCHECK(vd.IsVector() || vd.Is1D()); 2025 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL); 2026 } 2027 2028 void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) { 2029 DCHECK(vd.IsVector() || vd.Is1D()); 2030 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI); 2031 } 2032 2033 void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) { 2034 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm); 2035 } 2036 2037 void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) { 2038 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU); 2039 } 2040 2041 void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) { 2042 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm); 2043 } 2044 2045 void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) { 2046 DCHECK(vn.IsD()); 2047 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL); 2048 } 2049 2050 void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) { 2051 DCHECK(vn.IsQ()); 2052 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL); 2053 } 2054 2055 void Assembler::sxtl(const VRegister& vd, const VRegister& vn) { 2056 sshll(vd, vn, 0); 2057 } 2058 2059 void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) { 2060 sshll2(vd, vn, 0); 2061 } 2062 2063 void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) { 2064 DCHECK(vn.IsD()); 2065 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL); 2066 } 2067 2068 void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) { 2069 DCHECK(vn.IsQ()); 2070 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL); 2071 } 2072 2073 void Assembler::uxtl(const VRegister& vd, const VRegister& vn) { 2074 ushll(vd, vn, 0); 2075 } 2076 2077 void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) { 2078 ushll2(vd, vn, 0); 2079 } 2080 2081 void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) { 2082 DCHECK(vd.IsVector() || vd.Is1D()); 2083 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI); 2084 } 2085 2086 void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) { 2087 DCHECK(vd.IsVector() || vd.Is1D()); 2088 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR); 2089 } 2090 2091 void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) { 2092 DCHECK(vd.IsVector() || vd.Is1D()); 2093 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR); 2094 } 2095 2096 void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) { 2097 DCHECK(vd.IsVector() || vd.Is1D()); 2098 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR); 2099 } 2100 2101 void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) { 2102 DCHECK(vd.IsVector() || vd.Is1D()); 2103 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR); 2104 } 2105 2106 void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) { 2107 DCHECK(vd.IsVector() || vd.Is1D()); 2108 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA); 2109 } 2110 2111 void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) { 2112 DCHECK(vd.IsVector() || vd.Is1D()); 2113 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA); 2114 } 2115 2116 void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) { 2117 DCHECK(vd.IsVector() || vd.Is1D()); 2118 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA); 2119 } 2120 2121 void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) { 2122 DCHECK(vd.IsVector() || vd.Is1D()); 2123 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA); 2124 } 2125 2126 void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) { 2127 DCHECK(vn.IsVector() && vd.IsD()); 2128 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN); 2129 } 2130 2131 void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) { 2132 DCHECK(vn.IsVector() && vd.IsQ()); 2133 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN); 2134 } 2135 2136 void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) { 2137 DCHECK(vn.IsVector() && vd.IsD()); 2138 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN); 2139 } 2140 2141 void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) { 2142 DCHECK(vn.IsVector() && vd.IsQ()); 2143 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN); 2144 } 2145 2146 void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) { 2147 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 2148 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN); 2149 } 2150 2151 void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) { 2152 DCHECK(vn.IsVector() && vd.IsQ()); 2153 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN); 2154 } 2155 2156 void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) { 2157 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 2158 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN); 2159 } 2160 2161 void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) { 2162 DCHECK(vn.IsVector() && vd.IsQ()); 2163 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN); 2164 } 2165 2166 void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) { 2167 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 2168 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN); 2169 } 2170 2171 void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) { 2172 DCHECK(vn.IsVector() && vd.IsQ()); 2173 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN); 2174 } 2175 2176 void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) { 2177 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 2178 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN); 2179 } 2180 2181 void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) { 2182 DCHECK(vn.IsVector() && vd.IsQ()); 2183 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN); 2184 } 2185 2186 void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) { 2187 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 2188 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN); 2189 } 2190 2191 void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) { 2192 DCHECK(vn.IsVector() && vd.IsQ()); 2193 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN); 2194 } 2195 2196 void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) { 2197 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 2198 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN); 2199 } 2200 2201 void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) { 2202 DCHECK(vn.IsVector() && vd.IsQ()); 2203 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN); 2204 } 2205 2206 void Assembler::uaddw(const VRegister& vd, const VRegister& vn, 2207 const VRegister& vm) { 2208 DCHECK(vm.IsD()); 2209 NEON3DifferentW(vd, vn, vm, NEON_UADDW); 2210 } 2211 2212 void Assembler::uaddw2(const VRegister& vd, const VRegister& vn, 2213 const VRegister& vm) { 2214 DCHECK(vm.IsQ()); 2215 NEON3DifferentW(vd, vn, vm, NEON_UADDW2); 2216 } 2217 2218 void Assembler::saddw(const VRegister& vd, const VRegister& vn, 2219 const VRegister& vm) { 2220 DCHECK(vm.IsD()); 2221 NEON3DifferentW(vd, vn, vm, NEON_SADDW); 2222 } 2223 2224 void Assembler::saddw2(const VRegister& vd, const VRegister& vn, 2225 const VRegister& vm) { 2226 DCHECK(vm.IsQ()); 2227 NEON3DifferentW(vd, vn, vm, NEON_SADDW2); 2228 } 2229 2230 void Assembler::usubw(const VRegister& vd, const VRegister& vn, 2231 const VRegister& vm) { 2232 DCHECK(vm.IsD()); 2233 NEON3DifferentW(vd, vn, vm, NEON_USUBW); 2234 } 2235 2236 void Assembler::usubw2(const VRegister& vd, const VRegister& vn, 2237 const VRegister& vm) { 2238 DCHECK(vm.IsQ()); 2239 NEON3DifferentW(vd, vn, vm, NEON_USUBW2); 2240 } 2241 2242 void Assembler::ssubw(const VRegister& vd, const VRegister& vn, 2243 const VRegister& vm) { 2244 DCHECK(vm.IsD()); 2245 NEON3DifferentW(vd, vn, vm, NEON_SSUBW); 2246 } 2247 2248 void Assembler::ssubw2(const VRegister& vd, const VRegister& vn, 2249 const VRegister& vm) { 2250 DCHECK(vm.IsQ()); 2251 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2); 2252 } 2253 2254 void Assembler::mov(const Register& rd, const Register& rm) { 2255 // Moves involving the stack pointer are encoded as add immediate with 2256 // second operand of zero. Otherwise, orr with first operand zr is 2257 // used. 2258 if (rd.IsSP() || rm.IsSP()) { 2259 add(rd, rm, 0); 2260 } else { 2261 orr(rd, AppropriateZeroRegFor(rd), rm); 2262 } 2263 } 2264 2265 void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) { 2266 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the 2267 // number of lanes, and T is b, h, s or d. 2268 int lane_size = vd.LaneSizeInBytes(); 2269 NEONFormatField format; 2270 switch (lane_size) { 2271 case 1: 2272 format = NEON_16B; 2273 DCHECK(rn.IsW()); 2274 break; 2275 case 2: 2276 format = NEON_8H; 2277 DCHECK(rn.IsW()); 2278 break; 2279 case 4: 2280 format = NEON_4S; 2281 DCHECK(rn.IsW()); 2282 break; 2283 default: 2284 DCHECK_EQ(lane_size, 8); 2285 DCHECK(rn.IsX()); 2286 format = NEON_2D; 2287 break; 2288 } 2289 2290 DCHECK((0 <= vd_index) && 2291 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 2292 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd)); 2293 } 2294 2295 void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) { 2296 DCHECK_GE(vn.SizeInBytes(), 4); 2297 umov(rd, vn, vn_index); 2298 } 2299 2300 void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) { 2301 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the 2302 // number of lanes, and T is b, h, s. 2303 int lane_size = vn.LaneSizeInBytes(); 2304 NEONFormatField format; 2305 Instr q = 0; 2306 switch (lane_size) { 2307 case 1: 2308 format = NEON_16B; 2309 break; 2310 case 2: 2311 format = NEON_8H; 2312 break; 2313 default: 2314 DCHECK_EQ(lane_size, 4); 2315 DCHECK(rd.IsX()); 2316 format = NEON_4S; 2317 break; 2318 } 2319 q = rd.IsW() ? 0 : NEON_Q; 2320 DCHECK((0 <= vn_index) && 2321 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 2322 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd)); 2323 } 2324 2325 void Assembler::cls(const VRegister& vd, const VRegister& vn) { 2326 DCHECK(AreSameFormat(vd, vn)); 2327 DCHECK(!vd.Is1D() && !vd.Is2D()); 2328 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd)); 2329 } 2330 2331 void Assembler::clz(const VRegister& vd, const VRegister& vn) { 2332 DCHECK(AreSameFormat(vd, vn)); 2333 DCHECK(!vd.Is1D() && !vd.Is2D()); 2334 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd)); 2335 } 2336 2337 void Assembler::cnt(const VRegister& vd, const VRegister& vn) { 2338 DCHECK(AreSameFormat(vd, vn)); 2339 DCHECK(vd.Is8B() || vd.Is16B()); 2340 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd)); 2341 } 2342 2343 void Assembler::rev16(const VRegister& vd, const VRegister& vn) { 2344 DCHECK(AreSameFormat(vd, vn)); 2345 DCHECK(vd.Is8B() || vd.Is16B()); 2346 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd)); 2347 } 2348 2349 void Assembler::rev32(const VRegister& vd, const VRegister& vn) { 2350 DCHECK(AreSameFormat(vd, vn)); 2351 DCHECK(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H()); 2352 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd)); 2353 } 2354 2355 void Assembler::rev64(const VRegister& vd, const VRegister& vn) { 2356 DCHECK(AreSameFormat(vd, vn)); 2357 DCHECK(!vd.Is1D() && !vd.Is2D()); 2358 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd)); 2359 } 2360 2361 void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) { 2362 DCHECK(AreSameFormat(vd, vn)); 2363 DCHECK(vd.Is2S() || vd.Is4S()); 2364 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd)); 2365 } 2366 2367 void Assembler::urecpe(const VRegister& vd, const VRegister& vn) { 2368 DCHECK(AreSameFormat(vd, vn)); 2369 DCHECK(vd.Is2S() || vd.Is4S()); 2370 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd)); 2371 } 2372 2373 void Assembler::NEONAddlp(const VRegister& vd, const VRegister& vn, 2374 NEON2RegMiscOp op) { 2375 DCHECK((op == NEON_SADDLP) || (op == NEON_UADDLP) || (op == NEON_SADALP) || 2376 (op == NEON_UADALP)); 2377 2378 DCHECK((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) || 2379 (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) || 2380 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D())); 2381 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd)); 2382 } 2383 2384 void Assembler::saddlp(const VRegister& vd, const VRegister& vn) { 2385 NEONAddlp(vd, vn, NEON_SADDLP); 2386 } 2387 2388 void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) { 2389 NEONAddlp(vd, vn, NEON_UADDLP); 2390 } 2391 2392 void Assembler::sadalp(const VRegister& vd, const VRegister& vn) { 2393 NEONAddlp(vd, vn, NEON_SADALP); 2394 } 2395 2396 void Assembler::uadalp(const VRegister& vd, const VRegister& vn) { 2397 NEONAddlp(vd, vn, NEON_UADALP); 2398 } 2399 2400 void Assembler::NEONAcrossLanesL(const VRegister& vd, const VRegister& vn, 2401 NEONAcrossLanesOp op) { 2402 DCHECK((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) || 2403 (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) || 2404 (vn.Is4S() && vd.Is1D())); 2405 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd)); 2406 } 2407 2408 void Assembler::saddlv(const VRegister& vd, const VRegister& vn) { 2409 NEONAcrossLanesL(vd, vn, NEON_SADDLV); 2410 } 2411 2412 void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) { 2413 NEONAcrossLanesL(vd, vn, NEON_UADDLV); 2414 } 2415 2416 void Assembler::NEONAcrossLanes(const VRegister& vd, const VRegister& vn, 2417 NEONAcrossLanesOp op) { 2418 DCHECK((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) || 2419 (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) || 2420 (vn.Is4S() && vd.Is1S())); 2421 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) { 2422 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd)); 2423 } else { 2424 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd)); 2425 } 2426 } 2427 2428 #define NEON_ACROSSLANES_LIST(V) \ 2429 V(fmaxv, NEON_FMAXV, vd.Is1S()) \ 2430 V(fminv, NEON_FMINV, vd.Is1S()) \ 2431 V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \ 2432 V(fminnmv, NEON_FMINNMV, vd.Is1S()) \ 2433 V(addv, NEON_ADDV, true) \ 2434 V(smaxv, NEON_SMAXV, true) \ 2435 V(sminv, NEON_SMINV, true) \ 2436 V(umaxv, NEON_UMAXV, true) \ 2437 V(uminv, NEON_UMINV, true) 2438 2439 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 2440 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \ 2441 DCHECK(AS); \ 2442 NEONAcrossLanes(vd, vn, OP); \ 2443 } 2444 NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC) 2445 #undef DEFINE_ASM_FUNC 2446 2447 void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) { 2448 ins(vd, vd_index, rn); 2449 } 2450 2451 void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) { 2452 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the 2453 // number of lanes, and T is b, h, s or d. 2454 int lane_size = vn.LaneSizeInBytes(); 2455 NEONFormatField format; 2456 Instr q = 0; 2457 switch (lane_size) { 2458 case 1: 2459 format = NEON_16B; 2460 DCHECK(rd.IsW()); 2461 break; 2462 case 2: 2463 format = NEON_8H; 2464 DCHECK(rd.IsW()); 2465 break; 2466 case 4: 2467 format = NEON_4S; 2468 DCHECK(rd.IsW()); 2469 break; 2470 default: 2471 DCHECK_EQ(lane_size, 8); 2472 DCHECK(rd.IsX()); 2473 format = NEON_2D; 2474 q = NEON_Q; 2475 break; 2476 } 2477 2478 DCHECK((0 <= vn_index) && 2479 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 2480 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd)); 2481 } 2482 2483 void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) { 2484 DCHECK(vd.IsScalar()); 2485 dup(vd, vn, vn_index); 2486 } 2487 2488 void Assembler::dup(const VRegister& vd, const Register& rn) { 2489 DCHECK(!vd.Is1D()); 2490 DCHECK_EQ(vd.Is2D(), rn.IsX()); 2491 Instr q = vd.IsD() ? 0 : NEON_Q; 2492 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd)); 2493 } 2494 2495 void Assembler::ins(const VRegister& vd, int vd_index, const VRegister& vn, 2496 int vn_index) { 2497 DCHECK(AreSameFormat(vd, vn)); 2498 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the 2499 // number of lanes, and T is b, h, s or d. 2500 int lane_size = vd.LaneSizeInBytes(); 2501 NEONFormatField format; 2502 switch (lane_size) { 2503 case 1: 2504 format = NEON_16B; 2505 break; 2506 case 2: 2507 format = NEON_8H; 2508 break; 2509 case 4: 2510 format = NEON_4S; 2511 break; 2512 default: 2513 DCHECK_EQ(lane_size, 8); 2514 format = NEON_2D; 2515 break; 2516 } 2517 2518 DCHECK((0 <= vd_index) && 2519 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 2520 DCHECK((0 <= vn_index) && 2521 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 2522 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) | 2523 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd)); 2524 } 2525 2526 void Assembler::NEONTable(const VRegister& vd, const VRegister& vn, 2527 const VRegister& vm, NEONTableOp op) { 2528 DCHECK(vd.Is16B() || vd.Is8B()); 2529 DCHECK(vn.Is16B()); 2530 DCHECK(AreSameFormat(vd, vm)); 2531 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd)); 2532 } 2533 2534 void Assembler::tbl(const VRegister& vd, const VRegister& vn, 2535 const VRegister& vm) { 2536 NEONTable(vd, vn, vm, NEON_TBL_1v); 2537 } 2538 2539 void Assembler::tbl(const VRegister& vd, const VRegister& vn, 2540 const VRegister& vn2, const VRegister& vm) { 2541 USE(vn2); 2542 DCHECK(AreSameFormat(vn, vn2)); 2543 DCHECK(AreConsecutive(vn, vn2)); 2544 NEONTable(vd, vn, vm, NEON_TBL_2v); 2545 } 2546 2547 void Assembler::tbl(const VRegister& vd, const VRegister& vn, 2548 const VRegister& vn2, const VRegister& vn3, 2549 const VRegister& vm) { 2550 USE(vn2); 2551 USE(vn3); 2552 DCHECK(AreSameFormat(vn, vn2, vn3)); 2553 DCHECK(AreConsecutive(vn, vn2, vn3)); 2554 NEONTable(vd, vn, vm, NEON_TBL_3v); 2555 } 2556 2557 void Assembler::tbl(const VRegister& vd, const VRegister& vn, 2558 const VRegister& vn2, const VRegister& vn3, 2559 const VRegister& vn4, const VRegister& vm) { 2560 USE(vn2); 2561 USE(vn3); 2562 USE(vn4); 2563 DCHECK(AreSameFormat(vn, vn2, vn3, vn4)); 2564 DCHECK(AreConsecutive(vn, vn2, vn3, vn4)); 2565 NEONTable(vd, vn, vm, NEON_TBL_4v); 2566 } 2567 2568 void Assembler::tbx(const VRegister& vd, const VRegister& vn, 2569 const VRegister& vm) { 2570 NEONTable(vd, vn, vm, NEON_TBX_1v); 2571 } 2572 2573 void Assembler::tbx(const VRegister& vd, const VRegister& vn, 2574 const VRegister& vn2, const VRegister& vm) { 2575 USE(vn2); 2576 DCHECK(AreSameFormat(vn, vn2)); 2577 DCHECK(AreConsecutive(vn, vn2)); 2578 NEONTable(vd, vn, vm, NEON_TBX_2v); 2579 } 2580 2581 void Assembler::tbx(const VRegister& vd, const VRegister& vn, 2582 const VRegister& vn2, const VRegister& vn3, 2583 const VRegister& vm) { 2584 USE(vn2); 2585 USE(vn3); 2586 DCHECK(AreSameFormat(vn, vn2, vn3)); 2587 DCHECK(AreConsecutive(vn, vn2, vn3)); 2588 NEONTable(vd, vn, vm, NEON_TBX_3v); 2589 } 2590 2591 void Assembler::tbx(const VRegister& vd, const VRegister& vn, 2592 const VRegister& vn2, const VRegister& vn3, 2593 const VRegister& vn4, const VRegister& vm) { 2594 USE(vn2); 2595 USE(vn3); 2596 USE(vn4); 2597 DCHECK(AreSameFormat(vn, vn2, vn3, vn4)); 2598 DCHECK(AreConsecutive(vn, vn2, vn3, vn4)); 2599 NEONTable(vd, vn, vm, NEON_TBX_4v); 2600 } 2601 2602 void Assembler::mov(const VRegister& vd, int vd_index, const VRegister& vn, 2603 int vn_index) { 2604 ins(vd, vd_index, vn, vn_index); 2605 } 2606 2607 void Assembler::mvn(const Register& rd, const Operand& operand) { 2608 orn(rd, AppropriateZeroRegFor(rd), operand); 2609 } 2610 2611 void Assembler::mrs(const Register& rt, SystemRegister sysreg) { 2612 DCHECK(rt.Is64Bits()); 2613 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt)); 2614 } 2615 2616 void Assembler::msr(SystemRegister sysreg, const Register& rt) { 2617 DCHECK(rt.Is64Bits()); 2618 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg)); 2619 } 2620 2621 void Assembler::hint(SystemHint code) { Emit(HINT | ImmHint(code) | Rt(xzr)); } 2622 2623 // NEON structure loads and stores. 2624 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) { 2625 Instr addr_field = RnSP(addr.base()); 2626 2627 if (addr.IsPostIndex()) { 2628 static_assert(NEONLoadStoreMultiStructPostIndex == 2629 static_cast<NEONLoadStoreMultiStructPostIndexOp>( 2630 NEONLoadStoreSingleStructPostIndex), 2631 "Opcodes must match for NEON post index memop."); 2632 2633 addr_field |= NEONLoadStoreMultiStructPostIndex; 2634 if (addr.offset() == 0) { 2635 addr_field |= RmNot31(addr.regoffset()); 2636 } else { 2637 // The immediate post index addressing mode is indicated by rm = 31. 2638 // The immediate is implied by the number of vector registers used. 2639 addr_field |= (0x1F << Rm_offset); 2640 } 2641 } else { 2642 DCHECK(addr.IsImmediateOffset() && (addr.offset() == 0)); 2643 } 2644 return addr_field; 2645 } 2646 2647 void Assembler::LoadStoreStructVerify(const VRegister& vt, 2648 const MemOperand& addr, Instr op) { 2649 #ifdef DEBUG 2650 // Assert that addressing mode is either offset (with immediate 0), post 2651 // index by immediate of the size of the register list, or post index by a 2652 // value in a core register. 2653 if (addr.IsImmediateOffset()) { 2654 DCHECK_EQ(addr.offset(), 0); 2655 } else { 2656 int offset = vt.SizeInBytes(); 2657 switch (op) { 2658 case NEON_LD1_1v: 2659 case NEON_ST1_1v: 2660 offset *= 1; 2661 break; 2662 case NEONLoadStoreSingleStructLoad1: 2663 case NEONLoadStoreSingleStructStore1: 2664 case NEON_LD1R: 2665 offset = (offset / vt.LaneCount()) * 1; 2666 break; 2667 2668 case NEON_LD1_2v: 2669 case NEON_ST1_2v: 2670 case NEON_LD2: 2671 case NEON_ST2: 2672 offset *= 2; 2673 break; 2674 case NEONLoadStoreSingleStructLoad2: 2675 case NEONLoadStoreSingleStructStore2: 2676 case NEON_LD2R: 2677 offset = (offset / vt.LaneCount()) * 2; 2678 break; 2679 2680 case NEON_LD1_3v: 2681 case NEON_ST1_3v: 2682 case NEON_LD3: 2683 case NEON_ST3: 2684 offset *= 3; 2685 break; 2686 case NEONLoadStoreSingleStructLoad3: 2687 case NEONLoadStoreSingleStructStore3: 2688 case NEON_LD3R: 2689 offset = (offset / vt.LaneCount()) * 3; 2690 break; 2691 2692 case NEON_LD1_4v: 2693 case NEON_ST1_4v: 2694 case NEON_LD4: 2695 case NEON_ST4: 2696 offset *= 4; 2697 break; 2698 case NEONLoadStoreSingleStructLoad4: 2699 case NEONLoadStoreSingleStructStore4: 2700 case NEON_LD4R: 2701 offset = (offset / vt.LaneCount()) * 4; 2702 break; 2703 default: 2704 UNREACHABLE(); 2705 } 2706 DCHECK(!addr.regoffset().Is(NoReg) || addr.offset() == offset); 2707 } 2708 #else 2709 USE(vt); 2710 USE(addr); 2711 USE(op); 2712 #endif 2713 } 2714 2715 void Assembler::LoadStoreStruct(const VRegister& vt, const MemOperand& addr, 2716 NEONLoadStoreMultiStructOp op) { 2717 LoadStoreStructVerify(vt, addr, op); 2718 DCHECK(vt.IsVector() || vt.Is1D()); 2719 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt)); 2720 } 2721 2722 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt, 2723 const MemOperand& addr, 2724 NEONLoadStoreSingleStructOp op) { 2725 LoadStoreStructVerify(vt, addr, op); 2726 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt)); 2727 } 2728 2729 void Assembler::ld1(const VRegister& vt, const MemOperand& src) { 2730 LoadStoreStruct(vt, src, NEON_LD1_1v); 2731 } 2732 2733 void Assembler::ld1(const VRegister& vt, const VRegister& vt2, 2734 const MemOperand& src) { 2735 USE(vt2); 2736 DCHECK(AreSameFormat(vt, vt2)); 2737 DCHECK(AreConsecutive(vt, vt2)); 2738 LoadStoreStruct(vt, src, NEON_LD1_2v); 2739 } 2740 2741 void Assembler::ld1(const VRegister& vt, const VRegister& vt2, 2742 const VRegister& vt3, const MemOperand& src) { 2743 USE(vt2); 2744 USE(vt3); 2745 DCHECK(AreSameFormat(vt, vt2, vt3)); 2746 DCHECK(AreConsecutive(vt, vt2, vt3)); 2747 LoadStoreStruct(vt, src, NEON_LD1_3v); 2748 } 2749 2750 void Assembler::ld1(const VRegister& vt, const VRegister& vt2, 2751 const VRegister& vt3, const VRegister& vt4, 2752 const MemOperand& src) { 2753 USE(vt2); 2754 USE(vt3); 2755 USE(vt4); 2756 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2757 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2758 LoadStoreStruct(vt, src, NEON_LD1_4v); 2759 } 2760 2761 void Assembler::ld2(const VRegister& vt, const VRegister& vt2, 2762 const MemOperand& src) { 2763 USE(vt2); 2764 DCHECK(AreSameFormat(vt, vt2)); 2765 DCHECK(AreConsecutive(vt, vt2)); 2766 LoadStoreStruct(vt, src, NEON_LD2); 2767 } 2768 2769 void Assembler::ld2(const VRegister& vt, const VRegister& vt2, int lane, 2770 const MemOperand& src) { 2771 USE(vt2); 2772 DCHECK(AreSameFormat(vt, vt2)); 2773 DCHECK(AreConsecutive(vt, vt2)); 2774 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2); 2775 } 2776 2777 void Assembler::ld2r(const VRegister& vt, const VRegister& vt2, 2778 const MemOperand& src) { 2779 USE(vt2); 2780 DCHECK(AreSameFormat(vt, vt2)); 2781 DCHECK(AreConsecutive(vt, vt2)); 2782 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R); 2783 } 2784 2785 void Assembler::ld3(const VRegister& vt, const VRegister& vt2, 2786 const VRegister& vt3, const MemOperand& src) { 2787 USE(vt2); 2788 USE(vt3); 2789 DCHECK(AreSameFormat(vt, vt2, vt3)); 2790 DCHECK(AreConsecutive(vt, vt2, vt3)); 2791 LoadStoreStruct(vt, src, NEON_LD3); 2792 } 2793 2794 void Assembler::ld3(const VRegister& vt, const VRegister& vt2, 2795 const VRegister& vt3, int lane, const MemOperand& src) { 2796 USE(vt2); 2797 USE(vt3); 2798 DCHECK(AreSameFormat(vt, vt2, vt3)); 2799 DCHECK(AreConsecutive(vt, vt2, vt3)); 2800 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3); 2801 } 2802 2803 void Assembler::ld3r(const VRegister& vt, const VRegister& vt2, 2804 const VRegister& vt3, const MemOperand& src) { 2805 USE(vt2); 2806 USE(vt3); 2807 DCHECK(AreSameFormat(vt, vt2, vt3)); 2808 DCHECK(AreConsecutive(vt, vt2, vt3)); 2809 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R); 2810 } 2811 2812 void Assembler::ld4(const VRegister& vt, const VRegister& vt2, 2813 const VRegister& vt3, const VRegister& vt4, 2814 const MemOperand& src) { 2815 USE(vt2); 2816 USE(vt3); 2817 USE(vt4); 2818 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2819 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2820 LoadStoreStruct(vt, src, NEON_LD4); 2821 } 2822 2823 void Assembler::ld4(const VRegister& vt, const VRegister& vt2, 2824 const VRegister& vt3, const VRegister& vt4, int lane, 2825 const MemOperand& src) { 2826 USE(vt2); 2827 USE(vt3); 2828 USE(vt4); 2829 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2830 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2831 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4); 2832 } 2833 2834 void Assembler::ld4r(const VRegister& vt, const VRegister& vt2, 2835 const VRegister& vt3, const VRegister& vt4, 2836 const MemOperand& src) { 2837 USE(vt2); 2838 USE(vt3); 2839 USE(vt4); 2840 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2841 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2842 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R); 2843 } 2844 2845 void Assembler::st1(const VRegister& vt, const MemOperand& src) { 2846 LoadStoreStruct(vt, src, NEON_ST1_1v); 2847 } 2848 2849 void Assembler::st1(const VRegister& vt, const VRegister& vt2, 2850 const MemOperand& src) { 2851 USE(vt2); 2852 DCHECK(AreSameFormat(vt, vt2)); 2853 DCHECK(AreConsecutive(vt, vt2)); 2854 LoadStoreStruct(vt, src, NEON_ST1_2v); 2855 } 2856 2857 void Assembler::st1(const VRegister& vt, const VRegister& vt2, 2858 const VRegister& vt3, const MemOperand& src) { 2859 USE(vt2); 2860 USE(vt3); 2861 DCHECK(AreSameFormat(vt, vt2, vt3)); 2862 DCHECK(AreConsecutive(vt, vt2, vt3)); 2863 LoadStoreStruct(vt, src, NEON_ST1_3v); 2864 } 2865 2866 void Assembler::st1(const VRegister& vt, const VRegister& vt2, 2867 const VRegister& vt3, const VRegister& vt4, 2868 const MemOperand& src) { 2869 USE(vt2); 2870 USE(vt3); 2871 USE(vt4); 2872 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2873 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2874 LoadStoreStruct(vt, src, NEON_ST1_4v); 2875 } 2876 2877 void Assembler::st2(const VRegister& vt, const VRegister& vt2, 2878 const MemOperand& dst) { 2879 USE(vt2); 2880 DCHECK(AreSameFormat(vt, vt2)); 2881 DCHECK(AreConsecutive(vt, vt2)); 2882 LoadStoreStruct(vt, dst, NEON_ST2); 2883 } 2884 2885 void Assembler::st2(const VRegister& vt, const VRegister& vt2, int lane, 2886 const MemOperand& dst) { 2887 USE(vt2); 2888 DCHECK(AreSameFormat(vt, vt2)); 2889 DCHECK(AreConsecutive(vt, vt2)); 2890 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2); 2891 } 2892 2893 void Assembler::st3(const VRegister& vt, const VRegister& vt2, 2894 const VRegister& vt3, const MemOperand& dst) { 2895 USE(vt2); 2896 USE(vt3); 2897 DCHECK(AreSameFormat(vt, vt2, vt3)); 2898 DCHECK(AreConsecutive(vt, vt2, vt3)); 2899 LoadStoreStruct(vt, dst, NEON_ST3); 2900 } 2901 2902 void Assembler::st3(const VRegister& vt, const VRegister& vt2, 2903 const VRegister& vt3, int lane, const MemOperand& dst) { 2904 USE(vt2); 2905 USE(vt3); 2906 DCHECK(AreSameFormat(vt, vt2, vt3)); 2907 DCHECK(AreConsecutive(vt, vt2, vt3)); 2908 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3); 2909 } 2910 2911 void Assembler::st4(const VRegister& vt, const VRegister& vt2, 2912 const VRegister& vt3, const VRegister& vt4, 2913 const MemOperand& dst) { 2914 USE(vt2); 2915 USE(vt3); 2916 USE(vt4); 2917 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2918 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2919 LoadStoreStruct(vt, dst, NEON_ST4); 2920 } 2921 2922 void Assembler::st4(const VRegister& vt, const VRegister& vt2, 2923 const VRegister& vt3, const VRegister& vt4, int lane, 2924 const MemOperand& dst) { 2925 USE(vt2); 2926 USE(vt3); 2927 USE(vt4); 2928 DCHECK(AreSameFormat(vt, vt2, vt3, vt4)); 2929 DCHECK(AreConsecutive(vt, vt2, vt3, vt4)); 2930 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4); 2931 } 2932 2933 void Assembler::LoadStoreStructSingle(const VRegister& vt, uint32_t lane, 2934 const MemOperand& addr, 2935 NEONLoadStoreSingleStructOp op) { 2936 LoadStoreStructVerify(vt, addr, op); 2937 2938 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the 2939 // number of lanes, and T is b, h, s or d. 2940 unsigned lane_size = vt.LaneSizeInBytes(); 2941 DCHECK_LT(lane, kQRegSize / lane_size); 2942 2943 // Lane size is encoded in the opcode field. Lane index is encoded in the Q, 2944 // S and size fields. 2945 lane *= lane_size; 2946 2947 // Encodings for S[0]/D[0] and S[2]/D[1] are distinguished using the least- 2948 // significant bit of the size field, so we increment lane here to account for 2949 // that. 2950 if (lane_size == 8) lane++; 2951 2952 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask; 2953 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask; 2954 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask; 2955 2956 Instr instr = op; 2957 switch (lane_size) { 2958 case 1: 2959 instr |= NEONLoadStoreSingle_b; 2960 break; 2961 case 2: 2962 instr |= NEONLoadStoreSingle_h; 2963 break; 2964 case 4: 2965 instr |= NEONLoadStoreSingle_s; 2966 break; 2967 default: 2968 DCHECK_EQ(lane_size, 8U); 2969 instr |= NEONLoadStoreSingle_d; 2970 } 2971 2972 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt)); 2973 } 2974 2975 void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) { 2976 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1); 2977 } 2978 2979 void Assembler::ld1r(const VRegister& vt, const MemOperand& src) { 2980 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R); 2981 } 2982 2983 void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) { 2984 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1); 2985 } 2986 2987 void Assembler::dmb(BarrierDomain domain, BarrierType type) { 2988 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 2989 } 2990 2991 void Assembler::dsb(BarrierDomain domain, BarrierType type) { 2992 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 2993 } 2994 2995 void Assembler::isb() { 2996 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll)); 2997 } 2998 2999 void Assembler::csdb() { hint(CSDB); } 3000 3001 void Assembler::fmov(const VRegister& vd, double imm) { 3002 if (vd.IsScalar()) { 3003 DCHECK(vd.Is1D()); 3004 Emit(FMOV_d_imm | Rd(vd) | ImmFP(imm)); 3005 } else { 3006 DCHECK(vd.Is2D()); 3007 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit; 3008 Emit(NEON_Q | op | ImmNEONFP(imm) | NEONCmode(0xF) | Rd(vd)); 3009 } 3010 } 3011 3012 void Assembler::fmov(const VRegister& vd, float imm) { 3013 if (vd.IsScalar()) { 3014 DCHECK(vd.Is1S()); 3015 Emit(FMOV_s_imm | Rd(vd) | ImmFP(imm)); 3016 } else { 3017 DCHECK(vd.Is2S() | vd.Is4S()); 3018 Instr op = NEONModifiedImmediate_MOVI; 3019 Instr q = vd.Is4S() ? NEON_Q : 0; 3020 Emit(q | op | ImmNEONFP(imm) | NEONCmode(0xF) | Rd(vd)); 3021 } 3022 } 3023 3024 void Assembler::fmov(const Register& rd, const VRegister& fn) { 3025 DCHECK_EQ(rd.SizeInBits(), fn.SizeInBits()); 3026 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd; 3027 Emit(op | Rd(rd) | Rn(fn)); 3028 } 3029 3030 void Assembler::fmov(const VRegister& vd, const Register& rn) { 3031 DCHECK_EQ(vd.SizeInBits(), rn.SizeInBits()); 3032 FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx; 3033 Emit(op | Rd(vd) | Rn(rn)); 3034 } 3035 3036 void Assembler::fmov(const VRegister& vd, const VRegister& vn) { 3037 DCHECK_EQ(vd.SizeInBits(), vn.SizeInBits()); 3038 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn)); 3039 } 3040 3041 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) { 3042 DCHECK((index == 1) && vd.Is1D() && rn.IsX()); 3043 USE(index); 3044 Emit(FMOV_d1_x | Rd(vd) | Rn(rn)); 3045 } 3046 3047 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) { 3048 DCHECK((index == 1) && vn.Is1D() && rd.IsX()); 3049 USE(index); 3050 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn)); 3051 } 3052 3053 void Assembler::fmadd(const VRegister& fd, const VRegister& fn, 3054 const VRegister& fm, const VRegister& fa) { 3055 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMADD_s : FMADD_d); 3056 } 3057 3058 void Assembler::fmsub(const VRegister& fd, const VRegister& fn, 3059 const VRegister& fm, const VRegister& fa) { 3060 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMSUB_s : FMSUB_d); 3061 } 3062 3063 void Assembler::fnmadd(const VRegister& fd, const VRegister& fn, 3064 const VRegister& fm, const VRegister& fa) { 3065 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMADD_s : FNMADD_d); 3066 } 3067 3068 void Assembler::fnmsub(const VRegister& fd, const VRegister& fn, 3069 const VRegister& fm, const VRegister& fa) { 3070 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMSUB_s : FNMSUB_d); 3071 } 3072 3073 void Assembler::fnmul(const VRegister& vd, const VRegister& vn, 3074 const VRegister& vm) { 3075 DCHECK(AreSameSizeAndType(vd, vn, vm)); 3076 Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d; 3077 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd)); 3078 } 3079 3080 void Assembler::fcmp(const VRegister& fn, const VRegister& fm) { 3081 DCHECK_EQ(fn.SizeInBits(), fm.SizeInBits()); 3082 Emit(FPType(fn) | FCMP | Rm(fm) | Rn(fn)); 3083 } 3084 3085 void Assembler::fcmp(const VRegister& fn, double value) { 3086 USE(value); 3087 // Although the fcmp instruction can strictly only take an immediate value of 3088 // +0.0, we don't need to check for -0.0 because the sign of 0.0 doesn't 3089 // affect the result of the comparison. 3090 DCHECK_EQ(value, 0.0); 3091 Emit(FPType(fn) | FCMP_zero | Rn(fn)); 3092 } 3093 3094 void Assembler::fccmp(const VRegister& fn, const VRegister& fm, 3095 StatusFlags nzcv, Condition cond) { 3096 DCHECK_EQ(fn.SizeInBits(), fm.SizeInBits()); 3097 Emit(FPType(fn) | FCCMP | Rm(fm) | Cond(cond) | Rn(fn) | Nzcv(nzcv)); 3098 } 3099 3100 void Assembler::fcsel(const VRegister& fd, const VRegister& fn, 3101 const VRegister& fm, Condition cond) { 3102 DCHECK_EQ(fd.SizeInBits(), fn.SizeInBits()); 3103 DCHECK_EQ(fd.SizeInBits(), fm.SizeInBits()); 3104 Emit(FPType(fd) | FCSEL | Rm(fm) | Cond(cond) | Rn(fn) | Rd(fd)); 3105 } 3106 3107 void Assembler::NEONFPConvertToInt(const Register& rd, const VRegister& vn, 3108 Instr op) { 3109 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd)); 3110 } 3111 3112 void Assembler::NEONFPConvertToInt(const VRegister& vd, const VRegister& vn, 3113 Instr op) { 3114 if (vn.IsScalar()) { 3115 DCHECK((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D())); 3116 op |= NEON_Q | NEONScalar; 3117 } 3118 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd)); 3119 } 3120 3121 void Assembler::fcvt(const VRegister& vd, const VRegister& vn) { 3122 FPDataProcessing1SourceOp op; 3123 if (vd.Is1D()) { 3124 DCHECK(vn.Is1S() || vn.Is1H()); 3125 op = vn.Is1S() ? FCVT_ds : FCVT_dh; 3126 } else if (vd.Is1S()) { 3127 DCHECK(vn.Is1D() || vn.Is1H()); 3128 op = vn.Is1D() ? FCVT_sd : FCVT_sh; 3129 } else { 3130 DCHECK(vd.Is1H()); 3131 DCHECK(vn.Is1D() || vn.Is1S()); 3132 op = vn.Is1D() ? FCVT_hd : FCVT_hs; 3133 } 3134 FPDataProcessing1Source(vd, vn, op); 3135 } 3136 3137 void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) { 3138 DCHECK((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S())); 3139 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0; 3140 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd)); 3141 } 3142 3143 void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) { 3144 DCHECK((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S())); 3145 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0; 3146 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd)); 3147 } 3148 3149 void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) { 3150 DCHECK((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S())); 3151 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0; 3152 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd)); 3153 } 3154 3155 void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) { 3156 DCHECK((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S())); 3157 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0; 3158 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd)); 3159 } 3160 3161 void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) { 3162 Instr format = 1 << NEONSize_offset; 3163 if (vd.IsScalar()) { 3164 DCHECK(vd.Is1S() && vn.Is1D()); 3165 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd)); 3166 } else { 3167 DCHECK(vd.Is2S() && vn.Is2D()); 3168 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd)); 3169 } 3170 } 3171 3172 void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) { 3173 DCHECK(vd.Is4S() && vn.Is2D()); 3174 Instr format = 1 << NEONSize_offset; 3175 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd)); 3176 } 3177 3178 #define NEON_FP2REGMISC_FCVT_LIST(V) \ 3179 V(fcvtnu, NEON_FCVTNU, FCVTNU) \ 3180 V(fcvtns, NEON_FCVTNS, FCVTNS) \ 3181 V(fcvtpu, NEON_FCVTPU, FCVTPU) \ 3182 V(fcvtps, NEON_FCVTPS, FCVTPS) \ 3183 V(fcvtmu, NEON_FCVTMU, FCVTMU) \ 3184 V(fcvtms, NEON_FCVTMS, FCVTMS) \ 3185 V(fcvtau, NEON_FCVTAU, FCVTAU) \ 3186 V(fcvtas, NEON_FCVTAS, FCVTAS) 3187 3188 #define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \ 3189 void Assembler::FN(const Register& rd, const VRegister& vn) { \ 3190 NEONFPConvertToInt(rd, vn, SCA_OP); \ 3191 } \ 3192 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \ 3193 NEONFPConvertToInt(vd, vn, VEC_OP); \ 3194 } 3195 NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS) 3196 #undef DEFINE_ASM_FUNCS 3197 3198 void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) { 3199 DCHECK_GE(fbits, 0); 3200 if (fbits == 0) { 3201 NEONFP2RegMisc(vd, vn, NEON_SCVTF); 3202 } else { 3203 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 3204 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm); 3205 } 3206 } 3207 3208 void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) { 3209 DCHECK_GE(fbits, 0); 3210 if (fbits == 0) { 3211 NEONFP2RegMisc(vd, vn, NEON_UCVTF); 3212 } else { 3213 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 3214 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm); 3215 } 3216 } 3217 3218 void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) { 3219 DCHECK_GE(fbits, 0); 3220 if (fbits == 0) { 3221 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd)); 3222 } else { 3223 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 3224 Rd(vd)); 3225 } 3226 } 3227 3228 void Assembler::ucvtf(const VRegister& fd, const Register& rn, int fbits) { 3229 DCHECK_GE(fbits, 0); 3230 if (fbits == 0) { 3231 Emit(SF(rn) | FPType(fd) | UCVTF | Rn(rn) | Rd(fd)); 3232 } else { 3233 Emit(SF(rn) | FPType(fd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 3234 Rd(fd)); 3235 } 3236 } 3237 3238 void Assembler::NEON3Same(const VRegister& vd, const VRegister& vn, 3239 const VRegister& vm, NEON3SameOp vop) { 3240 DCHECK(AreSameFormat(vd, vn, vm)); 3241 DCHECK(vd.IsVector() || !vd.IsQ()); 3242 3243 Instr format, op = vop; 3244 if (vd.IsScalar()) { 3245 op |= NEON_Q | NEONScalar; 3246 format = SFormat(vd); 3247 } else { 3248 format = VFormat(vd); 3249 } 3250 3251 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd)); 3252 } 3253 3254 void Assembler::NEONFP3Same(const VRegister& vd, const VRegister& vn, 3255 const VRegister& vm, Instr op) { 3256 DCHECK(AreSameFormat(vd, vn, vm)); 3257 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd)); 3258 } 3259 3260 #define NEON_FP2REGMISC_LIST(V) \ 3261 V(fabs, NEON_FABS, FABS) \ 3262 V(fneg, NEON_FNEG, FNEG) \ 3263 V(fsqrt, NEON_FSQRT, FSQRT) \ 3264 V(frintn, NEON_FRINTN, FRINTN) \ 3265 V(frinta, NEON_FRINTA, FRINTA) \ 3266 V(frintp, NEON_FRINTP, FRINTP) \ 3267 V(frintm, NEON_FRINTM, FRINTM) \ 3268 V(frintx, NEON_FRINTX, FRINTX) \ 3269 V(frintz, NEON_FRINTZ, FRINTZ) \ 3270 V(frinti, NEON_FRINTI, FRINTI) \ 3271 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \ 3272 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar) 3273 3274 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \ 3275 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \ 3276 Instr op; \ 3277 if (vd.IsScalar()) { \ 3278 DCHECK(vd.Is1S() || vd.Is1D()); \ 3279 op = SCA_OP; \ 3280 } else { \ 3281 DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); \ 3282 op = VEC_OP; \ 3283 } \ 3284 NEONFP2RegMisc(vd, vn, op); \ 3285 } 3286 NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC) 3287 #undef DEFINE_ASM_FUNC 3288 3289 void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) { 3290 DCHECK((vd.Is8H() && vn.Is8B() && shift == 8) || 3291 (vd.Is4S() && vn.Is4H() && shift == 16) || 3292 (vd.Is2D() && vn.Is2S() && shift == 32)); 3293 USE(shift); 3294 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd)); 3295 } 3296 3297 void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) { 3298 USE(shift); 3299 DCHECK((vd.Is8H() && vn.Is16B() && shift == 8) || 3300 (vd.Is4S() && vn.Is8H() && shift == 16) || 3301 (vd.Is2D() && vn.Is4S() && shift == 32)); 3302 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd)); 3303 } 3304 3305 void Assembler::NEONFP2RegMisc(const VRegister& vd, const VRegister& vn, 3306 NEON2RegMiscOp vop, double value) { 3307 DCHECK(AreSameFormat(vd, vn)); 3308 DCHECK_EQ(value, 0.0); 3309 USE(value); 3310 3311 Instr op = vop; 3312 if (vd.IsScalar()) { 3313 DCHECK(vd.Is1S() || vd.Is1D()); 3314 op |= NEON_Q | NEONScalar; 3315 } else { 3316 DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); 3317 } 3318 3319 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd)); 3320 } 3321 3322 void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) { 3323 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value); 3324 } 3325 3326 void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) { 3327 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value); 3328 } 3329 3330 void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) { 3331 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value); 3332 } 3333 3334 void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) { 3335 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value); 3336 } 3337 3338 void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) { 3339 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value); 3340 } 3341 3342 void Assembler::frecpx(const VRegister& vd, const VRegister& vn) { 3343 DCHECK(vd.IsScalar()); 3344 DCHECK(AreSameFormat(vd, vn)); 3345 DCHECK(vd.Is1S() || vd.Is1D()); 3346 Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd)); 3347 } 3348 3349 void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) { 3350 DCHECK(vn.Is1S() || vn.Is1D()); 3351 DCHECK((fbits >= 0) && (fbits <= rd.SizeInBits())); 3352 if (fbits == 0) { 3353 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd)); 3354 } else { 3355 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) | 3356 Rd(rd)); 3357 } 3358 } 3359 3360 void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) { 3361 DCHECK_GE(fbits, 0); 3362 if (fbits == 0) { 3363 NEONFP2RegMisc(vd, vn, NEON_FCVTZS); 3364 } else { 3365 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 3366 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm); 3367 } 3368 } 3369 3370 void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) { 3371 DCHECK(vn.Is1S() || vn.Is1D()); 3372 DCHECK((fbits >= 0) && (fbits <= rd.SizeInBits())); 3373 if (fbits == 0) { 3374 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd)); 3375 } else { 3376 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) | 3377 Rd(rd)); 3378 } 3379 } 3380 3381 void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) { 3382 DCHECK_GE(fbits, 0); 3383 if (fbits == 0) { 3384 NEONFP2RegMisc(vd, vn, NEON_FCVTZU); 3385 } else { 3386 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 3387 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm); 3388 } 3389 } 3390 3391 void Assembler::NEONFP2RegMisc(const VRegister& vd, const VRegister& vn, 3392 Instr op) { 3393 DCHECK(AreSameFormat(vd, vn)); 3394 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd)); 3395 } 3396 3397 void Assembler::NEON2RegMisc(const VRegister& vd, const VRegister& vn, 3398 NEON2RegMiscOp vop, int value) { 3399 DCHECK(AreSameFormat(vd, vn)); 3400 DCHECK_EQ(value, 0); 3401 USE(value); 3402 3403 Instr format, op = vop; 3404 if (vd.IsScalar()) { 3405 op |= NEON_Q | NEONScalar; 3406 format = SFormat(vd); 3407 } else { 3408 format = VFormat(vd); 3409 } 3410 3411 Emit(format | op | Rn(vn) | Rd(vd)); 3412 } 3413 3414 void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) { 3415 DCHECK(vd.IsVector() || vd.Is1D()); 3416 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value); 3417 } 3418 3419 void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) { 3420 DCHECK(vd.IsVector() || vd.Is1D()); 3421 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value); 3422 } 3423 3424 void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) { 3425 DCHECK(vd.IsVector() || vd.Is1D()); 3426 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value); 3427 } 3428 3429 void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) { 3430 DCHECK(vd.IsVector() || vd.Is1D()); 3431 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value); 3432 } 3433 3434 void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) { 3435 DCHECK(vd.IsVector() || vd.Is1D()); 3436 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value); 3437 } 3438 3439 #define NEON_3SAME_LIST(V) \ 3440 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \ 3441 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \ 3442 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \ 3443 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \ 3444 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \ 3445 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \ 3446 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \ 3447 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \ 3448 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \ 3449 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \ 3450 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \ 3451 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \ 3452 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \ 3453 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \ 3454 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \ 3455 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3456 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3457 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3458 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3459 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \ 3460 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \ 3461 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \ 3462 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3463 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \ 3464 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3465 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \ 3466 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3467 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \ 3468 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3469 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \ 3470 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3471 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \ 3472 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3473 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \ 3474 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \ 3475 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \ 3476 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \ 3477 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \ 3478 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \ 3479 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \ 3480 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \ 3481 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \ 3482 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \ 3483 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \ 3484 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \ 3485 V(uqadd, NEON_UQADD, true) \ 3486 V(sqadd, NEON_SQADD, true) \ 3487 V(uqsub, NEON_UQSUB, true) \ 3488 V(sqsub, NEON_SQSUB, true) \ 3489 V(sqshl, NEON_SQSHL, true) \ 3490 V(uqshl, NEON_UQSHL, true) \ 3491 V(sqrshl, NEON_SQRSHL, true) \ 3492 V(uqrshl, NEON_UQRSHL, true) 3493 3494 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 3495 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 3496 const VRegister& vm) { \ 3497 DCHECK(AS); \ 3498 NEON3Same(vd, vn, vm, OP); \ 3499 } 3500 NEON_3SAME_LIST(DEFINE_ASM_FUNC) 3501 #undef DEFINE_ASM_FUNC 3502 3503 #define NEON_FP3SAME_LIST_V2(V) \ 3504 V(fadd, NEON_FADD, FADD) \ 3505 V(fsub, NEON_FSUB, FSUB) \ 3506 V(fmul, NEON_FMUL, FMUL) \ 3507 V(fdiv, NEON_FDIV, FDIV) \ 3508 V(fmax, NEON_FMAX, FMAX) \ 3509 V(fmaxnm, NEON_FMAXNM, FMAXNM) \ 3510 V(fmin, NEON_FMIN, FMIN) \ 3511 V(fminnm, NEON_FMINNM, FMINNM) \ 3512 V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \ 3513 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \ 3514 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \ 3515 V(fabd, NEON_FABD, NEON_FABD_scalar) \ 3516 V(fmla, NEON_FMLA, 0) \ 3517 V(fmls, NEON_FMLS, 0) \ 3518 V(facge, NEON_FACGE, NEON_FACGE_scalar) \ 3519 V(facgt, NEON_FACGT, NEON_FACGT_scalar) \ 3520 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \ 3521 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \ 3522 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \ 3523 V(faddp, NEON_FADDP, 0) \ 3524 V(fmaxp, NEON_FMAXP, 0) \ 3525 V(fminp, NEON_FMINP, 0) \ 3526 V(fmaxnmp, NEON_FMAXNMP, 0) \ 3527 V(fminnmp, NEON_FMINNMP, 0) 3528 3529 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \ 3530 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 3531 const VRegister& vm) { \ 3532 Instr op; \ 3533 if ((SCA_OP != 0) && vd.IsScalar()) { \ 3534 DCHECK(vd.Is1S() || vd.Is1D()); \ 3535 op = SCA_OP; \ 3536 } else { \ 3537 DCHECK(vd.IsVector()); \ 3538 DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); \ 3539 op = VEC_OP; \ 3540 } \ 3541 NEONFP3Same(vd, vn, vm, op); \ 3542 } 3543 NEON_FP3SAME_LIST_V2(DEFINE_ASM_FUNC) 3544 #undef DEFINE_ASM_FUNC 3545 3546 void Assembler::addp(const VRegister& vd, const VRegister& vn) { 3547 DCHECK((vd.Is1D() && vn.Is2D())); 3548 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd)); 3549 } 3550 3551 void Assembler::faddp(const VRegister& vd, const VRegister& vn) { 3552 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D())); 3553 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd)); 3554 } 3555 3556 void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) { 3557 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D())); 3558 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd)); 3559 } 3560 3561 void Assembler::fminp(const VRegister& vd, const VRegister& vn) { 3562 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D())); 3563 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd)); 3564 } 3565 3566 void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) { 3567 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D())); 3568 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd)); 3569 } 3570 3571 void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) { 3572 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D())); 3573 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd)); 3574 } 3575 3576 void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) { 3577 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR); 3578 } 3579 3580 void Assembler::mov(const VRegister& vd, const VRegister& vn) { 3581 DCHECK(AreSameFormat(vd, vn)); 3582 if (vd.IsD()) { 3583 orr(vd.V8B(), vn.V8B(), vn.V8B()); 3584 } else { 3585 DCHECK(vd.IsQ()); 3586 orr(vd.V16B(), vn.V16B(), vn.V16B()); 3587 } 3588 } 3589 3590 void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) { 3591 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC); 3592 } 3593 3594 void Assembler::movi(const VRegister& vd, const uint64_t imm, Shift shift, 3595 const int shift_amount) { 3596 DCHECK((shift == LSL) || (shift == MSL)); 3597 if (vd.Is2D() || vd.Is1D()) { 3598 DCHECK_EQ(shift_amount, 0); 3599 int imm8 = 0; 3600 for (int i = 0; i < 8; ++i) { 3601 int byte = (imm >> (i * 8)) & 0xFF; 3602 DCHECK((byte == 0) || (byte == 0xFF)); 3603 if (byte == 0xFF) { 3604 imm8 |= (1 << i); 3605 } 3606 } 3607 Instr q = vd.Is2D() ? NEON_Q : 0; 3608 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI | 3609 ImmNEONabcdefgh(imm8) | NEONCmode(0xE) | Rd(vd)); 3610 } else if (shift == LSL) { 3611 NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount, 3612 NEONModifiedImmediate_MOVI); 3613 } else { 3614 NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount, 3615 NEONModifiedImmediate_MOVI); 3616 } 3617 } 3618 3619 void Assembler::mvn(const VRegister& vd, const VRegister& vn) { 3620 DCHECK(AreSameFormat(vd, vn)); 3621 if (vd.IsD()) { 3622 not_(vd.V8B(), vn.V8B()); 3623 } else { 3624 DCHECK(vd.IsQ()); 3625 not_(vd.V16B(), vn.V16B()); 3626 } 3627 } 3628 3629 void Assembler::mvni(const VRegister& vd, const int imm8, Shift shift, 3630 const int shift_amount) { 3631 DCHECK((shift == LSL) || (shift == MSL)); 3632 if (shift == LSL) { 3633 NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI); 3634 } else { 3635 NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI); 3636 } 3637 } 3638 3639 void Assembler::NEONFPByElement(const VRegister& vd, const VRegister& vn, 3640 const VRegister& vm, int vm_index, 3641 NEONByIndexedElementOp vop) { 3642 DCHECK(AreSameFormat(vd, vn)); 3643 DCHECK((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) || 3644 (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) || 3645 (vd.Is1D() && vm.Is1D())); 3646 DCHECK((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2))); 3647 3648 Instr op = vop; 3649 int index_num_bits = vm.Is1S() ? 2 : 1; 3650 if (vd.IsScalar()) { 3651 op |= NEON_Q | NEONScalar; 3652 } 3653 3654 Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | 3655 Rn(vn) | Rd(vd)); 3656 } 3657 3658 void Assembler::NEONByElement(const VRegister& vd, const VRegister& vn, 3659 const VRegister& vm, int vm_index, 3660 NEONByIndexedElementOp vop) { 3661 DCHECK(AreSameFormat(vd, vn)); 3662 DCHECK((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) || 3663 (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) || 3664 (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S())); 3665 DCHECK((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) || 3666 (vm.Is1S() && (vm_index < 4))); 3667 3668 Instr format, op = vop; 3669 int index_num_bits = vm.Is1H() ? 3 : 2; 3670 if (vd.IsScalar()) { 3671 op |= NEONScalar | NEON_Q; 3672 format = SFormat(vn); 3673 } else { 3674 format = VFormat(vn); 3675 } 3676 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | 3677 Rd(vd)); 3678 } 3679 3680 void Assembler::NEONByElementL(const VRegister& vd, const VRegister& vn, 3681 const VRegister& vm, int vm_index, 3682 NEONByIndexedElementOp vop) { 3683 DCHECK((vd.Is4S() && vn.Is4H() && vm.Is1H()) || 3684 (vd.Is4S() && vn.Is8H() && vm.Is1H()) || 3685 (vd.Is1S() && vn.Is1H() && vm.Is1H()) || 3686 (vd.Is2D() && vn.Is2S() && vm.Is1S()) || 3687 (vd.Is2D() && vn.Is4S() && vm.Is1S()) || 3688 (vd.Is1D() && vn.Is1S() && vm.Is1S())); 3689 3690 DCHECK((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) || 3691 (vm.Is1S() && (vm_index < 4))); 3692 3693 Instr format, op = vop; 3694 int index_num_bits = vm.Is1H() ? 3 : 2; 3695 if (vd.IsScalar()) { 3696 op |= NEONScalar | NEON_Q; 3697 format = SFormat(vn); 3698 } else { 3699 format = VFormat(vn); 3700 } 3701 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | 3702 Rd(vd)); 3703 } 3704 3705 #define NEON_BYELEMENT_LIST(V) \ 3706 V(mul, NEON_MUL_byelement, vn.IsVector()) \ 3707 V(mla, NEON_MLA_byelement, vn.IsVector()) \ 3708 V(mls, NEON_MLS_byelement, vn.IsVector()) \ 3709 V(sqdmulh, NEON_SQDMULH_byelement, true) \ 3710 V(sqrdmulh, NEON_SQRDMULH_byelement, true) 3711 3712 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 3713 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 3714 const VRegister& vm, int vm_index) { \ 3715 DCHECK(AS); \ 3716 NEONByElement(vd, vn, vm, vm_index, OP); \ 3717 } 3718 NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC) 3719 #undef DEFINE_ASM_FUNC 3720 3721 #define NEON_FPBYELEMENT_LIST(V) \ 3722 V(fmul, NEON_FMUL_byelement) \ 3723 V(fmla, NEON_FMLA_byelement) \ 3724 V(fmls, NEON_FMLS_byelement) \ 3725 V(fmulx, NEON_FMULX_byelement) 3726 3727 #define DEFINE_ASM_FUNC(FN, OP) \ 3728 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 3729 const VRegister& vm, int vm_index) { \ 3730 NEONFPByElement(vd, vn, vm, vm_index, OP); \ 3731 } 3732 NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC) 3733 #undef DEFINE_ASM_FUNC 3734 3735 #define NEON_BYELEMENT_LONG_LIST(V) \ 3736 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \ 3737 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \ 3738 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \ 3739 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \ 3740 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \ 3741 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \ 3742 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \ 3743 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \ 3744 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \ 3745 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \ 3746 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \ 3747 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \ 3748 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \ 3749 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \ 3750 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \ 3751 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \ 3752 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \ 3753 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ()) 3754 3755 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 3756 void Assembler::FN(const VRegister& vd, const VRegister& vn, \ 3757 const VRegister& vm, int vm_index) { \ 3758 DCHECK(AS); \ 3759 NEONByElementL(vd, vn, vm, vm_index, OP); \ 3760 } 3761 NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC) 3762 #undef DEFINE_ASM_FUNC 3763 3764 void Assembler::suqadd(const VRegister& vd, const VRegister& vn) { 3765 NEON2RegMisc(vd, vn, NEON_SUQADD); 3766 } 3767 3768 void Assembler::usqadd(const VRegister& vd, const VRegister& vn) { 3769 NEON2RegMisc(vd, vn, NEON_USQADD); 3770 } 3771 3772 void Assembler::abs(const VRegister& vd, const VRegister& vn) { 3773 DCHECK(vd.IsVector() || vd.Is1D()); 3774 NEON2RegMisc(vd, vn, NEON_ABS); 3775 } 3776 3777 void Assembler::sqabs(const VRegister& vd, const VRegister& vn) { 3778 NEON2RegMisc(vd, vn, NEON_SQABS); 3779 } 3780 3781 void Assembler::neg(const VRegister& vd, const VRegister& vn) { 3782 DCHECK(vd.IsVector() || vd.Is1D()); 3783 NEON2RegMisc(vd, vn, NEON_NEG); 3784 } 3785 3786 void Assembler::sqneg(const VRegister& vd, const VRegister& vn) { 3787 NEON2RegMisc(vd, vn, NEON_SQNEG); 3788 } 3789 3790 void Assembler::NEONXtn(const VRegister& vd, const VRegister& vn, 3791 NEON2RegMiscOp vop) { 3792 Instr format, op = vop; 3793 if (vd.IsScalar()) { 3794 DCHECK((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) || 3795 (vd.Is1S() && vn.Is1D())); 3796 op |= NEON_Q | NEONScalar; 3797 format = SFormat(vd); 3798 } else { 3799 DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) || 3800 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) || 3801 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D())); 3802 format = VFormat(vd); 3803 } 3804 Emit(format | op | Rn(vn) | Rd(vd)); 3805 } 3806 3807 void Assembler::xtn(const VRegister& vd, const VRegister& vn) { 3808 DCHECK(vd.IsVector() && vd.IsD()); 3809 NEONXtn(vd, vn, NEON_XTN); 3810 } 3811 3812 void Assembler::xtn2(const VRegister& vd, const VRegister& vn) { 3813 DCHECK(vd.IsVector() && vd.IsQ()); 3814 NEONXtn(vd, vn, NEON_XTN); 3815 } 3816 3817 void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) { 3818 DCHECK(vd.IsScalar() || vd.IsD()); 3819 NEONXtn(vd, vn, NEON_SQXTN); 3820 } 3821 3822 void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) { 3823 DCHECK(vd.IsVector() && vd.IsQ()); 3824 NEONXtn(vd, vn, NEON_SQXTN); 3825 } 3826 3827 void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) { 3828 DCHECK(vd.IsScalar() || vd.IsD()); 3829 NEONXtn(vd, vn, NEON_SQXTUN); 3830 } 3831 3832 void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) { 3833 DCHECK(vd.IsVector() && vd.IsQ()); 3834 NEONXtn(vd, vn, NEON_SQXTUN); 3835 } 3836 3837 void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) { 3838 DCHECK(vd.IsScalar() || vd.IsD()); 3839 NEONXtn(vd, vn, NEON_UQXTN); 3840 } 3841 3842 void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) { 3843 DCHECK(vd.IsVector() && vd.IsQ()); 3844 NEONXtn(vd, vn, NEON_UQXTN); 3845 } 3846 3847 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size". 3848 void Assembler::not_(const VRegister& vd, const VRegister& vn) { 3849 DCHECK(AreSameFormat(vd, vn)); 3850 DCHECK(vd.Is8B() || vd.Is16B()); 3851 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd)); 3852 } 3853 3854 void Assembler::rbit(const VRegister& vd, const VRegister& vn) { 3855 DCHECK(AreSameFormat(vd, vn)); 3856 DCHECK(vd.Is8B() || vd.Is16B()); 3857 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd)); 3858 } 3859 3860 void Assembler::ext(const VRegister& vd, const VRegister& vn, 3861 const VRegister& vm, int index) { 3862 DCHECK(AreSameFormat(vd, vn, vm)); 3863 DCHECK(vd.Is8B() || vd.Is16B()); 3864 DCHECK((0 <= index) && (index < vd.LaneCount())); 3865 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd)); 3866 } 3867 3868 void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) { 3869 Instr q, scalar; 3870 3871 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the 3872 // number of lanes, and T is b, h, s or d. 3873 int lane_size = vn.LaneSizeInBytes(); 3874 NEONFormatField format; 3875 switch (lane_size) { 3876 case 1: 3877 format = NEON_16B; 3878 break; 3879 case 2: 3880 format = NEON_8H; 3881 break; 3882 case 4: 3883 format = NEON_4S; 3884 break; 3885 default: 3886 DCHECK_EQ(lane_size, 8); 3887 format = NEON_2D; 3888 break; 3889 } 3890 3891 if (vd.IsScalar()) { 3892 q = NEON_Q; 3893 scalar = NEONScalar; 3894 } else { 3895 DCHECK(!vd.Is1D()); 3896 q = vd.IsD() ? 0 : NEON_Q; 3897 scalar = 0; 3898 } 3899 Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) | 3900 Rd(vd)); 3901 } 3902 3903 void Assembler::dcptr(Label* label) { 3904 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 3905 if (label->is_bound()) { 3906 // The label is bound, so it does not need to be updated and the internal 3907 // reference should be emitted. 3908 // 3909 // In this case, label->pos() returns the offset of the label from the 3910 // start of the buffer. 3911 internal_reference_positions_.push_back(pc_offset()); 3912 dc64(reinterpret_cast<uintptr_t>(buffer_ + label->pos())); 3913 } else { 3914 int32_t offset; 3915 if (label->is_linked()) { 3916 // The label is linked, so the internal reference should be added 3917 // onto the end of the label's link chain. 3918 // 3919 // In this case, label->pos() returns the offset of the last linked 3920 // instruction from the start of the buffer. 3921 offset = label->pos() - pc_offset(); 3922 DCHECK_NE(offset, kStartOfLabelLinkChain); 3923 } else { 3924 // The label is unused, so it now becomes linked and the internal 3925 // reference is at the start of the new link chain. 3926 offset = kStartOfLabelLinkChain; 3927 } 3928 // The instruction at pc is now the last link in the label's chain. 3929 label->link_to(pc_offset()); 3930 3931 // Traditionally the offset to the previous instruction in the chain is 3932 // encoded in the instruction payload (e.g. branch range) but internal 3933 // references are not instructions so while unbound they are encoded as 3934 // two consecutive brk instructions. The two 16-bit immediates are used 3935 // to encode the offset. 3936 offset >>= kInstrSizeLog2; 3937 DCHECK(is_int32(offset)); 3938 uint32_t high16 = unsigned_bitextract_32(31, 16, offset); 3939 uint32_t low16 = unsigned_bitextract_32(15, 0, offset); 3940 3941 brk(high16); 3942 brk(low16); 3943 } 3944 } 3945 3946 // Below, a difference in case for the same letter indicates a 3947 // negated bit. If b is 1, then B is 0. 3948 uint32_t Assembler::FPToImm8(double imm) { 3949 DCHECK(IsImmFP64(imm)); 3950 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 3951 // 0000.0000.0000.0000.0000.0000.0000.0000 3952 uint64_t bits = bit_cast<uint64_t>(imm); 3953 // bit7: a000.0000 3954 uint64_t bit7 = ((bits >> 63) & 0x1) << 7; 3955 // bit6: 0b00.0000 3956 uint64_t bit6 = ((bits >> 61) & 0x1) << 6; 3957 // bit5_to_0: 00cd.efgh 3958 uint64_t bit5_to_0 = (bits >> 48) & 0x3F; 3959 3960 return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0); 3961 } 3962 3963 Instr Assembler::ImmFP(double imm) { return FPToImm8(imm) << ImmFP_offset; } 3964 Instr Assembler::ImmNEONFP(double imm) { 3965 return ImmNEONabcdefgh(FPToImm8(imm)); 3966 } 3967 3968 // Code generation helpers. 3969 void Assembler::MoveWide(const Register& rd, uint64_t imm, int shift, 3970 MoveWideImmediateOp mov_op) { 3971 // Ignore the top 32 bits of an immediate if we're moving to a W register. 3972 if (rd.Is32Bits()) { 3973 // Check that the top 32 bits are zero (a positive 32-bit number) or top 3974 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits). 3975 DCHECK(((imm >> kWRegSizeInBits) == 0) || 3976 ((imm >> (kWRegSizeInBits - 1)) == 0x1FFFFFFFF)); 3977 imm &= kWRegMask; 3978 } 3979 3980 if (shift >= 0) { 3981 // Explicit shift specified. 3982 DCHECK((shift == 0) || (shift == 16) || (shift == 32) || (shift == 48)); 3983 DCHECK(rd.Is64Bits() || (shift == 0) || (shift == 16)); 3984 shift /= 16; 3985 } else { 3986 // Calculate a new immediate and shift combination to encode the immediate 3987 // argument. 3988 shift = 0; 3989 if ((imm & ~0xFFFFUL) == 0) { 3990 // Nothing to do. 3991 } else if ((imm & ~(0xFFFFUL << 16)) == 0) { 3992 imm >>= 16; 3993 shift = 1; 3994 } else if ((imm & ~(0xFFFFUL << 32)) == 0) { 3995 DCHECK(rd.Is64Bits()); 3996 imm >>= 32; 3997 shift = 2; 3998 } else if ((imm & ~(0xFFFFUL << 48)) == 0) { 3999 DCHECK(rd.Is64Bits()); 4000 imm >>= 48; 4001 shift = 3; 4002 } 4003 } 4004 4005 DCHECK(is_uint16(imm)); 4006 4007 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | 4008 ImmMoveWide(static_cast<int>(imm)) | ShiftMoveWide(shift)); 4009 } 4010 4011 void Assembler::AddSub(const Register& rd, const Register& rn, 4012 const Operand& operand, FlagsUpdate S, AddSubOp op) { 4013 DCHECK_EQ(rd.SizeInBits(), rn.SizeInBits()); 4014 DCHECK(!operand.NeedsRelocation(this)); 4015 if (operand.IsImmediate()) { 4016 int64_t immediate = operand.ImmediateValue(); 4017 DCHECK(IsImmAddSub(immediate)); 4018 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 4019 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) | 4020 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn)); 4021 } else if (operand.IsShiftedRegister()) { 4022 DCHECK_EQ(operand.reg().SizeInBits(), rd.SizeInBits()); 4023 DCHECK_NE(operand.shift(), ROR); 4024 4025 // For instructions of the form: 4026 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ] 4027 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ] 4028 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ] 4029 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ] 4030 // or their 64-bit register equivalents, convert the operand from shifted to 4031 // extended register mode, and emit an add/sub extended instruction. 4032 if (rn.IsSP() || rd.IsSP()) { 4033 DCHECK(!(rd.IsSP() && (S == SetFlags))); 4034 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S, 4035 AddSubExtendedFixed | op); 4036 } else { 4037 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op); 4038 } 4039 } else { 4040 DCHECK(operand.IsExtendedRegister()); 4041 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op); 4042 } 4043 } 4044 4045 void Assembler::AddSubWithCarry(const Register& rd, const Register& rn, 4046 const Operand& operand, FlagsUpdate S, 4047 AddSubWithCarryOp op) { 4048 DCHECK_EQ(rd.SizeInBits(), rn.SizeInBits()); 4049 DCHECK_EQ(rd.SizeInBits(), operand.reg().SizeInBits()); 4050 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 4051 DCHECK(!operand.NeedsRelocation(this)); 4052 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd)); 4053 } 4054 4055 void Assembler::hlt(int code) { 4056 DCHECK(is_uint16(code)); 4057 Emit(HLT | ImmException(code)); 4058 } 4059 4060 void Assembler::brk(int code) { 4061 DCHECK(is_uint16(code)); 4062 Emit(BRK | ImmException(code)); 4063 } 4064 4065 void Assembler::EmitStringData(const char* string) { 4066 size_t len = strlen(string) + 1; 4067 DCHECK_LE(RoundUp(len, kInstrSize), static_cast<size_t>(kGap)); 4068 EmitData(string, static_cast<int>(len)); 4069 // Pad with nullptr characters until pc_ is aligned. 4070 const char pad[] = {'\0', '\0', '\0', '\0'}; 4071 static_assert(sizeof(pad) == kInstrSize, 4072 "Size of padding must match instruction size."); 4073 EmitData(pad, RoundUp(pc_offset(), kInstrSize) - pc_offset()); 4074 } 4075 4076 4077 void Assembler::debug(const char* message, uint32_t code, Instr params) { 4078 #ifdef USE_SIMULATOR 4079 if (options().enable_simulator_code) { 4080 // The arguments to the debug marker need to be contiguous in memory, so 4081 // make sure we don't try to emit pools. 4082 BlockPoolsScope scope(this); 4083 4084 Label start; 4085 bind(&start); 4086 4087 // Refer to instructions-arm64.h for a description of the marker and its 4088 // arguments. 4089 hlt(kImmExceptionIsDebug); 4090 DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugCodeOffset); 4091 dc32(code); 4092 DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugParamsOffset); 4093 dc32(params); 4094 DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugMessageOffset); 4095 EmitStringData(message); 4096 hlt(kImmExceptionIsUnreachable); 4097 4098 return; 4099 } 4100 // Fall through if Serializer is enabled. 4101 #else 4102 // Make sure we haven't dynamically enabled simulator code when there is no 4103 // simulator built in. 4104 DCHECK(!options().enable_simulator_code); 4105 #endif 4106 4107 if (params & BREAK) { 4108 brk(0); 4109 } 4110 } 4111 4112 4113 void Assembler::Logical(const Register& rd, 4114 const Register& rn, 4115 const Operand& operand, 4116 LogicalOp op) { 4117 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 4118 DCHECK(!operand.NeedsRelocation(this)); 4119 if (operand.IsImmediate()) { 4120 int64_t immediate = operand.ImmediateValue(); 4121 unsigned reg_size = rd.SizeInBits(); 4122 4123 DCHECK_NE(immediate, 0); 4124 DCHECK_NE(immediate, -1); 4125 DCHECK(rd.Is64Bits() || is_uint32(immediate)); 4126 4127 // If the operation is NOT, invert the operation and immediate. 4128 if ((op & NOT) == NOT) { 4129 op = static_cast<LogicalOp>(op & ~NOT); 4130 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask); 4131 } 4132 4133 unsigned n, imm_s, imm_r; 4134 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 4135 // Immediate can be encoded in the instruction. 4136 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 4137 } else { 4138 // This case is handled in the macro assembler. 4139 UNREACHABLE(); 4140 } 4141 } else { 4142 DCHECK(operand.IsShiftedRegister()); 4143 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits()); 4144 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed); 4145 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op); 4146 } 4147 } 4148 4149 4150 void Assembler::LogicalImmediate(const Register& rd, 4151 const Register& rn, 4152 unsigned n, 4153 unsigned imm_s, 4154 unsigned imm_r, 4155 LogicalOp op) { 4156 unsigned reg_size = rd.SizeInBits(); 4157 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd); 4158 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) | 4159 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg | 4160 Rn(rn)); 4161 } 4162 4163 4164 void Assembler::ConditionalCompare(const Register& rn, 4165 const Operand& operand, 4166 StatusFlags nzcv, 4167 Condition cond, 4168 ConditionalCompareOp op) { 4169 Instr ccmpop; 4170 DCHECK(!operand.NeedsRelocation(this)); 4171 if (operand.IsImmediate()) { 4172 int64_t immediate = operand.ImmediateValue(); 4173 DCHECK(IsImmConditionalCompare(immediate)); 4174 ccmpop = ConditionalCompareImmediateFixed | op | 4175 ImmCondCmp(static_cast<unsigned>(immediate)); 4176 } else { 4177 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 4178 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg()); 4179 } 4180 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv)); 4181 } 4182 4183 4184 void Assembler::DataProcessing1Source(const Register& rd, 4185 const Register& rn, 4186 DataProcessing1SourceOp op) { 4187 DCHECK(rd.SizeInBits() == rn.SizeInBits()); 4188 Emit(SF(rn) | op | Rn(rn) | Rd(rd)); 4189 } 4190 4191 void Assembler::FPDataProcessing1Source(const VRegister& vd, 4192 const VRegister& vn, 4193 FPDataProcessing1SourceOp op) { 4194 Emit(FPType(vn) | op | Rn(vn) | Rd(vd)); 4195 } 4196 4197 void Assembler::FPDataProcessing2Source(const VRegister& fd, 4198 const VRegister& fn, 4199 const VRegister& fm, 4200 FPDataProcessing2SourceOp op) { 4201 DCHECK(fd.SizeInBits() == fn.SizeInBits()); 4202 DCHECK(fd.SizeInBits() == fm.SizeInBits()); 4203 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd)); 4204 } 4205 4206 void Assembler::FPDataProcessing3Source(const VRegister& fd, 4207 const VRegister& fn, 4208 const VRegister& fm, 4209 const VRegister& fa, 4210 FPDataProcessing3SourceOp op) { 4211 DCHECK(AreSameSizeAndType(fd, fn, fm, fa)); 4212 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd) | Ra(fa)); 4213 } 4214 4215 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd, const int imm8, 4216 const int left_shift, 4217 NEONModifiedImmediateOp op) { 4218 DCHECK(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() || 4219 vd.Is4S()); 4220 DCHECK((left_shift == 0) || (left_shift == 8) || (left_shift == 16) || 4221 (left_shift == 24)); 4222 DCHECK(is_uint8(imm8)); 4223 4224 int cmode_1, cmode_2, cmode_3; 4225 if (vd.Is8B() || vd.Is16B()) { 4226 DCHECK_EQ(op, NEONModifiedImmediate_MOVI); 4227 cmode_1 = 1; 4228 cmode_2 = 1; 4229 cmode_3 = 1; 4230 } else { 4231 cmode_1 = (left_shift >> 3) & 1; 4232 cmode_2 = left_shift >> 4; 4233 cmode_3 = 0; 4234 if (vd.Is4H() || vd.Is8H()) { 4235 DCHECK((left_shift == 0) || (left_shift == 8)); 4236 cmode_3 = 1; 4237 } 4238 } 4239 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1); 4240 4241 Instr q = vd.IsQ() ? NEON_Q : 0; 4242 4243 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd)); 4244 } 4245 4246 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd, const int imm8, 4247 const int shift_amount, 4248 NEONModifiedImmediateOp op) { 4249 DCHECK(vd.Is2S() || vd.Is4S()); 4250 DCHECK((shift_amount == 8) || (shift_amount == 16)); 4251 DCHECK(is_uint8(imm8)); 4252 4253 int cmode_0 = (shift_amount >> 4) & 1; 4254 int cmode = 0xC | cmode_0; 4255 4256 Instr q = vd.IsQ() ? NEON_Q : 0; 4257 4258 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd)); 4259 } 4260 4261 void Assembler::EmitShift(const Register& rd, 4262 const Register& rn, 4263 Shift shift, 4264 unsigned shift_amount) { 4265 switch (shift) { 4266 case LSL: 4267 lsl(rd, rn, shift_amount); 4268 break; 4269 case LSR: 4270 lsr(rd, rn, shift_amount); 4271 break; 4272 case ASR: 4273 asr(rd, rn, shift_amount); 4274 break; 4275 case ROR: 4276 ror(rd, rn, shift_amount); 4277 break; 4278 default: 4279 UNREACHABLE(); 4280 } 4281 } 4282 4283 4284 void Assembler::EmitExtendShift(const Register& rd, 4285 const Register& rn, 4286 Extend extend, 4287 unsigned left_shift) { 4288 DCHECK(rd.SizeInBits() >= rn.SizeInBits()); 4289 unsigned reg_size = rd.SizeInBits(); 4290 // Use the correct size of register. 4291 Register rn_ = Register::Create(rn.code(), rd.SizeInBits()); 4292 // Bits extracted are high_bit:0. 4293 unsigned high_bit = (8 << (extend & 0x3)) - 1; 4294 // Number of bits left in the result that are not introduced by the shift. 4295 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1); 4296 4297 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) { 4298 switch (extend) { 4299 case UXTB: 4300 case UXTH: 4301 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break; 4302 case SXTB: 4303 case SXTH: 4304 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break; 4305 case UXTX: 4306 case SXTX: { 4307 DCHECK_EQ(rn.SizeInBits(), kXRegSizeInBits); 4308 // Nothing to extend. Just shift. 4309 lsl(rd, rn_, left_shift); 4310 break; 4311 } 4312 default: UNREACHABLE(); 4313 } 4314 } else { 4315 // No need to extend as the extended bits would be shifted away. 4316 lsl(rd, rn_, left_shift); 4317 } 4318 } 4319 4320 4321 void Assembler::DataProcShiftedRegister(const Register& rd, 4322 const Register& rn, 4323 const Operand& operand, 4324 FlagsUpdate S, 4325 Instr op) { 4326 DCHECK(operand.IsShiftedRegister()); 4327 DCHECK(rn.Is64Bits() || (rn.Is32Bits() && is_uint5(operand.shift_amount()))); 4328 DCHECK(!operand.NeedsRelocation(this)); 4329 Emit(SF(rd) | op | Flags(S) | 4330 ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) | 4331 Rm(operand.reg()) | Rn(rn) | Rd(rd)); 4332 } 4333 4334 4335 void Assembler::DataProcExtendedRegister(const Register& rd, 4336 const Register& rn, 4337 const Operand& operand, 4338 FlagsUpdate S, 4339 Instr op) { 4340 DCHECK(!operand.NeedsRelocation(this)); 4341 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 4342 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | 4343 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) | 4344 dest_reg | RnSP(rn)); 4345 } 4346 4347 4348 bool Assembler::IsImmAddSub(int64_t immediate) { 4349 return is_uint12(immediate) || 4350 (is_uint12(immediate >> 12) && ((immediate & 0xFFF) == 0)); 4351 } 4352 4353 void Assembler::LoadStore(const CPURegister& rt, 4354 const MemOperand& addr, 4355 LoadStoreOp op) { 4356 Instr memop = op | Rt(rt) | RnSP(addr.base()); 4357 4358 if (addr.IsImmediateOffset()) { 4359 unsigned size = CalcLSDataSize(op); 4360 if (IsImmLSScaled(addr.offset(), size)) { 4361 int offset = static_cast<int>(addr.offset()); 4362 // Use the scaled addressing mode. 4363 Emit(LoadStoreUnsignedOffsetFixed | memop | 4364 ImmLSUnsigned(offset >> size)); 4365 } else if (IsImmLSUnscaled(addr.offset())) { 4366 int offset = static_cast<int>(addr.offset()); 4367 // Use the unscaled addressing mode. 4368 Emit(LoadStoreUnscaledOffsetFixed | memop | ImmLS(offset)); 4369 } else { 4370 // This case is handled in the macro assembler. 4371 UNREACHABLE(); 4372 } 4373 } else if (addr.IsRegisterOffset()) { 4374 Extend ext = addr.extend(); 4375 Shift shift = addr.shift(); 4376 unsigned shift_amount = addr.shift_amount(); 4377 4378 // LSL is encoded in the option field as UXTX. 4379 if (shift == LSL) { 4380 ext = UXTX; 4381 } 4382 4383 // Shifts are encoded in one bit, indicating a left shift by the memory 4384 // access size. 4385 DCHECK((shift_amount == 0) || 4386 (shift_amount == static_cast<unsigned>(CalcLSDataSize(op)))); 4387 Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) | 4388 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0)); 4389 } else { 4390 // Pre-index and post-index modes. 4391 DCHECK(!rt.Is(addr.base())); 4392 if (IsImmLSUnscaled(addr.offset())) { 4393 int offset = static_cast<int>(addr.offset()); 4394 if (addr.IsPreIndex()) { 4395 Emit(LoadStorePreIndexFixed | memop | ImmLS(offset)); 4396 } else { 4397 DCHECK(addr.IsPostIndex()); 4398 Emit(LoadStorePostIndexFixed | memop | ImmLS(offset)); 4399 } 4400 } else { 4401 // This case is handled in the macro assembler. 4402 UNREACHABLE(); 4403 } 4404 } 4405 } 4406 4407 4408 bool Assembler::IsImmLSUnscaled(int64_t offset) { 4409 return is_int9(offset); 4410 } 4411 4412 bool Assembler::IsImmLSScaled(int64_t offset, unsigned size) { 4413 bool offset_is_size_multiple = (((offset >> size) << size) == offset); 4414 return offset_is_size_multiple && is_uint12(offset >> size); 4415 } 4416 4417 bool Assembler::IsImmLSPair(int64_t offset, unsigned size) { 4418 bool offset_is_size_multiple = (((offset >> size) << size) == offset); 4419 return offset_is_size_multiple && is_int7(offset >> size); 4420 } 4421 4422 4423 bool Assembler::IsImmLLiteral(int64_t offset) { 4424 int inst_size = static_cast<int>(kInstrSizeLog2); 4425 bool offset_is_inst_multiple = 4426 (((offset >> inst_size) << inst_size) == offset); 4427 DCHECK_GT(offset, 0); 4428 offset >>= kLoadLiteralScaleLog2; 4429 return offset_is_inst_multiple && is_intn(offset, ImmLLiteral_width); 4430 } 4431 4432 4433 // Test if a given value can be encoded in the immediate field of a logical 4434 // instruction. 4435 // If it can be encoded, the function returns true, and values pointed to by n, 4436 // imm_s and imm_r are updated with immediates encoded in the format required 4437 // by the corresponding fields in the logical instruction. 4438 // If it can not be encoded, the function returns false, and the values pointed 4439 // to by n, imm_s and imm_r are undefined. 4440 bool Assembler::IsImmLogical(uint64_t value, 4441 unsigned width, 4442 unsigned* n, 4443 unsigned* imm_s, 4444 unsigned* imm_r) { 4445 DCHECK((n != nullptr) && (imm_s != nullptr) && (imm_r != nullptr)); 4446 DCHECK((width == kWRegSizeInBits) || (width == kXRegSizeInBits)); 4447 4448 bool negate = false; 4449 4450 // Logical immediates are encoded using parameters n, imm_s and imm_r using 4451 // the following table: 4452 // 4453 // N imms immr size S R 4454 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr) 4455 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr) 4456 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr) 4457 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr) 4458 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr) 4459 // 0 11110s xxxxxr 2 UInt(s) UInt(r) 4460 // (s bits must not be all set) 4461 // 4462 // A pattern is constructed of size bits, where the least significant S+1 bits 4463 // are set. The pattern is rotated right by R, and repeated across a 32 or 4464 // 64-bit value, depending on destination register width. 4465 // 4466 // Put another way: the basic format of a logical immediate is a single 4467 // contiguous stretch of 1 bits, repeated across the whole word at intervals 4468 // given by a power of 2. To identify them quickly, we first locate the 4469 // lowest stretch of 1 bits, then the next 1 bit above that; that combination 4470 // is different for every logical immediate, so it gives us all the 4471 // information we need to identify the only logical immediate that our input 4472 // could be, and then we simply check if that's the value we actually have. 4473 // 4474 // (The rotation parameter does give the possibility of the stretch of 1 bits 4475 // going 'round the end' of the word. To deal with that, we observe that in 4476 // any situation where that happens the bitwise NOT of the value is also a 4477 // valid logical immediate. So we simply invert the input whenever its low bit 4478 // is set, and then we know that the rotated case can't arise.) 4479 4480 if (value & 1) { 4481 // If the low bit is 1, negate the value, and set a flag to remember that we 4482 // did (so that we can adjust the return values appropriately). 4483 negate = true; 4484 value = ~value; 4485 } 4486 4487 if (width == kWRegSizeInBits) { 4488 // To handle 32-bit logical immediates, the very easiest thing is to repeat 4489 // the input value twice to make a 64-bit word. The correct encoding of that 4490 // as a logical immediate will also be the correct encoding of the 32-bit 4491 // value. 4492 4493 // The most-significant 32 bits may not be zero (ie. negate is true) so 4494 // shift the value left before duplicating it. 4495 value <<= kWRegSizeInBits; 4496 value |= value >> kWRegSizeInBits; 4497 } 4498 4499 // The basic analysis idea: imagine our input word looks like this. 4500 // 4501 // 0011111000111110001111100011111000111110001111100011111000111110 4502 // c b a 4503 // |<--d-->| 4504 // 4505 // We find the lowest set bit (as an actual power-of-2 value, not its index) 4506 // and call it a. Then we add a to our original number, which wipes out the 4507 // bottommost stretch of set bits and replaces it with a 1 carried into the 4508 // next zero bit. Then we look for the new lowest set bit, which is in 4509 // position b, and subtract it, so now our number is just like the original 4510 // but with the lowest stretch of set bits completely gone. Now we find the 4511 // lowest set bit again, which is position c in the diagram above. Then we'll 4512 // measure the distance d between bit positions a and c (using CLZ), and that 4513 // tells us that the only valid logical immediate that could possibly be equal 4514 // to this number is the one in which a stretch of bits running from a to just 4515 // below b is replicated every d bits. 4516 uint64_t a = LargestPowerOf2Divisor(value); 4517 uint64_t value_plus_a = value + a; 4518 uint64_t b = LargestPowerOf2Divisor(value_plus_a); 4519 uint64_t value_plus_a_minus_b = value_plus_a - b; 4520 uint64_t c = LargestPowerOf2Divisor(value_plus_a_minus_b); 4521 4522 int d, clz_a, out_n; 4523 uint64_t mask; 4524 4525 if (c != 0) { 4526 // The general case, in which there is more than one stretch of set bits. 4527 // Compute the repeat distance d, and set up a bitmask covering the basic 4528 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all 4529 // of these cases the N bit of the output will be zero. 4530 clz_a = CountLeadingZeros(a, kXRegSizeInBits); 4531 int clz_c = CountLeadingZeros(c, kXRegSizeInBits); 4532 d = clz_a - clz_c; 4533 mask = ((uint64_t{1} << d) - 1); 4534 out_n = 0; 4535 } else { 4536 // Handle degenerate cases. 4537 // 4538 // If any of those 'find lowest set bit' operations didn't find a set bit at 4539 // all, then the word will have been zero thereafter, so in particular the 4540 // last lowest_set_bit operation will have returned zero. So we can test for 4541 // all the special case conditions in one go by seeing if c is zero. 4542 if (a == 0) { 4543 // The input was zero (or all 1 bits, which will come to here too after we 4544 // inverted it at the start of the function), for which we just return 4545 // false. 4546 return false; 4547 } else { 4548 // Otherwise, if c was zero but a was not, then there's just one stretch 4549 // of set bits in our word, meaning that we have the trivial case of 4550 // d == 64 and only one 'repetition'. Set up all the same variables as in 4551 // the general case above, and set the N bit in the output. 4552 clz_a = CountLeadingZeros(a, kXRegSizeInBits); 4553 d = 64; 4554 mask = ~uint64_t{0}; 4555 out_n = 1; 4556 } 4557 } 4558 4559 // If the repeat period d is not a power of two, it can't be encoded. 4560 if (!base::bits::IsPowerOfTwo(d)) { 4561 return false; 4562 } 4563 4564 if (((b - a) & ~mask) != 0) { 4565 // If the bit stretch (b - a) does not fit within the mask derived from the 4566 // repeat period, then fail. 4567 return false; 4568 } 4569 4570 // The only possible option is b - a repeated every d bits. Now we're going to 4571 // actually construct the valid logical immediate derived from that 4572 // specification, and see if it equals our original input. 4573 // 4574 // To repeat a value every d bits, we multiply it by a number of the form 4575 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can 4576 // be derived using a table lookup on CLZ(d). 4577 static const uint64_t multipliers[] = { 4578 0x0000000000000001UL, 4579 0x0000000100000001UL, 4580 0x0001000100010001UL, 4581 0x0101010101010101UL, 4582 0x1111111111111111UL, 4583 0x5555555555555555UL, 4584 }; 4585 int multiplier_idx = CountLeadingZeros(d, kXRegSizeInBits) - 57; 4586 // Ensure that the index to the multipliers array is within bounds. 4587 DCHECK((multiplier_idx >= 0) && 4588 (static_cast<size_t>(multiplier_idx) < arraysize(multipliers))); 4589 uint64_t multiplier = multipliers[multiplier_idx]; 4590 uint64_t candidate = (b - a) * multiplier; 4591 4592 if (value != candidate) { 4593 // The candidate pattern doesn't match our input value, so fail. 4594 return false; 4595 } 4596 4597 // We have a match! This is a valid logical immediate, so now we have to 4598 // construct the bits and pieces of the instruction encoding that generates 4599 // it. 4600 4601 // Count the set bits in our basic stretch. The special case of clz(0) == -1 4602 // makes the answer come out right for stretches that reach the very top of 4603 // the word (e.g. numbers like 0xFFFFC00000000000). 4604 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSizeInBits); 4605 int s = clz_a - clz_b; 4606 4607 // Decide how many bits to rotate right by, to put the low bit of that basic 4608 // stretch in position a. 4609 int r; 4610 if (negate) { 4611 // If we inverted the input right at the start of this function, here's 4612 // where we compensate: the number of set bits becomes the number of clear 4613 // bits, and the rotation count is based on position b rather than position 4614 // a (since b is the location of the 'lowest' 1 bit after inversion). 4615 s = d - s; 4616 r = (clz_b + 1) & (d - 1); 4617 } else { 4618 r = (clz_a + 1) & (d - 1); 4619 } 4620 4621 // Now we're done, except for having to encode the S output in such a way that 4622 // it gives both the number of set bits and the length of the repeated 4623 // segment. The s field is encoded like this: 4624 // 4625 // imms size S 4626 // ssssss 64 UInt(ssssss) 4627 // 0sssss 32 UInt(sssss) 4628 // 10ssss 16 UInt(ssss) 4629 // 110sss 8 UInt(sss) 4630 // 1110ss 4 UInt(ss) 4631 // 11110s 2 UInt(s) 4632 // 4633 // So we 'or' (-d << 1) with our computed s to form imms. 4634 *n = out_n; 4635 *imm_s = ((-d << 1) | (s - 1)) & 0x3F; 4636 *imm_r = r; 4637 4638 return true; 4639 } 4640 4641 4642 bool Assembler::IsImmConditionalCompare(int64_t immediate) { 4643 return is_uint5(immediate); 4644 } 4645 4646 4647 bool Assembler::IsImmFP32(float imm) { 4648 // Valid values will have the form: 4649 // aBbb.bbbc.defg.h000.0000.0000.0000.0000 4650 uint32_t bits = bit_cast<uint32_t>(imm); 4651 // bits[19..0] are cleared. 4652 if ((bits & 0x7FFFF) != 0) { 4653 return false; 4654 } 4655 4656 // bits[29..25] are all set or all cleared. 4657 uint32_t b_pattern = (bits >> 16) & 0x3E00; 4658 if (b_pattern != 0 && b_pattern != 0x3E00) { 4659 return false; 4660 } 4661 4662 // bit[30] and bit[29] are opposite. 4663 if (((bits ^ (bits << 1)) & 0x40000000) == 0) { 4664 return false; 4665 } 4666 4667 return true; 4668 } 4669 4670 4671 bool Assembler::IsImmFP64(double imm) { 4672 // Valid values will have the form: 4673 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 4674 // 0000.0000.0000.0000.0000.0000.0000.0000 4675 uint64_t bits = bit_cast<uint64_t>(imm); 4676 // bits[47..0] are cleared. 4677 if ((bits & 0xFFFFFFFFFFFFL) != 0) { 4678 return false; 4679 } 4680 4681 // bits[61..54] are all set or all cleared. 4682 uint32_t b_pattern = (bits >> 48) & 0x3FC0; 4683 if (b_pattern != 0 && b_pattern != 0x3FC0) { 4684 return false; 4685 } 4686 4687 // bit[62] and bit[61] are opposite. 4688 if (((bits ^ (bits << 1)) & 0x4000000000000000L) == 0) { 4689 return false; 4690 } 4691 4692 return true; 4693 } 4694 4695 4696 void Assembler::GrowBuffer() { 4697 if (!own_buffer_) FATAL("external code buffer is too small"); 4698 4699 // Compute new buffer size. 4700 CodeDesc desc; // the new buffer 4701 if (buffer_size_ < 1 * MB) { 4702 desc.buffer_size = 2 * buffer_size_; 4703 } else { 4704 desc.buffer_size = buffer_size_ + 1 * MB; 4705 } 4706 4707 // Some internal data structures overflow for very large buffers, 4708 // they must ensure that kMaximalBufferSize is not too large. 4709 if (desc.buffer_size > kMaximalBufferSize) { 4710 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer"); 4711 } 4712 4713 byte* buffer = reinterpret_cast<byte*>(buffer_); 4714 4715 // Set up new buffer. 4716 desc.buffer = NewArray<byte>(desc.buffer_size); 4717 desc.origin = this; 4718 4719 desc.instr_size = pc_offset(); 4720 desc.reloc_size = 4721 static_cast<int>((buffer + buffer_size_) - reloc_info_writer.pos()); 4722 4723 // Copy the data. 4724 intptr_t pc_delta = desc.buffer - buffer; 4725 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 4726 (buffer + buffer_size_); 4727 memmove(desc.buffer, buffer, desc.instr_size); 4728 memmove(reloc_info_writer.pos() + rc_delta, 4729 reloc_info_writer.pos(), desc.reloc_size); 4730 4731 // Switch buffers. 4732 DeleteArray(buffer_); 4733 buffer_ = desc.buffer; 4734 buffer_size_ = desc.buffer_size; 4735 pc_ = pc_ + pc_delta; 4736 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 4737 reloc_info_writer.last_pc() + pc_delta); 4738 4739 // None of our relocation types are pc relative pointing outside the code 4740 // buffer nor pc absolute pointing inside the code buffer, so there is no need 4741 // to relocate any emitted relocation entries. 4742 4743 // Relocate internal references. 4744 for (auto pos : internal_reference_positions_) { 4745 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 4746 *p += pc_delta; 4747 } 4748 4749 // Pending relocation entries are also relative, no need to relocate. 4750 } 4751 4752 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data, 4753 ConstantPoolMode constant_pool_mode) { 4754 // Non-relocatable constants should not end up in the literal pool. 4755 DCHECK(!RelocInfo::IsNone(rmode)); 4756 if (options().disable_reloc_info_for_patching) return; 4757 4758 // We do not try to reuse pool constants. 4759 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, nullptr); 4760 bool write_reloc_info = true; 4761 4762 if ((rmode == RelocInfo::COMMENT) || 4763 (rmode == RelocInfo::INTERNAL_REFERENCE) || 4764 (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) || 4765 (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) || 4766 (rmode == RelocInfo::DEOPT_INLINING_ID) || 4767 (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID)) { 4768 // Adjust code for new modes. 4769 DCHECK(RelocInfo::IsComment(rmode) || RelocInfo::IsDeoptReason(rmode) || 4770 RelocInfo::IsDeoptId(rmode) || RelocInfo::IsDeoptPosition(rmode) || 4771 RelocInfo::IsInternalReference(rmode) || 4772 RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode)); 4773 // These modes do not need an entry in the constant pool. 4774 } else if (constant_pool_mode == NEEDS_POOL_ENTRY) { 4775 write_reloc_info = constpool_.RecordEntry(data, rmode); 4776 // Make sure the constant pool is not emitted in place of the next 4777 // instruction for which we just recorded relocation info. 4778 BlockConstPoolFor(1); 4779 } 4780 // For modes that cannot use the constant pool, a different sequence of 4781 // instructions will be emitted by this function's caller. 4782 4783 if (write_reloc_info) { 4784 // Don't record external references unless the heap will be serialized. 4785 if (RelocInfo::IsOnlyForSerializer(rmode) && 4786 !options().record_reloc_info_for_serialization && !emit_debug_code()) { 4787 return; 4788 } 4789 DCHECK_GE(buffer_space(), kMaxRelocSize); // too late to grow buffer here 4790 reloc_info_writer.Write(&rinfo); 4791 } 4792 } 4793 4794 void Assembler::near_jump(int offset, RelocInfo::Mode rmode) { 4795 if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode, offset, NO_POOL_ENTRY); 4796 b(offset); 4797 } 4798 4799 void Assembler::near_call(int offset, RelocInfo::Mode rmode) { 4800 if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode, offset, NO_POOL_ENTRY); 4801 bl(offset); 4802 } 4803 4804 void Assembler::near_call(HeapObjectRequest request) { 4805 RequestHeapObject(request); 4806 int index = AddCodeTarget(Handle<Code>()); 4807 RecordRelocInfo(RelocInfo::CODE_TARGET, index, NO_POOL_ENTRY); 4808 bl(index); 4809 } 4810 4811 void Assembler::BlockConstPoolFor(int instructions) { 4812 int pc_limit = pc_offset() + instructions * kInstrSize; 4813 if (no_const_pool_before_ < pc_limit) { 4814 no_const_pool_before_ = pc_limit; 4815 // Make sure the pool won't be blocked for too long. 4816 DCHECK(pc_limit < constpool_.MaxPcOffset()); 4817 } 4818 4819 if (next_constant_pool_check_ < no_const_pool_before_) { 4820 next_constant_pool_check_ = no_const_pool_before_; 4821 } 4822 } 4823 4824 4825 void Assembler::CheckConstPool(bool force_emit, bool require_jump) { 4826 // Some short sequence of instruction mustn't be broken up by constant pool 4827 // emission, such sequences are protected by calls to BlockConstPoolFor and 4828 // BlockConstPoolScope. 4829 if (is_const_pool_blocked()) { 4830 // Something is wrong if emission is forced and blocked at the same time. 4831 DCHECK(!force_emit); 4832 return; 4833 } 4834 4835 // There is nothing to do if there are no pending constant pool entries. 4836 if (constpool_.IsEmpty()) { 4837 // Calculate the offset of the next check. 4838 SetNextConstPoolCheckIn(kCheckConstPoolInterval); 4839 return; 4840 } 4841 4842 // We emit a constant pool when: 4843 // * requested to do so by parameter force_emit (e.g. after each function). 4844 // * the distance to the first instruction accessing the constant pool is 4845 // kApproxMaxDistToConstPool or more. 4846 // * the number of entries in the pool is kApproxMaxPoolEntryCount or more. 4847 int dist = constpool_.DistanceToFirstUse(); 4848 int count = constpool_.EntryCount(); 4849 if (!force_emit && 4850 (dist < kApproxMaxDistToConstPool) && 4851 (count < kApproxMaxPoolEntryCount)) { 4852 return; 4853 } 4854 4855 4856 // Emit veneers for branches that would go out of range during emission of the 4857 // constant pool. 4858 int worst_case_size = constpool_.WorstCaseSize(); 4859 CheckVeneerPool(false, require_jump, 4860 kVeneerDistanceMargin + worst_case_size); 4861 4862 // Check that the code buffer is large enough before emitting the constant 4863 // pool (this includes the gap to the relocation information). 4864 int needed_space = worst_case_size + kGap + 1 * kInstrSize; 4865 while (buffer_space() <= needed_space) { 4866 GrowBuffer(); 4867 } 4868 4869 Label size_check; 4870 bind(&size_check); 4871 constpool_.Emit(require_jump); 4872 DCHECK(SizeOfCodeGeneratedSince(&size_check) <= 4873 static_cast<unsigned>(worst_case_size)); 4874 4875 // Since a constant pool was just emitted, move the check offset forward by 4876 // the standard interval. 4877 SetNextConstPoolCheckIn(kCheckConstPoolInterval); 4878 } 4879 4880 4881 bool Assembler::ShouldEmitVeneer(int max_reachable_pc, int margin) { 4882 // Account for the branch around the veneers and the guard. 4883 int protection_offset = 2 * kInstrSize; 4884 return pc_offset() > max_reachable_pc - margin - protection_offset - 4885 static_cast<int>(unresolved_branches_.size() * kMaxVeneerCodeSize); 4886 } 4887 4888 4889 void Assembler::RecordVeneerPool(int location_offset, int size) { 4890 RelocInfo rinfo(reinterpret_cast<Address>(buffer_) + location_offset, 4891 RelocInfo::VENEER_POOL, static_cast<intptr_t>(size), nullptr); 4892 reloc_info_writer.Write(&rinfo); 4893 } 4894 4895 4896 void Assembler::EmitVeneers(bool force_emit, bool need_protection, int margin) { 4897 BlockPoolsScope scope(this); 4898 RecordComment("[ Veneers"); 4899 4900 // The exact size of the veneer pool must be recorded (see the comment at the 4901 // declaration site of RecordConstPool()), but computing the number of 4902 // veneers that will be generated is not obvious. So instead we remember the 4903 // current position and will record the size after the pool has been 4904 // generated. 4905 Label size_check; 4906 bind(&size_check); 4907 int veneer_pool_relocinfo_loc = pc_offset(); 4908 4909 Label end; 4910 if (need_protection) { 4911 b(&end); 4912 } 4913 4914 EmitVeneersGuard(); 4915 4916 Label veneer_size_check; 4917 4918 std::multimap<int, FarBranchInfo>::iterator it, it_to_delete; 4919 4920 it = unresolved_branches_.begin(); 4921 while (it != unresolved_branches_.end()) { 4922 if (force_emit || ShouldEmitVeneer(it->first, margin)) { 4923 Instruction* branch = InstructionAt(it->second.pc_offset_); 4924 Label* label = it->second.label_; 4925 4926 #ifdef DEBUG 4927 bind(&veneer_size_check); 4928 #endif 4929 // Patch the branch to point to the current position, and emit a branch 4930 // to the label. 4931 Instruction* veneer = reinterpret_cast<Instruction*>(pc_); 4932 RemoveBranchFromLabelLinkChain(branch, label, veneer); 4933 branch->SetImmPCOffsetTarget(options(), veneer); 4934 b(label); 4935 #ifdef DEBUG 4936 DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <= 4937 static_cast<uint64_t>(kMaxVeneerCodeSize)); 4938 veneer_size_check.Unuse(); 4939 #endif 4940 4941 it_to_delete = it++; 4942 unresolved_branches_.erase(it_to_delete); 4943 } else { 4944 ++it; 4945 } 4946 } 4947 4948 // Record the veneer pool size. 4949 int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check)); 4950 RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size); 4951 4952 if (unresolved_branches_.empty()) { 4953 next_veneer_pool_check_ = kMaxInt; 4954 } else { 4955 next_veneer_pool_check_ = 4956 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 4957 } 4958 4959 bind(&end); 4960 4961 RecordComment("]"); 4962 } 4963 4964 4965 void Assembler::CheckVeneerPool(bool force_emit, bool require_jump, 4966 int margin) { 4967 // There is nothing to do if there are no pending veneer pool entries. 4968 if (unresolved_branches_.empty()) { 4969 DCHECK_EQ(next_veneer_pool_check_, kMaxInt); 4970 return; 4971 } 4972 4973 DCHECK(pc_offset() < unresolved_branches_first_limit()); 4974 4975 // Some short sequence of instruction mustn't be broken up by veneer pool 4976 // emission, such sequences are protected by calls to BlockVeneerPoolFor and 4977 // BlockVeneerPoolScope. 4978 if (is_veneer_pool_blocked()) { 4979 DCHECK(!force_emit); 4980 return; 4981 } 4982 4983 if (!require_jump) { 4984 // Prefer emitting veneers protected by an existing instruction. 4985 margin *= kVeneerNoProtectionFactor; 4986 } 4987 if (force_emit || ShouldEmitVeneers(margin)) { 4988 EmitVeneers(force_emit, require_jump, margin); 4989 } else { 4990 next_veneer_pool_check_ = 4991 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; 4992 } 4993 } 4994 4995 4996 int Assembler::buffer_space() const { 4997 return static_cast<int>(reloc_info_writer.pos() - pc_); 4998 } 4999 5000 5001 void Assembler::RecordConstPool(int size) { 5002 // We only need this for debugger support, to correctly compute offsets in the 5003 // code. 5004 RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size)); 5005 } 5006 5007 5008 void PatchingAssembler::PatchAdrFar(int64_t target_offset) { 5009 // The code at the current instruction should be: 5010 // adr rd, 0 5011 // nop (adr_far) 5012 // nop (adr_far) 5013 // movz scratch, 0 5014 5015 // Verify the expected code. 5016 Instruction* expected_adr = InstructionAt(0); 5017 CHECK(expected_adr->IsAdr() && (expected_adr->ImmPCRel() == 0)); 5018 int rd_code = expected_adr->Rd(); 5019 for (int i = 0; i < kAdrFarPatchableNNops; ++i) { 5020 CHECK(InstructionAt((i + 1) * kInstrSize)->IsNop(ADR_FAR_NOP)); 5021 } 5022 Instruction* expected_movz = 5023 InstructionAt((kAdrFarPatchableNInstrs - 1) * kInstrSize); 5024 CHECK(expected_movz->IsMovz() && 5025 (expected_movz->ImmMoveWide() == 0) && 5026 (expected_movz->ShiftMoveWide() == 0)); 5027 int scratch_code = expected_movz->Rd(); 5028 5029 // Patch to load the correct address. 5030 Register rd = Register::XRegFromCode(rd_code); 5031 Register scratch = Register::XRegFromCode(scratch_code); 5032 // Addresses are only 48 bits. 5033 adr(rd, target_offset & 0xFFFF); 5034 movz(scratch, (target_offset >> 16) & 0xFFFF, 16); 5035 movk(scratch, (target_offset >> 32) & 0xFFFF, 32); 5036 DCHECK_EQ(target_offset >> 48, 0); 5037 add(rd, rd, scratch); 5038 } 5039 5040 void PatchingAssembler::PatchSubSp(uint32_t immediate) { 5041 // The code at the current instruction should be: 5042 // sub sp, sp, #0 5043 5044 // Verify the expected code. 5045 Instruction* expected_adr = InstructionAt(0); 5046 CHECK(expected_adr->IsAddSubImmediate()); 5047 sub(sp, sp, immediate); 5048 } 5049 5050 } // namespace internal 5051 } // namespace v8 5052 5053 #endif // V8_TARGET_ARCH_ARM64 5054