1 // Copyright 2015, ARM Limited 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // * Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // * Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // * Neither the name of ARM Limited nor the names of its contributors may be 13 // used to endorse or promote products derived from this software without 14 // specific prior written permission. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27 28 #include <cmath> 29 #include "vixl/a64/assembler-a64.h" 30 31 namespace vixl { 32 33 // CPURegList utilities. 34 CPURegister CPURegList::PopLowestIndex() { 35 if (IsEmpty()) { 36 return NoCPUReg; 37 } 38 int index = CountTrailingZeros(list_); 39 VIXL_ASSERT((1 << index) & list_); 40 Remove(index); 41 return CPURegister(index, size_, type_); 42 } 43 44 45 CPURegister CPURegList::PopHighestIndex() { 46 VIXL_ASSERT(IsValid()); 47 if (IsEmpty()) { 48 return NoCPUReg; 49 } 50 int index = CountLeadingZeros(list_); 51 index = kRegListSizeInBits - 1 - index; 52 VIXL_ASSERT((1 << index) & list_); 53 Remove(index); 54 return CPURegister(index, size_, type_); 55 } 56 57 58 bool CPURegList::IsValid() const { 59 if ((type_ == CPURegister::kRegister) || 60 (type_ == CPURegister::kVRegister)) { 61 bool is_valid = true; 62 // Try to create a CPURegister for each element in the list. 63 for (int i = 0; i < kRegListSizeInBits; i++) { 64 if (((list_ >> i) & 1) != 0) { 65 is_valid &= CPURegister(i, size_, type_).IsValid(); 66 } 67 } 68 return is_valid; 69 } else if (type_ == CPURegister::kNoRegister) { 70 // We can't use IsEmpty here because that asserts IsValid(). 71 return list_ == 0; 72 } else { 73 return false; 74 } 75 } 76 77 78 void CPURegList::RemoveCalleeSaved() { 79 if (type() == CPURegister::kRegister) { 80 Remove(GetCalleeSaved(RegisterSizeInBits())); 81 } else if (type() == CPURegister::kVRegister) { 82 Remove(GetCalleeSavedV(RegisterSizeInBits())); 83 } else { 84 VIXL_ASSERT(type() == CPURegister::kNoRegister); 85 VIXL_ASSERT(IsEmpty()); 86 // The list must already be empty, so do nothing. 87 } 88 } 89 90 91 CPURegList CPURegList::Union(const CPURegList& list_1, 92 const CPURegList& list_2, 93 const CPURegList& list_3) { 94 return Union(list_1, Union(list_2, list_3)); 95 } 96 97 98 CPURegList CPURegList::Union(const CPURegList& list_1, 99 const CPURegList& list_2, 100 const CPURegList& list_3, 101 const CPURegList& list_4) { 102 return Union(Union(list_1, list_2), Union(list_3, list_4)); 103 } 104 105 106 CPURegList CPURegList::Intersection(const CPURegList& list_1, 107 const CPURegList& list_2, 108 const CPURegList& list_3) { 109 return Intersection(list_1, Intersection(list_2, list_3)); 110 } 111 112 113 CPURegList CPURegList::Intersection(const CPURegList& list_1, 114 const CPURegList& list_2, 115 const CPURegList& list_3, 116 const CPURegList& list_4) { 117 return Intersection(Intersection(list_1, list_2), 118 Intersection(list_3, list_4)); 119 } 120 121 122 CPURegList CPURegList::GetCalleeSaved(unsigned size) { 123 return CPURegList(CPURegister::kRegister, size, 19, 29); 124 } 125 126 127 CPURegList CPURegList::GetCalleeSavedV(unsigned size) { 128 return CPURegList(CPURegister::kVRegister, size, 8, 15); 129 } 130 131 132 CPURegList CPURegList::GetCallerSaved(unsigned size) { 133 // Registers x0-x18 and lr (x30) are caller-saved. 134 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18); 135 // Temporary workaround for bug 21273194: Create a Register object equivalent 136 // to register lr (x30) 137 // list.Combine(lr); 138 list.Combine(Register(30, size)); 139 return list; 140 } 141 142 143 CPURegList CPURegList::GetCallerSavedV(unsigned size) { 144 // Registers d0-d7 and d16-d31 are caller-saved. 145 CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7); 146 list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31)); 147 return list; 148 } 149 150 151 const CPURegList kCalleeSaved = CPURegList::GetCalleeSaved(); 152 const CPURegList kCalleeSavedV = CPURegList::GetCalleeSavedV(); 153 const CPURegList kCallerSaved = CPURegList::GetCallerSaved(); 154 const CPURegList kCallerSavedV = CPURegList::GetCallerSavedV(); 155 156 157 // Registers. 158 #define WREG(n) w##n, 159 const Register Register::wregisters[] = { 160 REGISTER_CODE_LIST(WREG) 161 }; 162 #undef WREG 163 164 #define XREG(n) x##n, 165 const Register Register::xregisters[] = { 166 REGISTER_CODE_LIST(XREG) 167 }; 168 #undef XREG 169 170 #define BREG(n) b##n, 171 const VRegister VRegister::bregisters[] = { 172 REGISTER_CODE_LIST(BREG) 173 }; 174 #undef BREG 175 176 #define HREG(n) h##n, 177 const VRegister VRegister::hregisters[] = { 178 REGISTER_CODE_LIST(HREG) 179 }; 180 #undef HREG 181 182 #define SREG(n) s##n, 183 const VRegister VRegister::sregisters[] = { 184 REGISTER_CODE_LIST(SREG) 185 }; 186 #undef SREG 187 188 #define DREG(n) d##n, 189 const VRegister VRegister::dregisters[] = { 190 REGISTER_CODE_LIST(DREG) 191 }; 192 #undef DREG 193 194 #define QREG(n) q##n, 195 const VRegister VRegister::qregisters[] = { 196 REGISTER_CODE_LIST(QREG) 197 }; 198 #undef QREG 199 200 #define VREG(n) v##n, 201 const VRegister VRegister::vregisters[] = { 202 REGISTER_CODE_LIST(VREG) 203 }; 204 #undef VREG 205 206 207 const Register& Register::WRegFromCode(unsigned code) { 208 if (code == kSPRegInternalCode) { 209 return wsp; 210 } else { 211 VIXL_ASSERT(code < kNumberOfRegisters); 212 return wregisters[code]; 213 } 214 } 215 216 217 const Register& Register::XRegFromCode(unsigned code) { 218 if (code == kSPRegInternalCode) { 219 return sp; 220 } else { 221 VIXL_ASSERT(code < kNumberOfRegisters); 222 return xregisters[code]; 223 } 224 } 225 226 227 const VRegister& VRegister::BRegFromCode(unsigned code) { 228 VIXL_ASSERT(code < kNumberOfVRegisters); 229 return bregisters[code]; 230 } 231 232 233 const VRegister& VRegister::HRegFromCode(unsigned code) { 234 VIXL_ASSERT(code < kNumberOfVRegisters); 235 return hregisters[code]; 236 } 237 238 239 const VRegister& VRegister::SRegFromCode(unsigned code) { 240 VIXL_ASSERT(code < kNumberOfVRegisters); 241 return sregisters[code]; 242 } 243 244 245 const VRegister& VRegister::DRegFromCode(unsigned code) { 246 VIXL_ASSERT(code < kNumberOfVRegisters); 247 return dregisters[code]; 248 } 249 250 251 const VRegister& VRegister::QRegFromCode(unsigned code) { 252 VIXL_ASSERT(code < kNumberOfVRegisters); 253 return qregisters[code]; 254 } 255 256 257 const VRegister& VRegister::VRegFromCode(unsigned code) { 258 VIXL_ASSERT(code < kNumberOfVRegisters); 259 return vregisters[code]; 260 } 261 262 263 const Register& CPURegister::W() const { 264 VIXL_ASSERT(IsValidRegister()); 265 return Register::WRegFromCode(code_); 266 } 267 268 269 const Register& CPURegister::X() const { 270 VIXL_ASSERT(IsValidRegister()); 271 return Register::XRegFromCode(code_); 272 } 273 274 275 const VRegister& CPURegister::B() const { 276 VIXL_ASSERT(IsValidVRegister()); 277 return VRegister::BRegFromCode(code_); 278 } 279 280 281 const VRegister& CPURegister::H() const { 282 VIXL_ASSERT(IsValidVRegister()); 283 return VRegister::HRegFromCode(code_); 284 } 285 286 287 const VRegister& CPURegister::S() const { 288 VIXL_ASSERT(IsValidVRegister()); 289 return VRegister::SRegFromCode(code_); 290 } 291 292 293 const VRegister& CPURegister::D() const { 294 VIXL_ASSERT(IsValidVRegister()); 295 return VRegister::DRegFromCode(code_); 296 } 297 298 299 const VRegister& CPURegister::Q() const { 300 VIXL_ASSERT(IsValidVRegister()); 301 return VRegister::QRegFromCode(code_); 302 } 303 304 305 const VRegister& CPURegister::V() const { 306 VIXL_ASSERT(IsValidVRegister()); 307 return VRegister::VRegFromCode(code_); 308 } 309 310 311 // Operand. 312 Operand::Operand(int64_t immediate) 313 : immediate_(immediate), 314 reg_(NoReg), 315 shift_(NO_SHIFT), 316 extend_(NO_EXTEND), 317 shift_amount_(0) {} 318 319 320 Operand::Operand(Register reg, Shift shift, unsigned shift_amount) 321 : reg_(reg), 322 shift_(shift), 323 extend_(NO_EXTEND), 324 shift_amount_(shift_amount) { 325 VIXL_ASSERT(shift != MSL); 326 VIXL_ASSERT(reg.Is64Bits() || (shift_amount < kWRegSize)); 327 VIXL_ASSERT(reg.Is32Bits() || (shift_amount < kXRegSize)); 328 VIXL_ASSERT(!reg.IsSP()); 329 } 330 331 332 Operand::Operand(Register reg, Extend extend, unsigned shift_amount) 333 : reg_(reg), 334 shift_(NO_SHIFT), 335 extend_(extend), 336 shift_amount_(shift_amount) { 337 VIXL_ASSERT(reg.IsValid()); 338 VIXL_ASSERT(shift_amount <= 4); 339 VIXL_ASSERT(!reg.IsSP()); 340 341 // Extend modes SXTX and UXTX require a 64-bit register. 342 VIXL_ASSERT(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX))); 343 } 344 345 346 bool Operand::IsImmediate() const { 347 return reg_.Is(NoReg); 348 } 349 350 351 bool Operand::IsShiftedRegister() const { 352 return reg_.IsValid() && (shift_ != NO_SHIFT); 353 } 354 355 356 bool Operand::IsExtendedRegister() const { 357 return reg_.IsValid() && (extend_ != NO_EXTEND); 358 } 359 360 361 bool Operand::IsZero() const { 362 if (IsImmediate()) { 363 return immediate() == 0; 364 } else { 365 return reg().IsZero(); 366 } 367 } 368 369 370 Operand Operand::ToExtendedRegister() const { 371 VIXL_ASSERT(IsShiftedRegister()); 372 VIXL_ASSERT((shift_ == LSL) && (shift_amount_ <= 4)); 373 return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_); 374 } 375 376 377 // MemOperand 378 MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode) 379 : base_(base), regoffset_(NoReg), offset_(offset), addrmode_(addrmode) { 380 VIXL_ASSERT(base.Is64Bits() && !base.IsZero()); 381 } 382 383 384 MemOperand::MemOperand(Register base, 385 Register regoffset, 386 Extend extend, 387 unsigned shift_amount) 388 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset), 389 shift_(NO_SHIFT), extend_(extend), shift_amount_(shift_amount) { 390 VIXL_ASSERT(base.Is64Bits() && !base.IsZero()); 391 VIXL_ASSERT(!regoffset.IsSP()); 392 VIXL_ASSERT((extend == UXTW) || (extend == SXTW) || (extend == SXTX)); 393 394 // SXTX extend mode requires a 64-bit offset register. 395 VIXL_ASSERT(regoffset.Is64Bits() || (extend != SXTX)); 396 } 397 398 399 MemOperand::MemOperand(Register base, 400 Register regoffset, 401 Shift shift, 402 unsigned shift_amount) 403 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset), 404 shift_(shift), extend_(NO_EXTEND), shift_amount_(shift_amount) { 405 VIXL_ASSERT(base.Is64Bits() && !base.IsZero()); 406 VIXL_ASSERT(regoffset.Is64Bits() && !regoffset.IsSP()); 407 VIXL_ASSERT(shift == LSL); 408 } 409 410 411 MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode) 412 : base_(base), regoffset_(NoReg), addrmode_(addrmode) { 413 VIXL_ASSERT(base.Is64Bits() && !base.IsZero()); 414 415 if (offset.IsImmediate()) { 416 offset_ = offset.immediate(); 417 } else if (offset.IsShiftedRegister()) { 418 VIXL_ASSERT((addrmode == Offset) || (addrmode == PostIndex)); 419 420 regoffset_ = offset.reg(); 421 shift_ = offset.shift(); 422 shift_amount_ = offset.shift_amount(); 423 424 extend_ = NO_EXTEND; 425 offset_ = 0; 426 427 // These assertions match those in the shifted-register constructor. 428 VIXL_ASSERT(regoffset_.Is64Bits() && !regoffset_.IsSP()); 429 VIXL_ASSERT(shift_ == LSL); 430 } else { 431 VIXL_ASSERT(offset.IsExtendedRegister()); 432 VIXL_ASSERT(addrmode == Offset); 433 434 regoffset_ = offset.reg(); 435 extend_ = offset.extend(); 436 shift_amount_ = offset.shift_amount(); 437 438 shift_ = NO_SHIFT; 439 offset_ = 0; 440 441 // These assertions match those in the extended-register constructor. 442 VIXL_ASSERT(!regoffset_.IsSP()); 443 VIXL_ASSERT((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX)); 444 VIXL_ASSERT((regoffset_.Is64Bits() || (extend_ != SXTX))); 445 } 446 } 447 448 449 bool MemOperand::IsImmediateOffset() const { 450 return (addrmode_ == Offset) && regoffset_.Is(NoReg); 451 } 452 453 454 bool MemOperand::IsRegisterOffset() const { 455 return (addrmode_ == Offset) && !regoffset_.Is(NoReg); 456 } 457 458 459 bool MemOperand::IsPreIndex() const { 460 return addrmode_ == PreIndex; 461 } 462 463 464 bool MemOperand::IsPostIndex() const { 465 return addrmode_ == PostIndex; 466 } 467 468 469 void MemOperand::AddOffset(int64_t offset) { 470 VIXL_ASSERT(IsImmediateOffset()); 471 offset_ += offset; 472 } 473 474 475 // Assembler 476 Assembler::Assembler(byte* buffer, size_t capacity, 477 PositionIndependentCodeOption pic) 478 : pic_(pic) { 479 #ifdef VIXL_DEBUG 480 buffer_monitor_ = 0; 481 #endif 482 buffer_ = new CodeBuffer(buffer, capacity); 483 } 484 485 486 Assembler::Assembler(size_t capacity, PositionIndependentCodeOption pic) 487 : pic_(pic) { 488 #ifdef VIXL_DEBUG 489 buffer_monitor_ = 0; 490 #endif 491 buffer_ = new CodeBuffer(capacity); 492 } 493 494 495 Assembler::~Assembler() { 496 VIXL_ASSERT(buffer_monitor_ == 0); 497 delete buffer_; 498 } 499 500 501 void Assembler::Reset() { 502 buffer_->Reset(); 503 } 504 505 506 void Assembler::FinalizeCode() { 507 buffer_->SetClean(); 508 } 509 510 511 void Assembler::bind(Label* label) { 512 BindToOffset(label, buffer_->CursorOffset()); 513 } 514 515 516 void Assembler::BindToOffset(Label* label, ptrdiff_t offset) { 517 VIXL_ASSERT((offset >= 0) && (offset <= buffer_->CursorOffset())); 518 VIXL_ASSERT(offset % kInstructionSize == 0); 519 520 label->Bind(offset); 521 522 for (Label::LabelLinksIterator it(label); !it.Done(); it.Advance()) { 523 Instruction* link = GetOffsetAddress<Instruction*>(*it.Current()); 524 link->SetImmPCOffsetTarget(GetLabelAddress<Instruction*>(label)); 525 } 526 label->ClearAllLinks(); 527 } 528 529 530 // A common implementation for the LinkAndGet<Type>OffsetTo helpers. 531 // 532 // The offset is calculated by aligning the PC and label addresses down to a 533 // multiple of 1 << element_shift, then calculating the (scaled) offset between 534 // them. This matches the semantics of adrp, for example. 535 template <int element_shift> 536 ptrdiff_t Assembler::LinkAndGetOffsetTo(Label* label) { 537 VIXL_STATIC_ASSERT(element_shift < (sizeof(ptrdiff_t) * 8)); 538 539 if (label->IsBound()) { 540 uintptr_t pc_offset = GetCursorAddress<uintptr_t>() >> element_shift; 541 uintptr_t label_offset = 542 GetLabelAddress<uintptr_t>(label) >> element_shift; 543 return label_offset - pc_offset; 544 } else { 545 label->AddLink(buffer_->CursorOffset()); 546 return 0; 547 } 548 } 549 550 551 ptrdiff_t Assembler::LinkAndGetByteOffsetTo(Label* label) { 552 return LinkAndGetOffsetTo<0>(label); 553 } 554 555 556 ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(Label* label) { 557 return LinkAndGetOffsetTo<kInstructionSizeLog2>(label); 558 } 559 560 561 ptrdiff_t Assembler::LinkAndGetPageOffsetTo(Label* label) { 562 return LinkAndGetOffsetTo<kPageSizeLog2>(label); 563 } 564 565 566 void Assembler::place(RawLiteral* literal) { 567 VIXL_ASSERT(!literal->IsPlaced()); 568 569 // Patch instructions using this literal. 570 if (literal->IsUsed()) { 571 Instruction* target = GetCursorAddress<Instruction*>(); 572 ptrdiff_t offset = literal->last_use(); 573 bool done; 574 do { 575 Instruction* ldr = GetOffsetAddress<Instruction*>(offset); 576 VIXL_ASSERT(ldr->IsLoadLiteral()); 577 578 ptrdiff_t imm19 = ldr->ImmLLiteral(); 579 VIXL_ASSERT(imm19 <= 0); 580 done = (imm19 == 0); 581 offset += imm19 * kLiteralEntrySize; 582 583 ldr->SetImmLLiteral(target); 584 } while (!done); 585 } 586 587 // "bind" the literal. 588 literal->set_offset(CursorOffset()); 589 // Copy the data into the pool. 590 switch (literal->size()) { 591 case kSRegSizeInBytes: dc32(literal->raw_value32()); break; 592 case kDRegSizeInBytes: dc64(literal->raw_value64()); break; 593 default: 594 VIXL_ASSERT(literal->size() == kQRegSizeInBytes); 595 dc64(literal->raw_value128_low64()); 596 dc64(literal->raw_value128_high64()); 597 } 598 } 599 600 601 ptrdiff_t Assembler::LinkAndGetWordOffsetTo(RawLiteral* literal) { 602 VIXL_ASSERT(IsWordAligned(CursorOffset())); 603 604 if (literal->IsPlaced()) { 605 // The literal is "behind", the offset will be negative. 606 VIXL_ASSERT((literal->offset() - CursorOffset()) <= 0); 607 return (literal->offset() - CursorOffset()) >> kLiteralEntrySizeLog2; 608 } 609 610 ptrdiff_t offset = 0; 611 // Link all uses together. 612 if (literal->IsUsed()) { 613 offset = (literal->last_use() - CursorOffset()) >> kLiteralEntrySizeLog2; 614 } 615 literal->set_last_use(CursorOffset()); 616 617 return offset; 618 } 619 620 621 // Code generation. 622 void Assembler::br(const Register& xn) { 623 VIXL_ASSERT(xn.Is64Bits()); 624 Emit(BR | Rn(xn)); 625 } 626 627 628 void Assembler::blr(const Register& xn) { 629 VIXL_ASSERT(xn.Is64Bits()); 630 Emit(BLR | Rn(xn)); 631 } 632 633 634 void Assembler::ret(const Register& xn) { 635 VIXL_ASSERT(xn.Is64Bits()); 636 Emit(RET | Rn(xn)); 637 } 638 639 640 void Assembler::b(int imm26) { 641 Emit(B | ImmUncondBranch(imm26)); 642 } 643 644 645 void Assembler::b(int imm19, Condition cond) { 646 Emit(B_cond | ImmCondBranch(imm19) | cond); 647 } 648 649 650 void Assembler::b(Label* label) { 651 b(LinkAndGetInstructionOffsetTo(label)); 652 } 653 654 655 void Assembler::b(Label* label, Condition cond) { 656 b(LinkAndGetInstructionOffsetTo(label), cond); 657 } 658 659 660 void Assembler::bl(int imm26) { 661 Emit(BL | ImmUncondBranch(imm26)); 662 } 663 664 665 void Assembler::bl(Label* label) { 666 bl(LinkAndGetInstructionOffsetTo(label)); 667 } 668 669 670 void Assembler::cbz(const Register& rt, 671 int imm19) { 672 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt)); 673 } 674 675 676 void Assembler::cbz(const Register& rt, 677 Label* label) { 678 cbz(rt, LinkAndGetInstructionOffsetTo(label)); 679 } 680 681 682 void Assembler::cbnz(const Register& rt, 683 int imm19) { 684 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt)); 685 } 686 687 688 void Assembler::cbnz(const Register& rt, 689 Label* label) { 690 cbnz(rt, LinkAndGetInstructionOffsetTo(label)); 691 } 692 693 694 void Assembler::NEONTable(const VRegister& vd, 695 const VRegister& vn, 696 const VRegister& vm, 697 NEONTableOp op) { 698 VIXL_ASSERT(vd.Is16B() || vd.Is8B()); 699 VIXL_ASSERT(vn.Is16B()); 700 VIXL_ASSERT(AreSameFormat(vd, vm)); 701 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd)); 702 } 703 704 705 void Assembler::tbl(const VRegister& vd, 706 const VRegister& vn, 707 const VRegister& vm) { 708 NEONTable(vd, vn, vm, NEON_TBL_1v); 709 } 710 711 712 void Assembler::tbl(const VRegister& vd, 713 const VRegister& vn, 714 const VRegister& vn2, 715 const VRegister& vm) { 716 USE(vn2); 717 VIXL_ASSERT(AreSameFormat(vn, vn2)); 718 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters)); 719 720 NEONTable(vd, vn, vm, NEON_TBL_2v); 721 } 722 723 724 void Assembler::tbl(const VRegister& vd, 725 const VRegister& vn, 726 const VRegister& vn2, 727 const VRegister& vn3, 728 const VRegister& vm) { 729 USE(vn2); 730 USE(vn3); 731 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3)); 732 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters)); 733 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters)); 734 735 NEONTable(vd, vn, vm, NEON_TBL_3v); 736 } 737 738 739 void Assembler::tbl(const VRegister& vd, 740 const VRegister& vn, 741 const VRegister& vn2, 742 const VRegister& vn3, 743 const VRegister& vn4, 744 const VRegister& vm) { 745 USE(vn2); 746 USE(vn3); 747 USE(vn4); 748 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4)); 749 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters)); 750 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters)); 751 VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters)); 752 753 NEONTable(vd, vn, vm, NEON_TBL_4v); 754 } 755 756 757 void Assembler::tbx(const VRegister& vd, 758 const VRegister& vn, 759 const VRegister& vm) { 760 NEONTable(vd, vn, vm, NEON_TBX_1v); 761 } 762 763 764 void Assembler::tbx(const VRegister& vd, 765 const VRegister& vn, 766 const VRegister& vn2, 767 const VRegister& vm) { 768 USE(vn2); 769 VIXL_ASSERT(AreSameFormat(vn, vn2)); 770 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters)); 771 772 NEONTable(vd, vn, vm, NEON_TBX_2v); 773 } 774 775 776 void Assembler::tbx(const VRegister& vd, 777 const VRegister& vn, 778 const VRegister& vn2, 779 const VRegister& vn3, 780 const VRegister& vm) { 781 USE(vn2); 782 USE(vn3); 783 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3)); 784 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters)); 785 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters)); 786 787 NEONTable(vd, vn, vm, NEON_TBX_3v); 788 } 789 790 791 void Assembler::tbx(const VRegister& vd, 792 const VRegister& vn, 793 const VRegister& vn2, 794 const VRegister& vn3, 795 const VRegister& vn4, 796 const VRegister& vm) { 797 USE(vn2); 798 USE(vn3); 799 USE(vn4); 800 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4)); 801 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters)); 802 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters)); 803 VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters)); 804 805 NEONTable(vd, vn, vm, NEON_TBX_4v); 806 } 807 808 809 void Assembler::tbz(const Register& rt, 810 unsigned bit_pos, 811 int imm14) { 812 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize))); 813 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 814 } 815 816 817 void Assembler::tbz(const Register& rt, 818 unsigned bit_pos, 819 Label* label) { 820 tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 821 } 822 823 824 void Assembler::tbnz(const Register& rt, 825 unsigned bit_pos, 826 int imm14) { 827 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize))); 828 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt)); 829 } 830 831 832 void Assembler::tbnz(const Register& rt, 833 unsigned bit_pos, 834 Label* label) { 835 tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label)); 836 } 837 838 839 void Assembler::adr(const Register& rd, int imm21) { 840 VIXL_ASSERT(rd.Is64Bits()); 841 Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd)); 842 } 843 844 845 void Assembler::adr(const Register& rd, Label* label) { 846 adr(rd, LinkAndGetByteOffsetTo(label)); 847 } 848 849 850 void Assembler::adrp(const Register& rd, int imm21) { 851 VIXL_ASSERT(rd.Is64Bits()); 852 Emit(ADRP | ImmPCRelAddress(imm21) | Rd(rd)); 853 } 854 855 856 void Assembler::adrp(const Register& rd, Label* label) { 857 VIXL_ASSERT(AllowPageOffsetDependentCode()); 858 adrp(rd, LinkAndGetPageOffsetTo(label)); 859 } 860 861 862 void Assembler::add(const Register& rd, 863 const Register& rn, 864 const Operand& operand) { 865 AddSub(rd, rn, operand, LeaveFlags, ADD); 866 } 867 868 869 void Assembler::adds(const Register& rd, 870 const Register& rn, 871 const Operand& operand) { 872 AddSub(rd, rn, operand, SetFlags, ADD); 873 } 874 875 876 void Assembler::cmn(const Register& rn, 877 const Operand& operand) { 878 Register zr = AppropriateZeroRegFor(rn); 879 adds(zr, rn, operand); 880 } 881 882 883 void Assembler::sub(const Register& rd, 884 const Register& rn, 885 const Operand& operand) { 886 AddSub(rd, rn, operand, LeaveFlags, SUB); 887 } 888 889 890 void Assembler::subs(const Register& rd, 891 const Register& rn, 892 const Operand& operand) { 893 AddSub(rd, rn, operand, SetFlags, SUB); 894 } 895 896 897 void Assembler::cmp(const Register& rn, const Operand& operand) { 898 Register zr = AppropriateZeroRegFor(rn); 899 subs(zr, rn, operand); 900 } 901 902 903 void Assembler::neg(const Register& rd, const Operand& operand) { 904 Register zr = AppropriateZeroRegFor(rd); 905 sub(rd, zr, operand); 906 } 907 908 909 void Assembler::negs(const Register& rd, const Operand& operand) { 910 Register zr = AppropriateZeroRegFor(rd); 911 subs(rd, zr, operand); 912 } 913 914 915 void Assembler::adc(const Register& rd, 916 const Register& rn, 917 const Operand& operand) { 918 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC); 919 } 920 921 922 void Assembler::adcs(const Register& rd, 923 const Register& rn, 924 const Operand& operand) { 925 AddSubWithCarry(rd, rn, operand, SetFlags, ADC); 926 } 927 928 929 void Assembler::sbc(const Register& rd, 930 const Register& rn, 931 const Operand& operand) { 932 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC); 933 } 934 935 936 void Assembler::sbcs(const Register& rd, 937 const Register& rn, 938 const Operand& operand) { 939 AddSubWithCarry(rd, rn, operand, SetFlags, SBC); 940 } 941 942 943 void Assembler::ngc(const Register& rd, const Operand& operand) { 944 Register zr = AppropriateZeroRegFor(rd); 945 sbc(rd, zr, operand); 946 } 947 948 949 void Assembler::ngcs(const Register& rd, const Operand& operand) { 950 Register zr = AppropriateZeroRegFor(rd); 951 sbcs(rd, zr, operand); 952 } 953 954 955 // Logical instructions. 956 void Assembler::and_(const Register& rd, 957 const Register& rn, 958 const Operand& operand) { 959 Logical(rd, rn, operand, AND); 960 } 961 962 963 void Assembler::ands(const Register& rd, 964 const Register& rn, 965 const Operand& operand) { 966 Logical(rd, rn, operand, ANDS); 967 } 968 969 970 void Assembler::tst(const Register& rn, 971 const Operand& operand) { 972 ands(AppropriateZeroRegFor(rn), rn, operand); 973 } 974 975 976 void Assembler::bic(const Register& rd, 977 const Register& rn, 978 const Operand& operand) { 979 Logical(rd, rn, operand, BIC); 980 } 981 982 983 void Assembler::bics(const Register& rd, 984 const Register& rn, 985 const Operand& operand) { 986 Logical(rd, rn, operand, BICS); 987 } 988 989 990 void Assembler::orr(const Register& rd, 991 const Register& rn, 992 const Operand& operand) { 993 Logical(rd, rn, operand, ORR); 994 } 995 996 997 void Assembler::orn(const Register& rd, 998 const Register& rn, 999 const Operand& operand) { 1000 Logical(rd, rn, operand, ORN); 1001 } 1002 1003 1004 void Assembler::eor(const Register& rd, 1005 const Register& rn, 1006 const Operand& operand) { 1007 Logical(rd, rn, operand, EOR); 1008 } 1009 1010 1011 void Assembler::eon(const Register& rd, 1012 const Register& rn, 1013 const Operand& operand) { 1014 Logical(rd, rn, operand, EON); 1015 } 1016 1017 1018 void Assembler::lslv(const Register& rd, 1019 const Register& rn, 1020 const Register& rm) { 1021 VIXL_ASSERT(rd.size() == rn.size()); 1022 VIXL_ASSERT(rd.size() == rm.size()); 1023 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd)); 1024 } 1025 1026 1027 void Assembler::lsrv(const Register& rd, 1028 const Register& rn, 1029 const Register& rm) { 1030 VIXL_ASSERT(rd.size() == rn.size()); 1031 VIXL_ASSERT(rd.size() == rm.size()); 1032 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd)); 1033 } 1034 1035 1036 void Assembler::asrv(const Register& rd, 1037 const Register& rn, 1038 const Register& rm) { 1039 VIXL_ASSERT(rd.size() == rn.size()); 1040 VIXL_ASSERT(rd.size() == rm.size()); 1041 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd)); 1042 } 1043 1044 1045 void Assembler::rorv(const Register& rd, 1046 const Register& rn, 1047 const Register& rm) { 1048 VIXL_ASSERT(rd.size() == rn.size()); 1049 VIXL_ASSERT(rd.size() == rm.size()); 1050 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd)); 1051 } 1052 1053 1054 // Bitfield operations. 1055 void Assembler::bfm(const Register& rd, 1056 const Register& rn, 1057 unsigned immr, 1058 unsigned imms) { 1059 VIXL_ASSERT(rd.size() == rn.size()); 1060 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1061 Emit(SF(rd) | BFM | N | 1062 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd)); 1063 } 1064 1065 1066 void Assembler::sbfm(const Register& rd, 1067 const Register& rn, 1068 unsigned immr, 1069 unsigned imms) { 1070 VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits()); 1071 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1072 Emit(SF(rd) | SBFM | N | 1073 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd)); 1074 } 1075 1076 1077 void Assembler::ubfm(const Register& rd, 1078 const Register& rn, 1079 unsigned immr, 1080 unsigned imms) { 1081 VIXL_ASSERT(rd.size() == rn.size()); 1082 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1083 Emit(SF(rd) | UBFM | N | 1084 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd)); 1085 } 1086 1087 1088 void Assembler::extr(const Register& rd, 1089 const Register& rn, 1090 const Register& rm, 1091 unsigned lsb) { 1092 VIXL_ASSERT(rd.size() == rn.size()); 1093 VIXL_ASSERT(rd.size() == rm.size()); 1094 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset); 1095 Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.size()) | Rn(rn) | Rd(rd)); 1096 } 1097 1098 1099 void Assembler::csel(const Register& rd, 1100 const Register& rn, 1101 const Register& rm, 1102 Condition cond) { 1103 ConditionalSelect(rd, rn, rm, cond, CSEL); 1104 } 1105 1106 1107 void Assembler::csinc(const Register& rd, 1108 const Register& rn, 1109 const Register& rm, 1110 Condition cond) { 1111 ConditionalSelect(rd, rn, rm, cond, CSINC); 1112 } 1113 1114 1115 void Assembler::csinv(const Register& rd, 1116 const Register& rn, 1117 const Register& rm, 1118 Condition cond) { 1119 ConditionalSelect(rd, rn, rm, cond, CSINV); 1120 } 1121 1122 1123 void Assembler::csneg(const Register& rd, 1124 const Register& rn, 1125 const Register& rm, 1126 Condition cond) { 1127 ConditionalSelect(rd, rn, rm, cond, CSNEG); 1128 } 1129 1130 1131 void Assembler::cset(const Register &rd, Condition cond) { 1132 VIXL_ASSERT((cond != al) && (cond != nv)); 1133 Register zr = AppropriateZeroRegFor(rd); 1134 csinc(rd, zr, zr, InvertCondition(cond)); 1135 } 1136 1137 1138 void Assembler::csetm(const Register &rd, Condition cond) { 1139 VIXL_ASSERT((cond != al) && (cond != nv)); 1140 Register zr = AppropriateZeroRegFor(rd); 1141 csinv(rd, zr, zr, InvertCondition(cond)); 1142 } 1143 1144 1145 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) { 1146 VIXL_ASSERT((cond != al) && (cond != nv)); 1147 csinc(rd, rn, rn, InvertCondition(cond)); 1148 } 1149 1150 1151 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) { 1152 VIXL_ASSERT((cond != al) && (cond != nv)); 1153 csinv(rd, rn, rn, InvertCondition(cond)); 1154 } 1155 1156 1157 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) { 1158 VIXL_ASSERT((cond != al) && (cond != nv)); 1159 csneg(rd, rn, rn, InvertCondition(cond)); 1160 } 1161 1162 1163 void Assembler::ConditionalSelect(const Register& rd, 1164 const Register& rn, 1165 const Register& rm, 1166 Condition cond, 1167 ConditionalSelectOp op) { 1168 VIXL_ASSERT(rd.size() == rn.size()); 1169 VIXL_ASSERT(rd.size() == rm.size()); 1170 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd)); 1171 } 1172 1173 1174 void Assembler::ccmn(const Register& rn, 1175 const Operand& operand, 1176 StatusFlags nzcv, 1177 Condition cond) { 1178 ConditionalCompare(rn, operand, nzcv, cond, CCMN); 1179 } 1180 1181 1182 void Assembler::ccmp(const Register& rn, 1183 const Operand& operand, 1184 StatusFlags nzcv, 1185 Condition cond) { 1186 ConditionalCompare(rn, operand, nzcv, cond, CCMP); 1187 } 1188 1189 1190 void Assembler::DataProcessing3Source(const Register& rd, 1191 const Register& rn, 1192 const Register& rm, 1193 const Register& ra, 1194 DataProcessing3SourceOp op) { 1195 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd)); 1196 } 1197 1198 1199 void Assembler::crc32b(const Register& rd, 1200 const Register& rn, 1201 const Register& rm) { 1202 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits()); 1203 Emit(SF(rm) | Rm(rm) | CRC32B | Rn(rn) | Rd(rd)); 1204 } 1205 1206 1207 void Assembler::crc32h(const Register& rd, 1208 const Register& rn, 1209 const Register& rm) { 1210 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits()); 1211 Emit(SF(rm) | Rm(rm) | CRC32H | Rn(rn) | Rd(rd)); 1212 } 1213 1214 1215 void Assembler::crc32w(const Register& rd, 1216 const Register& rn, 1217 const Register& rm) { 1218 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits()); 1219 Emit(SF(rm) | Rm(rm) | CRC32W | Rn(rn) | Rd(rd)); 1220 } 1221 1222 1223 void Assembler::crc32x(const Register& rd, 1224 const Register& rn, 1225 const Register& rm) { 1226 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits()); 1227 Emit(SF(rm) | Rm(rm) | CRC32X | Rn(rn) | Rd(rd)); 1228 } 1229 1230 1231 void Assembler::crc32cb(const Register& rd, 1232 const Register& rn, 1233 const Register& rm) { 1234 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits()); 1235 Emit(SF(rm) | Rm(rm) | CRC32CB | Rn(rn) | Rd(rd)); 1236 } 1237 1238 1239 void Assembler::crc32ch(const Register& rd, 1240 const Register& rn, 1241 const Register& rm) { 1242 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits()); 1243 Emit(SF(rm) | Rm(rm) | CRC32CH | Rn(rn) | Rd(rd)); 1244 } 1245 1246 1247 void Assembler::crc32cw(const Register& rd, 1248 const Register& rn, 1249 const Register& rm) { 1250 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits()); 1251 Emit(SF(rm) | Rm(rm) | CRC32CW | Rn(rn) | Rd(rd)); 1252 } 1253 1254 1255 void Assembler::crc32cx(const Register& rd, 1256 const Register& rn, 1257 const Register& rm) { 1258 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits()); 1259 Emit(SF(rm) | Rm(rm) | CRC32CX | Rn(rn) | Rd(rd)); 1260 } 1261 1262 1263 void Assembler::mul(const Register& rd, 1264 const Register& rn, 1265 const Register& rm) { 1266 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm)); 1267 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD); 1268 } 1269 1270 1271 void Assembler::madd(const Register& rd, 1272 const Register& rn, 1273 const Register& rm, 1274 const Register& ra) { 1275 DataProcessing3Source(rd, rn, rm, ra, MADD); 1276 } 1277 1278 1279 void Assembler::mneg(const Register& rd, 1280 const Register& rn, 1281 const Register& rm) { 1282 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm)); 1283 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB); 1284 } 1285 1286 1287 void Assembler::msub(const Register& rd, 1288 const Register& rn, 1289 const Register& rm, 1290 const Register& ra) { 1291 DataProcessing3Source(rd, rn, rm, ra, MSUB); 1292 } 1293 1294 1295 void Assembler::umaddl(const Register& rd, 1296 const Register& rn, 1297 const Register& rm, 1298 const Register& ra) { 1299 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits()); 1300 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits()); 1301 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x); 1302 } 1303 1304 1305 void Assembler::smaddl(const Register& rd, 1306 const Register& rn, 1307 const Register& rm, 1308 const Register& ra) { 1309 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits()); 1310 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits()); 1311 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x); 1312 } 1313 1314 1315 void Assembler::umsubl(const Register& rd, 1316 const Register& rn, 1317 const Register& rm, 1318 const Register& ra) { 1319 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits()); 1320 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits()); 1321 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x); 1322 } 1323 1324 1325 void Assembler::smsubl(const Register& rd, 1326 const Register& rn, 1327 const Register& rm, 1328 const Register& ra) { 1329 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits()); 1330 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits()); 1331 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x); 1332 } 1333 1334 1335 void Assembler::smull(const Register& rd, 1336 const Register& rn, 1337 const Register& rm) { 1338 VIXL_ASSERT(rd.Is64Bits()); 1339 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits()); 1340 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x); 1341 } 1342 1343 1344 void Assembler::sdiv(const Register& rd, 1345 const Register& rn, 1346 const Register& rm) { 1347 VIXL_ASSERT(rd.size() == rn.size()); 1348 VIXL_ASSERT(rd.size() == rm.size()); 1349 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1350 } 1351 1352 1353 void Assembler::smulh(const Register& xd, 1354 const Register& xn, 1355 const Register& xm) { 1356 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits()); 1357 DataProcessing3Source(xd, xn, xm, xzr, SMULH_x); 1358 } 1359 1360 1361 void Assembler::umulh(const Register& xd, 1362 const Register& xn, 1363 const Register& xm) { 1364 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits()); 1365 DataProcessing3Source(xd, xn, xm, xzr, UMULH_x); 1366 } 1367 1368 1369 void Assembler::udiv(const Register& rd, 1370 const Register& rn, 1371 const Register& rm) { 1372 VIXL_ASSERT(rd.size() == rn.size()); 1373 VIXL_ASSERT(rd.size() == rm.size()); 1374 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd)); 1375 } 1376 1377 1378 void Assembler::rbit(const Register& rd, 1379 const Register& rn) { 1380 DataProcessing1Source(rd, rn, RBIT); 1381 } 1382 1383 1384 void Assembler::rev16(const Register& rd, 1385 const Register& rn) { 1386 DataProcessing1Source(rd, rn, REV16); 1387 } 1388 1389 1390 void Assembler::rev32(const Register& rd, 1391 const Register& rn) { 1392 VIXL_ASSERT(rd.Is64Bits()); 1393 DataProcessing1Source(rd, rn, REV); 1394 } 1395 1396 1397 void Assembler::rev(const Register& rd, 1398 const Register& rn) { 1399 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w); 1400 } 1401 1402 1403 void Assembler::clz(const Register& rd, 1404 const Register& rn) { 1405 DataProcessing1Source(rd, rn, CLZ); 1406 } 1407 1408 1409 void Assembler::cls(const Register& rd, 1410 const Register& rn) { 1411 DataProcessing1Source(rd, rn, CLS); 1412 } 1413 1414 1415 void Assembler::ldp(const CPURegister& rt, 1416 const CPURegister& rt2, 1417 const MemOperand& src) { 1418 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2)); 1419 } 1420 1421 1422 void Assembler::stp(const CPURegister& rt, 1423 const CPURegister& rt2, 1424 const MemOperand& dst) { 1425 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2)); 1426 } 1427 1428 1429 void Assembler::ldpsw(const Register& rt, 1430 const Register& rt2, 1431 const MemOperand& src) { 1432 VIXL_ASSERT(rt.Is64Bits()); 1433 LoadStorePair(rt, rt2, src, LDPSW_x); 1434 } 1435 1436 1437 void Assembler::LoadStorePair(const CPURegister& rt, 1438 const CPURegister& rt2, 1439 const MemOperand& addr, 1440 LoadStorePairOp op) { 1441 // 'rt' and 'rt2' can only be aliased for stores. 1442 VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2)); 1443 VIXL_ASSERT(AreSameSizeAndType(rt, rt2)); 1444 1445 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | 1446 ImmLSPair(addr.offset(), CalcLSPairDataSize(op)); 1447 1448 Instr addrmodeop; 1449 if (addr.IsImmediateOffset()) { 1450 addrmodeop = LoadStorePairOffsetFixed; 1451 } else { 1452 VIXL_ASSERT(addr.offset() != 0); 1453 if (addr.IsPreIndex()) { 1454 addrmodeop = LoadStorePairPreIndexFixed; 1455 } else { 1456 VIXL_ASSERT(addr.IsPostIndex()); 1457 addrmodeop = LoadStorePairPostIndexFixed; 1458 } 1459 } 1460 Emit(addrmodeop | memop); 1461 } 1462 1463 1464 void Assembler::ldnp(const CPURegister& rt, 1465 const CPURegister& rt2, 1466 const MemOperand& src) { 1467 LoadStorePairNonTemporal(rt, rt2, src, 1468 LoadPairNonTemporalOpFor(rt, rt2)); 1469 } 1470 1471 1472 void Assembler::stnp(const CPURegister& rt, 1473 const CPURegister& rt2, 1474 const MemOperand& dst) { 1475 LoadStorePairNonTemporal(rt, rt2, dst, 1476 StorePairNonTemporalOpFor(rt, rt2)); 1477 } 1478 1479 1480 void Assembler::LoadStorePairNonTemporal(const CPURegister& rt, 1481 const CPURegister& rt2, 1482 const MemOperand& addr, 1483 LoadStorePairNonTemporalOp op) { 1484 VIXL_ASSERT(!rt.Is(rt2)); 1485 VIXL_ASSERT(AreSameSizeAndType(rt, rt2)); 1486 VIXL_ASSERT(addr.IsImmediateOffset()); 1487 1488 unsigned size = CalcLSPairDataSize( 1489 static_cast<LoadStorePairOp>(op & LoadStorePairMask)); 1490 Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | 1491 ImmLSPair(addr.offset(), size)); 1492 } 1493 1494 1495 // Memory instructions. 1496 void Assembler::ldrb(const Register& rt, const MemOperand& src, 1497 LoadStoreScalingOption option) { 1498 VIXL_ASSERT(option != RequireUnscaledOffset); 1499 VIXL_ASSERT(option != PreferUnscaledOffset); 1500 LoadStore(rt, src, LDRB_w, option); 1501 } 1502 1503 1504 void Assembler::strb(const Register& rt, const MemOperand& dst, 1505 LoadStoreScalingOption option) { 1506 VIXL_ASSERT(option != RequireUnscaledOffset); 1507 VIXL_ASSERT(option != PreferUnscaledOffset); 1508 LoadStore(rt, dst, STRB_w, option); 1509 } 1510 1511 1512 void Assembler::ldrsb(const Register& rt, const MemOperand& src, 1513 LoadStoreScalingOption option) { 1514 VIXL_ASSERT(option != RequireUnscaledOffset); 1515 VIXL_ASSERT(option != PreferUnscaledOffset); 1516 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option); 1517 } 1518 1519 1520 void Assembler::ldrh(const Register& rt, const MemOperand& src, 1521 LoadStoreScalingOption option) { 1522 VIXL_ASSERT(option != RequireUnscaledOffset); 1523 VIXL_ASSERT(option != PreferUnscaledOffset); 1524 LoadStore(rt, src, LDRH_w, option); 1525 } 1526 1527 1528 void Assembler::strh(const Register& rt, const MemOperand& dst, 1529 LoadStoreScalingOption option) { 1530 VIXL_ASSERT(option != RequireUnscaledOffset); 1531 VIXL_ASSERT(option != PreferUnscaledOffset); 1532 LoadStore(rt, dst, STRH_w, option); 1533 } 1534 1535 1536 void Assembler::ldrsh(const Register& rt, const MemOperand& src, 1537 LoadStoreScalingOption option) { 1538 VIXL_ASSERT(option != RequireUnscaledOffset); 1539 VIXL_ASSERT(option != PreferUnscaledOffset); 1540 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option); 1541 } 1542 1543 1544 void Assembler::ldr(const CPURegister& rt, const MemOperand& src, 1545 LoadStoreScalingOption option) { 1546 VIXL_ASSERT(option != RequireUnscaledOffset); 1547 VIXL_ASSERT(option != PreferUnscaledOffset); 1548 LoadStore(rt, src, LoadOpFor(rt), option); 1549 } 1550 1551 1552 void Assembler::str(const CPURegister& rt, const MemOperand& dst, 1553 LoadStoreScalingOption option) { 1554 VIXL_ASSERT(option != RequireUnscaledOffset); 1555 VIXL_ASSERT(option != PreferUnscaledOffset); 1556 LoadStore(rt, dst, StoreOpFor(rt), option); 1557 } 1558 1559 1560 void Assembler::ldrsw(const Register& rt, const MemOperand& src, 1561 LoadStoreScalingOption option) { 1562 VIXL_ASSERT(rt.Is64Bits()); 1563 VIXL_ASSERT(option != RequireUnscaledOffset); 1564 VIXL_ASSERT(option != PreferUnscaledOffset); 1565 LoadStore(rt, src, LDRSW_x, option); 1566 } 1567 1568 1569 void Assembler::ldurb(const Register& rt, const MemOperand& src, 1570 LoadStoreScalingOption option) { 1571 VIXL_ASSERT(option != RequireScaledOffset); 1572 VIXL_ASSERT(option != PreferScaledOffset); 1573 LoadStore(rt, src, LDRB_w, option); 1574 } 1575 1576 1577 void Assembler::sturb(const Register& rt, const MemOperand& dst, 1578 LoadStoreScalingOption option) { 1579 VIXL_ASSERT(option != RequireScaledOffset); 1580 VIXL_ASSERT(option != PreferScaledOffset); 1581 LoadStore(rt, dst, STRB_w, option); 1582 } 1583 1584 1585 void Assembler::ldursb(const Register& rt, const MemOperand& src, 1586 LoadStoreScalingOption option) { 1587 VIXL_ASSERT(option != RequireScaledOffset); 1588 VIXL_ASSERT(option != PreferScaledOffset); 1589 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option); 1590 } 1591 1592 1593 void Assembler::ldurh(const Register& rt, const MemOperand& src, 1594 LoadStoreScalingOption option) { 1595 VIXL_ASSERT(option != RequireScaledOffset); 1596 VIXL_ASSERT(option != PreferScaledOffset); 1597 LoadStore(rt, src, LDRH_w, option); 1598 } 1599 1600 1601 void Assembler::sturh(const Register& rt, const MemOperand& dst, 1602 LoadStoreScalingOption option) { 1603 VIXL_ASSERT(option != RequireScaledOffset); 1604 VIXL_ASSERT(option != PreferScaledOffset); 1605 LoadStore(rt, dst, STRH_w, option); 1606 } 1607 1608 1609 void Assembler::ldursh(const Register& rt, const MemOperand& src, 1610 LoadStoreScalingOption option) { 1611 VIXL_ASSERT(option != RequireScaledOffset); 1612 VIXL_ASSERT(option != PreferScaledOffset); 1613 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option); 1614 } 1615 1616 1617 void Assembler::ldur(const CPURegister& rt, const MemOperand& src, 1618 LoadStoreScalingOption option) { 1619 VIXL_ASSERT(option != RequireScaledOffset); 1620 VIXL_ASSERT(option != PreferScaledOffset); 1621 LoadStore(rt, src, LoadOpFor(rt), option); 1622 } 1623 1624 1625 void Assembler::stur(const CPURegister& rt, const MemOperand& dst, 1626 LoadStoreScalingOption option) { 1627 VIXL_ASSERT(option != RequireScaledOffset); 1628 VIXL_ASSERT(option != PreferScaledOffset); 1629 LoadStore(rt, dst, StoreOpFor(rt), option); 1630 } 1631 1632 1633 void Assembler::ldursw(const Register& rt, const MemOperand& src, 1634 LoadStoreScalingOption option) { 1635 VIXL_ASSERT(rt.Is64Bits()); 1636 VIXL_ASSERT(option != RequireScaledOffset); 1637 VIXL_ASSERT(option != PreferScaledOffset); 1638 LoadStore(rt, src, LDRSW_x, option); 1639 } 1640 1641 1642 void Assembler::ldrsw(const Register& rt, RawLiteral* literal) { 1643 VIXL_ASSERT(rt.Is64Bits()); 1644 VIXL_ASSERT(literal->size() == kWRegSizeInBytes); 1645 ldrsw(rt, LinkAndGetWordOffsetTo(literal)); 1646 } 1647 1648 1649 void Assembler::ldr(const CPURegister& rt, RawLiteral* literal) { 1650 VIXL_ASSERT(literal->size() == static_cast<size_t>(rt.SizeInBytes())); 1651 ldr(rt, LinkAndGetWordOffsetTo(literal)); 1652 } 1653 1654 1655 void Assembler::ldrsw(const Register& rt, int imm19) { 1656 Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt)); 1657 } 1658 1659 1660 void Assembler::ldr(const CPURegister& rt, int imm19) { 1661 LoadLiteralOp op = LoadLiteralOpFor(rt); 1662 Emit(op | ImmLLiteral(imm19) | Rt(rt)); 1663 } 1664 1665 1666 void Assembler::prfm(PrefetchOperation op, int imm19) { 1667 Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19)); 1668 } 1669 1670 1671 // Exclusive-access instructions. 1672 void Assembler::stxrb(const Register& rs, 1673 const Register& rt, 1674 const MemOperand& dst) { 1675 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1676 Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1677 } 1678 1679 1680 void Assembler::stxrh(const Register& rs, 1681 const Register& rt, 1682 const MemOperand& dst) { 1683 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1684 Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1685 } 1686 1687 1688 void Assembler::stxr(const Register& rs, 1689 const Register& rt, 1690 const MemOperand& dst) { 1691 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1692 LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w; 1693 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1694 } 1695 1696 1697 void Assembler::ldxrb(const Register& rt, 1698 const MemOperand& src) { 1699 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1700 Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1701 } 1702 1703 1704 void Assembler::ldxrh(const Register& rt, 1705 const MemOperand& src) { 1706 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1707 Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1708 } 1709 1710 1711 void Assembler::ldxr(const Register& rt, 1712 const MemOperand& src) { 1713 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1714 LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w; 1715 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1716 } 1717 1718 1719 void Assembler::stxp(const Register& rs, 1720 const Register& rt, 1721 const Register& rt2, 1722 const MemOperand& dst) { 1723 VIXL_ASSERT(rt.size() == rt2.size()); 1724 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1725 LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w; 1726 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base())); 1727 } 1728 1729 1730 void Assembler::ldxp(const Register& rt, 1731 const Register& rt2, 1732 const MemOperand& src) { 1733 VIXL_ASSERT(rt.size() == rt2.size()); 1734 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1735 LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w; 1736 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base())); 1737 } 1738 1739 1740 void Assembler::stlxrb(const Register& rs, 1741 const Register& rt, 1742 const MemOperand& dst) { 1743 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1744 Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1745 } 1746 1747 1748 void Assembler::stlxrh(const Register& rs, 1749 const Register& rt, 1750 const MemOperand& dst) { 1751 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1752 Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1753 } 1754 1755 1756 void Assembler::stlxr(const Register& rs, 1757 const Register& rt, 1758 const MemOperand& dst) { 1759 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1760 LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w; 1761 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1762 } 1763 1764 1765 void Assembler::ldaxrb(const Register& rt, 1766 const MemOperand& src) { 1767 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1768 Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1769 } 1770 1771 1772 void Assembler::ldaxrh(const Register& rt, 1773 const MemOperand& src) { 1774 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1775 Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1776 } 1777 1778 1779 void Assembler::ldaxr(const Register& rt, 1780 const MemOperand& src) { 1781 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1782 LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w; 1783 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1784 } 1785 1786 1787 void Assembler::stlxp(const Register& rs, 1788 const Register& rt, 1789 const Register& rt2, 1790 const MemOperand& dst) { 1791 VIXL_ASSERT(rt.size() == rt2.size()); 1792 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1793 LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w; 1794 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base())); 1795 } 1796 1797 1798 void Assembler::ldaxp(const Register& rt, 1799 const Register& rt2, 1800 const MemOperand& src) { 1801 VIXL_ASSERT(rt.size() == rt2.size()); 1802 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1803 LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w; 1804 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base())); 1805 } 1806 1807 1808 void Assembler::stlrb(const Register& rt, 1809 const MemOperand& dst) { 1810 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1811 Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1812 } 1813 1814 1815 void Assembler::stlrh(const Register& rt, 1816 const MemOperand& dst) { 1817 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1818 Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1819 } 1820 1821 1822 void Assembler::stlr(const Register& rt, 1823 const MemOperand& dst) { 1824 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0)); 1825 LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w; 1826 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base())); 1827 } 1828 1829 1830 void Assembler::ldarb(const Register& rt, 1831 const MemOperand& src) { 1832 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1833 Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1834 } 1835 1836 1837 void Assembler::ldarh(const Register& rt, 1838 const MemOperand& src) { 1839 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1840 Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1841 } 1842 1843 1844 void Assembler::ldar(const Register& rt, 1845 const MemOperand& src) { 1846 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0)); 1847 LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w; 1848 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base())); 1849 } 1850 1851 1852 void Assembler::prfm(PrefetchOperation op, const MemOperand& address, 1853 LoadStoreScalingOption option) { 1854 VIXL_ASSERT(option != RequireUnscaledOffset); 1855 VIXL_ASSERT(option != PreferUnscaledOffset); 1856 Prefetch(op, address, option); 1857 } 1858 1859 1860 void Assembler::prfum(PrefetchOperation op, const MemOperand& address, 1861 LoadStoreScalingOption option) { 1862 VIXL_ASSERT(option != RequireScaledOffset); 1863 VIXL_ASSERT(option != PreferScaledOffset); 1864 Prefetch(op, address, option); 1865 } 1866 1867 1868 void Assembler::prfm(PrefetchOperation op, RawLiteral* literal) { 1869 prfm(op, LinkAndGetWordOffsetTo(literal)); 1870 } 1871 1872 1873 void Assembler::sys(int op1, int crn, int crm, int op2, const Register& rt) { 1874 Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(rt)); 1875 } 1876 1877 1878 void Assembler::sys(int op, const Register& rt) { 1879 Emit(SYS | SysOp(op) | Rt(rt)); 1880 } 1881 1882 1883 void Assembler::dc(DataCacheOp op, const Register& rt) { 1884 VIXL_ASSERT((op == CVAC) || (op == CVAU) || (op == CIVAC) || (op == ZVA)); 1885 sys(op, rt); 1886 } 1887 1888 1889 void Assembler::ic(InstructionCacheOp op, const Register& rt) { 1890 VIXL_ASSERT(op == IVAU); 1891 sys(op, rt); 1892 } 1893 1894 1895 void Assembler::hint(SystemHint code) { 1896 Emit(HINT | ImmHint(code) | Rt(xzr)); 1897 } 1898 1899 1900 // NEON structure loads and stores. 1901 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) { 1902 Instr addr_field = RnSP(addr.base()); 1903 1904 if (addr.IsPostIndex()) { 1905 VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex == 1906 static_cast<NEONLoadStoreMultiStructPostIndexOp>( 1907 NEONLoadStoreSingleStructPostIndex)); 1908 1909 addr_field |= NEONLoadStoreMultiStructPostIndex; 1910 if (addr.offset() == 0) { 1911 addr_field |= RmNot31(addr.regoffset()); 1912 } else { 1913 // The immediate post index addressing mode is indicated by rm = 31. 1914 // The immediate is implied by the number of vector registers used. 1915 addr_field |= (0x1f << Rm_offset); 1916 } 1917 } else { 1918 VIXL_ASSERT(addr.IsImmediateOffset() && (addr.offset() == 0)); 1919 } 1920 return addr_field; 1921 } 1922 1923 void Assembler::LoadStoreStructVerify(const VRegister& vt, 1924 const MemOperand& addr, 1925 Instr op) { 1926 #ifdef VIXL_DEBUG 1927 // Assert that addressing mode is either offset (with immediate 0), post 1928 // index by immediate of the size of the register list, or post index by a 1929 // value in a core register. 1930 if (addr.IsImmediateOffset()) { 1931 VIXL_ASSERT(addr.offset() == 0); 1932 } else { 1933 int offset = vt.SizeInBytes(); 1934 switch (op) { 1935 case NEON_LD1_1v: 1936 case NEON_ST1_1v: 1937 offset *= 1; break; 1938 case NEONLoadStoreSingleStructLoad1: 1939 case NEONLoadStoreSingleStructStore1: 1940 case NEON_LD1R: 1941 offset = (offset / vt.lanes()) * 1; break; 1942 1943 case NEON_LD1_2v: 1944 case NEON_ST1_2v: 1945 case NEON_LD2: 1946 case NEON_ST2: 1947 offset *= 2; 1948 break; 1949 case NEONLoadStoreSingleStructLoad2: 1950 case NEONLoadStoreSingleStructStore2: 1951 case NEON_LD2R: 1952 offset = (offset / vt.lanes()) * 2; break; 1953 1954 case NEON_LD1_3v: 1955 case NEON_ST1_3v: 1956 case NEON_LD3: 1957 case NEON_ST3: 1958 offset *= 3; break; 1959 case NEONLoadStoreSingleStructLoad3: 1960 case NEONLoadStoreSingleStructStore3: 1961 case NEON_LD3R: 1962 offset = (offset / vt.lanes()) * 3; break; 1963 1964 case NEON_LD1_4v: 1965 case NEON_ST1_4v: 1966 case NEON_LD4: 1967 case NEON_ST4: 1968 offset *= 4; break; 1969 case NEONLoadStoreSingleStructLoad4: 1970 case NEONLoadStoreSingleStructStore4: 1971 case NEON_LD4R: 1972 offset = (offset / vt.lanes()) * 4; break; 1973 default: 1974 VIXL_UNREACHABLE(); 1975 } 1976 VIXL_ASSERT(!addr.regoffset().Is(NoReg) || 1977 addr.offset() == offset); 1978 } 1979 #else 1980 USE(vt); 1981 USE(addr); 1982 USE(op); 1983 #endif 1984 } 1985 1986 void Assembler::LoadStoreStruct(const VRegister& vt, 1987 const MemOperand& addr, 1988 NEONLoadStoreMultiStructOp op) { 1989 LoadStoreStructVerify(vt, addr, op); 1990 VIXL_ASSERT(vt.IsVector() || vt.Is1D()); 1991 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt)); 1992 } 1993 1994 1995 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt, 1996 const MemOperand& addr, 1997 NEONLoadStoreSingleStructOp op) { 1998 LoadStoreStructVerify(vt, addr, op); 1999 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt)); 2000 } 2001 2002 2003 void Assembler::ld1(const VRegister& vt, 2004 const MemOperand& src) { 2005 LoadStoreStruct(vt, src, NEON_LD1_1v); 2006 } 2007 2008 2009 void Assembler::ld1(const VRegister& vt, 2010 const VRegister& vt2, 2011 const MemOperand& src) { 2012 USE(vt2); 2013 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2014 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2015 LoadStoreStruct(vt, src, NEON_LD1_2v); 2016 } 2017 2018 2019 void Assembler::ld1(const VRegister& vt, 2020 const VRegister& vt2, 2021 const VRegister& vt3, 2022 const MemOperand& src) { 2023 USE(vt2, vt3); 2024 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2025 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2026 LoadStoreStruct(vt, src, NEON_LD1_3v); 2027 } 2028 2029 2030 void Assembler::ld1(const VRegister& vt, 2031 const VRegister& vt2, 2032 const VRegister& vt3, 2033 const VRegister& vt4, 2034 const MemOperand& src) { 2035 USE(vt2, vt3, vt4); 2036 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2037 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2038 LoadStoreStruct(vt, src, NEON_LD1_4v); 2039 } 2040 2041 2042 void Assembler::ld2(const VRegister& vt, 2043 const VRegister& vt2, 2044 const MemOperand& src) { 2045 USE(vt2); 2046 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2047 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2048 LoadStoreStruct(vt, src, NEON_LD2); 2049 } 2050 2051 2052 void Assembler::ld2(const VRegister& vt, 2053 const VRegister& vt2, 2054 int lane, 2055 const MemOperand& src) { 2056 USE(vt2); 2057 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2058 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2059 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2); 2060 } 2061 2062 2063 void Assembler::ld2r(const VRegister& vt, 2064 const VRegister& vt2, 2065 const MemOperand& src) { 2066 USE(vt2); 2067 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2068 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2069 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R); 2070 } 2071 2072 2073 void Assembler::ld3(const VRegister& vt, 2074 const VRegister& vt2, 2075 const VRegister& vt3, 2076 const MemOperand& src) { 2077 USE(vt2, vt3); 2078 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2079 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2080 LoadStoreStruct(vt, src, NEON_LD3); 2081 } 2082 2083 2084 void Assembler::ld3(const VRegister& vt, 2085 const VRegister& vt2, 2086 const VRegister& vt3, 2087 int lane, 2088 const MemOperand& src) { 2089 USE(vt2, vt3); 2090 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2091 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2092 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3); 2093 } 2094 2095 2096 void Assembler::ld3r(const VRegister& vt, 2097 const VRegister& vt2, 2098 const VRegister& vt3, 2099 const MemOperand& src) { 2100 USE(vt2, vt3); 2101 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2102 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2103 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R); 2104 } 2105 2106 2107 void Assembler::ld4(const VRegister& vt, 2108 const VRegister& vt2, 2109 const VRegister& vt3, 2110 const VRegister& vt4, 2111 const MemOperand& src) { 2112 USE(vt2, vt3, vt4); 2113 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2114 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2115 LoadStoreStruct(vt, src, NEON_LD4); 2116 } 2117 2118 2119 void Assembler::ld4(const VRegister& vt, 2120 const VRegister& vt2, 2121 const VRegister& vt3, 2122 const VRegister& vt4, 2123 int lane, 2124 const MemOperand& src) { 2125 USE(vt2, vt3, vt4); 2126 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2127 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2128 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4); 2129 } 2130 2131 2132 void Assembler::ld4r(const VRegister& vt, 2133 const VRegister& vt2, 2134 const VRegister& vt3, 2135 const VRegister& vt4, 2136 const MemOperand& src) { 2137 USE(vt2, vt3, vt4); 2138 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2139 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2140 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R); 2141 } 2142 2143 2144 void Assembler::st1(const VRegister& vt, 2145 const MemOperand& src) { 2146 LoadStoreStruct(vt, src, NEON_ST1_1v); 2147 } 2148 2149 2150 void Assembler::st1(const VRegister& vt, 2151 const VRegister& vt2, 2152 const MemOperand& src) { 2153 USE(vt2); 2154 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2155 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2156 LoadStoreStruct(vt, src, NEON_ST1_2v); 2157 } 2158 2159 2160 void Assembler::st1(const VRegister& vt, 2161 const VRegister& vt2, 2162 const VRegister& vt3, 2163 const MemOperand& src) { 2164 USE(vt2, vt3); 2165 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2166 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2167 LoadStoreStruct(vt, src, NEON_ST1_3v); 2168 } 2169 2170 2171 void Assembler::st1(const VRegister& vt, 2172 const VRegister& vt2, 2173 const VRegister& vt3, 2174 const VRegister& vt4, 2175 const MemOperand& src) { 2176 USE(vt2, vt3, vt4); 2177 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2178 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2179 LoadStoreStruct(vt, src, NEON_ST1_4v); 2180 } 2181 2182 2183 void Assembler::st2(const VRegister& vt, 2184 const VRegister& vt2, 2185 const MemOperand& dst) { 2186 USE(vt2); 2187 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2188 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2189 LoadStoreStruct(vt, dst, NEON_ST2); 2190 } 2191 2192 2193 void Assembler::st2(const VRegister& vt, 2194 const VRegister& vt2, 2195 int lane, 2196 const MemOperand& dst) { 2197 USE(vt2); 2198 VIXL_ASSERT(AreSameFormat(vt, vt2)); 2199 VIXL_ASSERT(AreConsecutive(vt, vt2)); 2200 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2); 2201 } 2202 2203 2204 void Assembler::st3(const VRegister& vt, 2205 const VRegister& vt2, 2206 const VRegister& vt3, 2207 const MemOperand& dst) { 2208 USE(vt2, vt3); 2209 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2210 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2211 LoadStoreStruct(vt, dst, NEON_ST3); 2212 } 2213 2214 2215 void Assembler::st3(const VRegister& vt, 2216 const VRegister& vt2, 2217 const VRegister& vt3, 2218 int lane, 2219 const MemOperand& dst) { 2220 USE(vt2, vt3); 2221 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3)); 2222 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3)); 2223 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3); 2224 } 2225 2226 2227 void Assembler::st4(const VRegister& vt, 2228 const VRegister& vt2, 2229 const VRegister& vt3, 2230 const VRegister& vt4, 2231 const MemOperand& dst) { 2232 USE(vt2, vt3, vt4); 2233 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2234 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2235 LoadStoreStruct(vt, dst, NEON_ST4); 2236 } 2237 2238 2239 void Assembler::st4(const VRegister& vt, 2240 const VRegister& vt2, 2241 const VRegister& vt3, 2242 const VRegister& vt4, 2243 int lane, 2244 const MemOperand& dst) { 2245 USE(vt2, vt3, vt4); 2246 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4)); 2247 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4)); 2248 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4); 2249 } 2250 2251 2252 void Assembler::LoadStoreStructSingle(const VRegister& vt, 2253 uint32_t lane, 2254 const MemOperand& addr, 2255 NEONLoadStoreSingleStructOp op) { 2256 LoadStoreStructVerify(vt, addr, op); 2257 2258 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the 2259 // number of lanes, and T is b, h, s or d. 2260 unsigned lane_size = vt.LaneSizeInBytes(); 2261 VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size)); 2262 2263 // Lane size is encoded in the opcode field. Lane index is encoded in the Q, 2264 // S and size fields. 2265 lane *= lane_size; 2266 if (lane_size == 8) lane++; 2267 2268 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask; 2269 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask; 2270 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask; 2271 2272 Instr instr = op; 2273 switch (lane_size) { 2274 case 1: instr |= NEONLoadStoreSingle_b; break; 2275 case 2: instr |= NEONLoadStoreSingle_h; break; 2276 case 4: instr |= NEONLoadStoreSingle_s; break; 2277 default: 2278 VIXL_ASSERT(lane_size == 8); 2279 instr |= NEONLoadStoreSingle_d; 2280 } 2281 2282 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt)); 2283 } 2284 2285 2286 void Assembler::ld1(const VRegister& vt, 2287 int lane, 2288 const MemOperand& src) { 2289 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1); 2290 } 2291 2292 2293 void Assembler::ld1r(const VRegister& vt, 2294 const MemOperand& src) { 2295 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R); 2296 } 2297 2298 2299 void Assembler::st1(const VRegister& vt, 2300 int lane, 2301 const MemOperand& dst) { 2302 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1); 2303 } 2304 2305 2306 void Assembler::NEON3DifferentL(const VRegister& vd, 2307 const VRegister& vn, 2308 const VRegister& vm, 2309 NEON3DifferentOp vop) { 2310 VIXL_ASSERT(AreSameFormat(vn, vm)); 2311 VIXL_ASSERT((vn.Is1H() && vd.Is1S()) || 2312 (vn.Is1S() && vd.Is1D()) || 2313 (vn.Is8B() && vd.Is8H()) || 2314 (vn.Is4H() && vd.Is4S()) || 2315 (vn.Is2S() && vd.Is2D()) || 2316 (vn.Is16B() && vd.Is8H())|| 2317 (vn.Is8H() && vd.Is4S()) || 2318 (vn.Is4S() && vd.Is2D())); 2319 Instr format, op = vop; 2320 if (vd.IsScalar()) { 2321 op |= NEON_Q | NEONScalar; 2322 format = SFormat(vn); 2323 } else { 2324 format = VFormat(vn); 2325 } 2326 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd)); 2327 } 2328 2329 2330 void Assembler::NEON3DifferentW(const VRegister& vd, 2331 const VRegister& vn, 2332 const VRegister& vm, 2333 NEON3DifferentOp vop) { 2334 VIXL_ASSERT(AreSameFormat(vd, vn)); 2335 VIXL_ASSERT((vm.Is8B() && vd.Is8H()) || 2336 (vm.Is4H() && vd.Is4S()) || 2337 (vm.Is2S() && vd.Is2D()) || 2338 (vm.Is16B() && vd.Is8H())|| 2339 (vm.Is8H() && vd.Is4S()) || 2340 (vm.Is4S() && vd.Is2D())); 2341 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd)); 2342 } 2343 2344 2345 void Assembler::NEON3DifferentHN(const VRegister& vd, 2346 const VRegister& vn, 2347 const VRegister& vm, 2348 NEON3DifferentOp vop) { 2349 VIXL_ASSERT(AreSameFormat(vm, vn)); 2350 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || 2351 (vd.Is4H() && vn.Is4S()) || 2352 (vd.Is2S() && vn.Is2D()) || 2353 (vd.Is16B() && vn.Is8H())|| 2354 (vd.Is8H() && vn.Is4S()) || 2355 (vd.Is4S() && vn.Is2D())); 2356 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd)); 2357 } 2358 2359 2360 #define NEON_3DIFF_LONG_LIST(V) \ 2361 V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \ 2362 V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \ 2363 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \ 2364 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \ 2365 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \ 2366 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \ 2367 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \ 2368 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \ 2369 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \ 2370 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \ 2371 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \ 2372 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \ 2373 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \ 2374 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \ 2375 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \ 2376 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \ 2377 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \ 2378 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \ 2379 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \ 2380 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \ 2381 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \ 2382 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \ 2383 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \ 2384 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \ 2385 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \ 2386 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \ 2387 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \ 2388 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \ 2389 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \ 2390 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \ 2391 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \ 2392 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \ 2393 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \ 2394 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \ 2395 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \ 2396 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \ 2397 2398 2399 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 2400 void Assembler::FN(const VRegister& vd, \ 2401 const VRegister& vn, \ 2402 const VRegister& vm) { \ 2403 VIXL_ASSERT(AS); \ 2404 NEON3DifferentL(vd, vn, vm, OP); \ 2405 } 2406 NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC) 2407 #undef DEFINE_ASM_FUNC 2408 2409 #define NEON_3DIFF_HN_LIST(V) \ 2410 V(addhn, NEON_ADDHN, vd.IsD()) \ 2411 V(addhn2, NEON_ADDHN2, vd.IsQ()) \ 2412 V(raddhn, NEON_RADDHN, vd.IsD()) \ 2413 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \ 2414 V(subhn, NEON_SUBHN, vd.IsD()) \ 2415 V(subhn2, NEON_SUBHN2, vd.IsQ()) \ 2416 V(rsubhn, NEON_RSUBHN, vd.IsD()) \ 2417 V(rsubhn2, NEON_RSUBHN2, vd.IsQ()) 2418 2419 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 2420 void Assembler::FN(const VRegister& vd, \ 2421 const VRegister& vn, \ 2422 const VRegister& vm) { \ 2423 VIXL_ASSERT(AS); \ 2424 NEON3DifferentHN(vd, vn, vm, OP); \ 2425 } 2426 NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC) 2427 #undef DEFINE_ASM_FUNC 2428 2429 void Assembler::uaddw(const VRegister& vd, 2430 const VRegister& vn, 2431 const VRegister& vm) { 2432 VIXL_ASSERT(vm.IsD()); 2433 NEON3DifferentW(vd, vn, vm, NEON_UADDW); 2434 } 2435 2436 2437 void Assembler::uaddw2(const VRegister& vd, 2438 const VRegister& vn, 2439 const VRegister& vm) { 2440 VIXL_ASSERT(vm.IsQ()); 2441 NEON3DifferentW(vd, vn, vm, NEON_UADDW2); 2442 } 2443 2444 2445 void Assembler::saddw(const VRegister& vd, 2446 const VRegister& vn, 2447 const VRegister& vm) { 2448 VIXL_ASSERT(vm.IsD()); 2449 NEON3DifferentW(vd, vn, vm, NEON_SADDW); 2450 } 2451 2452 2453 void Assembler::saddw2(const VRegister& vd, 2454 const VRegister& vn, 2455 const VRegister& vm) { 2456 VIXL_ASSERT(vm.IsQ()); 2457 NEON3DifferentW(vd, vn, vm, NEON_SADDW2); 2458 } 2459 2460 2461 void Assembler::usubw(const VRegister& vd, 2462 const VRegister& vn, 2463 const VRegister& vm) { 2464 VIXL_ASSERT(vm.IsD()); 2465 NEON3DifferentW(vd, vn, vm, NEON_USUBW); 2466 } 2467 2468 2469 void Assembler::usubw2(const VRegister& vd, 2470 const VRegister& vn, 2471 const VRegister& vm) { 2472 VIXL_ASSERT(vm.IsQ()); 2473 NEON3DifferentW(vd, vn, vm, NEON_USUBW2); 2474 } 2475 2476 2477 void Assembler::ssubw(const VRegister& vd, 2478 const VRegister& vn, 2479 const VRegister& vm) { 2480 VIXL_ASSERT(vm.IsD()); 2481 NEON3DifferentW(vd, vn, vm, NEON_SSUBW); 2482 } 2483 2484 2485 void Assembler::ssubw2(const VRegister& vd, 2486 const VRegister& vn, 2487 const VRegister& vm) { 2488 VIXL_ASSERT(vm.IsQ()); 2489 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2); 2490 } 2491 2492 2493 void Assembler::mov(const Register& rd, const Register& rm) { 2494 // Moves involving the stack pointer are encoded as add immediate with 2495 // second operand of zero. Otherwise, orr with first operand zr is 2496 // used. 2497 if (rd.IsSP() || rm.IsSP()) { 2498 add(rd, rm, 0); 2499 } else { 2500 orr(rd, AppropriateZeroRegFor(rd), rm); 2501 } 2502 } 2503 2504 2505 void Assembler::mvn(const Register& rd, const Operand& operand) { 2506 orn(rd, AppropriateZeroRegFor(rd), operand); 2507 } 2508 2509 2510 void Assembler::mrs(const Register& rt, SystemRegister sysreg) { 2511 VIXL_ASSERT(rt.Is64Bits()); 2512 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt)); 2513 } 2514 2515 2516 void Assembler::msr(SystemRegister sysreg, const Register& rt) { 2517 VIXL_ASSERT(rt.Is64Bits()); 2518 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg)); 2519 } 2520 2521 2522 void Assembler::clrex(int imm4) { 2523 Emit(CLREX | CRm(imm4)); 2524 } 2525 2526 2527 void Assembler::dmb(BarrierDomain domain, BarrierType type) { 2528 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 2529 } 2530 2531 2532 void Assembler::dsb(BarrierDomain domain, BarrierType type) { 2533 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type)); 2534 } 2535 2536 2537 void Assembler::isb() { 2538 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll)); 2539 } 2540 2541 2542 void Assembler::fmov(const VRegister& vd, double imm) { 2543 if (vd.IsScalar()) { 2544 VIXL_ASSERT(vd.Is1D()); 2545 Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm)); 2546 } else { 2547 VIXL_ASSERT(vd.Is2D()); 2548 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit; 2549 Instr q = NEON_Q; 2550 uint32_t encoded_imm = FP64ToImm8(imm); 2551 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd)); 2552 } 2553 } 2554 2555 2556 void Assembler::fmov(const VRegister& vd, float imm) { 2557 if (vd.IsScalar()) { 2558 VIXL_ASSERT(vd.Is1S()); 2559 Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm)); 2560 } else { 2561 VIXL_ASSERT(vd.Is2S() | vd.Is4S()); 2562 Instr op = NEONModifiedImmediate_MOVI; 2563 Instr q = vd.Is4S() ? NEON_Q : 0; 2564 uint32_t encoded_imm = FP32ToImm8(imm); 2565 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd)); 2566 } 2567 } 2568 2569 2570 void Assembler::fmov(const Register& rd, const VRegister& vn) { 2571 VIXL_ASSERT(vn.Is1S() || vn.Is1D()); 2572 VIXL_ASSERT(rd.size() == vn.size()); 2573 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd; 2574 Emit(op | Rd(rd) | Rn(vn)); 2575 } 2576 2577 2578 void Assembler::fmov(const VRegister& vd, const Register& rn) { 2579 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 2580 VIXL_ASSERT(vd.size() == rn.size()); 2581 FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx; 2582 Emit(op | Rd(vd) | Rn(rn)); 2583 } 2584 2585 2586 void Assembler::fmov(const VRegister& vd, const VRegister& vn) { 2587 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 2588 VIXL_ASSERT(vd.IsSameFormat(vn)); 2589 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn)); 2590 } 2591 2592 2593 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) { 2594 VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX()); 2595 USE(index); 2596 Emit(FMOV_d1_x | Rd(vd) | Rn(rn)); 2597 } 2598 2599 2600 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) { 2601 VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX()); 2602 USE(index); 2603 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn)); 2604 } 2605 2606 2607 void Assembler::fmadd(const VRegister& vd, 2608 const VRegister& vn, 2609 const VRegister& vm, 2610 const VRegister& va) { 2611 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMADD_s : FMADD_d); 2612 } 2613 2614 2615 void Assembler::fmsub(const VRegister& vd, 2616 const VRegister& vn, 2617 const VRegister& vm, 2618 const VRegister& va) { 2619 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMSUB_s : FMSUB_d); 2620 } 2621 2622 2623 void Assembler::fnmadd(const VRegister& vd, 2624 const VRegister& vn, 2625 const VRegister& vm, 2626 const VRegister& va) { 2627 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMADD_s : FNMADD_d); 2628 } 2629 2630 2631 void Assembler::fnmsub(const VRegister& vd, 2632 const VRegister& vn, 2633 const VRegister& vm, 2634 const VRegister& va) { 2635 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMSUB_s : FNMSUB_d); 2636 } 2637 2638 2639 void Assembler::fnmul(const VRegister& vd, 2640 const VRegister& vn, 2641 const VRegister& vm) { 2642 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm)); 2643 Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d; 2644 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd)); 2645 } 2646 2647 2648 void Assembler::FPCompareMacro(const VRegister& vn, 2649 double value, 2650 FPTrapFlags trap) { 2651 USE(value); 2652 // Although the fcmp{e} instructions can strictly only take an immediate 2653 // value of +0.0, we don't need to check for -0.0 because the sign of 0.0 2654 // doesn't affect the result of the comparison. 2655 VIXL_ASSERT(value == 0.0); 2656 VIXL_ASSERT(vn.Is1S() || vn.Is1D()); 2657 Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero; 2658 Emit(FPType(vn) | op | Rn(vn)); 2659 } 2660 2661 2662 void Assembler::FPCompareMacro(const VRegister& vn, 2663 const VRegister& vm, 2664 FPTrapFlags trap) { 2665 VIXL_ASSERT(vn.Is1S() || vn.Is1D()); 2666 VIXL_ASSERT(vn.IsSameSizeAndType(vm)); 2667 Instr op = (trap == EnableTrap) ? FCMPE : FCMP; 2668 Emit(FPType(vn) | op | Rm(vm) | Rn(vn)); 2669 } 2670 2671 2672 void Assembler::fcmp(const VRegister& vn, 2673 const VRegister& vm) { 2674 FPCompareMacro(vn, vm, DisableTrap); 2675 } 2676 2677 2678 void Assembler::fcmpe(const VRegister& vn, 2679 const VRegister& vm) { 2680 FPCompareMacro(vn, vm, EnableTrap); 2681 } 2682 2683 2684 void Assembler::fcmp(const VRegister& vn, 2685 double value) { 2686 FPCompareMacro(vn, value, DisableTrap); 2687 } 2688 2689 2690 void Assembler::fcmpe(const VRegister& vn, 2691 double value) { 2692 FPCompareMacro(vn, value, EnableTrap); 2693 } 2694 2695 2696 void Assembler::FPCCompareMacro(const VRegister& vn, 2697 const VRegister& vm, 2698 StatusFlags nzcv, 2699 Condition cond, 2700 FPTrapFlags trap) { 2701 VIXL_ASSERT(vn.Is1S() || vn.Is1D()); 2702 VIXL_ASSERT(vn.IsSameSizeAndType(vm)); 2703 Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP; 2704 Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv)); 2705 } 2706 2707 void Assembler::fccmp(const VRegister& vn, 2708 const VRegister& vm, 2709 StatusFlags nzcv, 2710 Condition cond) { 2711 FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap); 2712 } 2713 2714 2715 void Assembler::fccmpe(const VRegister& vn, 2716 const VRegister& vm, 2717 StatusFlags nzcv, 2718 Condition cond) { 2719 FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap); 2720 } 2721 2722 2723 void Assembler::fcsel(const VRegister& vd, 2724 const VRegister& vn, 2725 const VRegister& vm, 2726 Condition cond) { 2727 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 2728 VIXL_ASSERT(AreSameFormat(vd, vn, vm)); 2729 Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd)); 2730 } 2731 2732 2733 void Assembler::NEONFPConvertToInt(const Register& rd, 2734 const VRegister& vn, 2735 Instr op) { 2736 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd)); 2737 } 2738 2739 2740 void Assembler::NEONFPConvertToInt(const VRegister& vd, 2741 const VRegister& vn, 2742 Instr op) { 2743 if (vn.IsScalar()) { 2744 VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D())); 2745 op |= NEON_Q | NEONScalar; 2746 } 2747 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd)); 2748 } 2749 2750 2751 void Assembler::fcvt(const VRegister& vd, 2752 const VRegister& vn) { 2753 FPDataProcessing1SourceOp op; 2754 if (vd.Is1D()) { 2755 VIXL_ASSERT(vn.Is1S() || vn.Is1H()); 2756 op = vn.Is1S() ? FCVT_ds : FCVT_dh; 2757 } else if (vd.Is1S()) { 2758 VIXL_ASSERT(vn.Is1D() || vn.Is1H()); 2759 op = vn.Is1D() ? FCVT_sd : FCVT_sh; 2760 } else { 2761 VIXL_ASSERT(vd.Is1H()); 2762 VIXL_ASSERT(vn.Is1D() || vn.Is1S()); 2763 op = vn.Is1D() ? FCVT_hd : FCVT_hs; 2764 } 2765 FPDataProcessing1Source(vd, vn, op); 2766 } 2767 2768 2769 void Assembler::fcvtl(const VRegister& vd, 2770 const VRegister& vn) { 2771 VIXL_ASSERT((vd.Is4S() && vn.Is4H()) || 2772 (vd.Is2D() && vn.Is2S())); 2773 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0; 2774 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd)); 2775 } 2776 2777 2778 void Assembler::fcvtl2(const VRegister& vd, 2779 const VRegister& vn) { 2780 VIXL_ASSERT((vd.Is4S() && vn.Is8H()) || 2781 (vd.Is2D() && vn.Is4S())); 2782 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0; 2783 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd)); 2784 } 2785 2786 2787 void Assembler::fcvtn(const VRegister& vd, 2788 const VRegister& vn) { 2789 VIXL_ASSERT((vn.Is4S() && vd.Is4H()) || 2790 (vn.Is2D() && vd.Is2S())); 2791 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0; 2792 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd)); 2793 } 2794 2795 2796 void Assembler::fcvtn2(const VRegister& vd, 2797 const VRegister& vn) { 2798 VIXL_ASSERT((vn.Is4S() && vd.Is8H()) || 2799 (vn.Is2D() && vd.Is4S())); 2800 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0; 2801 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd)); 2802 } 2803 2804 2805 void Assembler::fcvtxn(const VRegister& vd, 2806 const VRegister& vn) { 2807 Instr format = 1 << NEONSize_offset; 2808 if (vd.IsScalar()) { 2809 VIXL_ASSERT(vd.Is1S() && vn.Is1D()); 2810 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd)); 2811 } else { 2812 VIXL_ASSERT(vd.Is2S() && vn.Is2D()); 2813 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd)); 2814 } 2815 } 2816 2817 2818 void Assembler::fcvtxn2(const VRegister& vd, 2819 const VRegister& vn) { 2820 VIXL_ASSERT(vd.Is4S() && vn.Is2D()); 2821 Instr format = 1 << NEONSize_offset; 2822 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd)); 2823 } 2824 2825 2826 #define NEON_FP2REGMISC_FCVT_LIST(V) \ 2827 V(fcvtnu, NEON_FCVTNU, FCVTNU) \ 2828 V(fcvtns, NEON_FCVTNS, FCVTNS) \ 2829 V(fcvtpu, NEON_FCVTPU, FCVTPU) \ 2830 V(fcvtps, NEON_FCVTPS, FCVTPS) \ 2831 V(fcvtmu, NEON_FCVTMU, FCVTMU) \ 2832 V(fcvtms, NEON_FCVTMS, FCVTMS) \ 2833 V(fcvtau, NEON_FCVTAU, FCVTAU) \ 2834 V(fcvtas, NEON_FCVTAS, FCVTAS) 2835 2836 #define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \ 2837 void Assembler::FN(const Register& rd, \ 2838 const VRegister& vn) { \ 2839 NEONFPConvertToInt(rd, vn, SCA_OP); \ 2840 } \ 2841 void Assembler::FN(const VRegister& vd, \ 2842 const VRegister& vn) { \ 2843 NEONFPConvertToInt(vd, vn, VEC_OP); \ 2844 } 2845 NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS) 2846 #undef DEFINE_ASM_FUNCS 2847 2848 2849 void Assembler::fcvtzs(const Register& rd, 2850 const VRegister& vn, 2851 int fbits) { 2852 VIXL_ASSERT(vn.Is1S() || vn.Is1D()); 2853 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits())); 2854 if (fbits == 0) { 2855 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd)); 2856 } else { 2857 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) | 2858 Rd(rd)); 2859 } 2860 } 2861 2862 2863 void Assembler::fcvtzs(const VRegister& vd, 2864 const VRegister& vn, 2865 int fbits) { 2866 VIXL_ASSERT(fbits >= 0); 2867 if (fbits == 0) { 2868 NEONFP2RegMisc(vd, vn, NEON_FCVTZS); 2869 } else { 2870 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 2871 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm); 2872 } 2873 } 2874 2875 2876 void Assembler::fcvtzu(const Register& rd, 2877 const VRegister& vn, 2878 int fbits) { 2879 VIXL_ASSERT(vn.Is1S() || vn.Is1D()); 2880 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits())); 2881 if (fbits == 0) { 2882 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd)); 2883 } else { 2884 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) | 2885 Rd(rd)); 2886 } 2887 } 2888 2889 2890 void Assembler::fcvtzu(const VRegister& vd, 2891 const VRegister& vn, 2892 int fbits) { 2893 VIXL_ASSERT(fbits >= 0); 2894 if (fbits == 0) { 2895 NEONFP2RegMisc(vd, vn, NEON_FCVTZU); 2896 } else { 2897 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 2898 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm); 2899 } 2900 } 2901 2902 void Assembler::ucvtf(const VRegister& vd, 2903 const VRegister& vn, 2904 int fbits) { 2905 VIXL_ASSERT(fbits >= 0); 2906 if (fbits == 0) { 2907 NEONFP2RegMisc(vd, vn, NEON_UCVTF); 2908 } else { 2909 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 2910 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm); 2911 } 2912 } 2913 2914 void Assembler::scvtf(const VRegister& vd, 2915 const VRegister& vn, 2916 int fbits) { 2917 VIXL_ASSERT(fbits >= 0); 2918 if (fbits == 0) { 2919 NEONFP2RegMisc(vd, vn, NEON_SCVTF); 2920 } else { 2921 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S()); 2922 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm); 2923 } 2924 } 2925 2926 2927 void Assembler::scvtf(const VRegister& vd, 2928 const Register& rn, 2929 int fbits) { 2930 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 2931 VIXL_ASSERT(fbits >= 0); 2932 if (fbits == 0) { 2933 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd)); 2934 } else { 2935 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 2936 Rd(vd)); 2937 } 2938 } 2939 2940 2941 void Assembler::ucvtf(const VRegister& vd, 2942 const Register& rn, 2943 int fbits) { 2944 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 2945 VIXL_ASSERT(fbits >= 0); 2946 if (fbits == 0) { 2947 Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd)); 2948 } else { 2949 Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) | 2950 Rd(vd)); 2951 } 2952 } 2953 2954 2955 void Assembler::NEON3Same(const VRegister& vd, 2956 const VRegister& vn, 2957 const VRegister& vm, 2958 NEON3SameOp vop) { 2959 VIXL_ASSERT(AreSameFormat(vd, vn, vm)); 2960 VIXL_ASSERT(vd.IsVector() || !vd.IsQ()); 2961 2962 Instr format, op = vop; 2963 if (vd.IsScalar()) { 2964 op |= NEON_Q | NEONScalar; 2965 format = SFormat(vd); 2966 } else { 2967 format = VFormat(vd); 2968 } 2969 2970 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd)); 2971 } 2972 2973 2974 void Assembler::NEONFP3Same(const VRegister& vd, 2975 const VRegister& vn, 2976 const VRegister& vm, 2977 Instr op) { 2978 VIXL_ASSERT(AreSameFormat(vd, vn, vm)); 2979 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd)); 2980 } 2981 2982 2983 #define NEON_FP2REGMISC_LIST(V) \ 2984 V(fabs, NEON_FABS, FABS) \ 2985 V(fneg, NEON_FNEG, FNEG) \ 2986 V(fsqrt, NEON_FSQRT, FSQRT) \ 2987 V(frintn, NEON_FRINTN, FRINTN) \ 2988 V(frinta, NEON_FRINTA, FRINTA) \ 2989 V(frintp, NEON_FRINTP, FRINTP) \ 2990 V(frintm, NEON_FRINTM, FRINTM) \ 2991 V(frintx, NEON_FRINTX, FRINTX) \ 2992 V(frintz, NEON_FRINTZ, FRINTZ) \ 2993 V(frinti, NEON_FRINTI, FRINTI) \ 2994 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \ 2995 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar ) 2996 2997 2998 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \ 2999 void Assembler::FN(const VRegister& vd, \ 3000 const VRegister& vn) { \ 3001 Instr op; \ 3002 if (vd.IsScalar()) { \ 3003 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \ 3004 op = SCA_OP; \ 3005 } else { \ 3006 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \ 3007 op = VEC_OP; \ 3008 } \ 3009 NEONFP2RegMisc(vd, vn, op); \ 3010 } 3011 NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC) 3012 #undef DEFINE_ASM_FUNC 3013 3014 3015 void Assembler::NEONFP2RegMisc(const VRegister& vd, 3016 const VRegister& vn, 3017 Instr op) { 3018 VIXL_ASSERT(AreSameFormat(vd, vn)); 3019 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd)); 3020 } 3021 3022 3023 void Assembler::NEON2RegMisc(const VRegister& vd, 3024 const VRegister& vn, 3025 NEON2RegMiscOp vop, 3026 int value) { 3027 VIXL_ASSERT(AreSameFormat(vd, vn)); 3028 VIXL_ASSERT(value == 0); 3029 USE(value); 3030 3031 Instr format, op = vop; 3032 if (vd.IsScalar()) { 3033 op |= NEON_Q | NEONScalar; 3034 format = SFormat(vd); 3035 } else { 3036 format = VFormat(vd); 3037 } 3038 3039 Emit(format | op | Rn(vn) | Rd(vd)); 3040 } 3041 3042 3043 void Assembler::cmeq(const VRegister& vd, 3044 const VRegister& vn, 3045 int value) { 3046 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3047 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value); 3048 } 3049 3050 3051 void Assembler::cmge(const VRegister& vd, 3052 const VRegister& vn, 3053 int value) { 3054 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3055 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value); 3056 } 3057 3058 3059 void Assembler::cmgt(const VRegister& vd, 3060 const VRegister& vn, 3061 int value) { 3062 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3063 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value); 3064 } 3065 3066 3067 void Assembler::cmle(const VRegister& vd, 3068 const VRegister& vn, 3069 int value) { 3070 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3071 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value); 3072 } 3073 3074 3075 void Assembler::cmlt(const VRegister& vd, 3076 const VRegister& vn, 3077 int value) { 3078 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3079 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value); 3080 } 3081 3082 3083 void Assembler::shll(const VRegister& vd, 3084 const VRegister& vn, 3085 int shift) { 3086 VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) || 3087 (vd.Is4S() && vn.Is4H() && shift == 16) || 3088 (vd.Is2D() && vn.Is2S() && shift == 32)); 3089 USE(shift); 3090 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd)); 3091 } 3092 3093 3094 void Assembler::shll2(const VRegister& vd, 3095 const VRegister& vn, 3096 int shift) { 3097 USE(shift); 3098 VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) || 3099 (vd.Is4S() && vn.Is8H() && shift == 16) || 3100 (vd.Is2D() && vn.Is4S() && shift == 32)); 3101 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd)); 3102 } 3103 3104 3105 void Assembler::NEONFP2RegMisc(const VRegister& vd, 3106 const VRegister& vn, 3107 NEON2RegMiscOp vop, 3108 double value) { 3109 VIXL_ASSERT(AreSameFormat(vd, vn)); 3110 VIXL_ASSERT(value == 0.0); 3111 USE(value); 3112 3113 Instr op = vop; 3114 if (vd.IsScalar()) { 3115 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 3116 op |= NEON_Q | NEONScalar; 3117 } else { 3118 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); 3119 } 3120 3121 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd)); 3122 } 3123 3124 3125 void Assembler::fcmeq(const VRegister& vd, 3126 const VRegister& vn, 3127 double value) { 3128 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value); 3129 } 3130 3131 3132 void Assembler::fcmge(const VRegister& vd, 3133 const VRegister& vn, 3134 double value) { 3135 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value); 3136 } 3137 3138 3139 void Assembler::fcmgt(const VRegister& vd, 3140 const VRegister& vn, 3141 double value) { 3142 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value); 3143 } 3144 3145 3146 void Assembler::fcmle(const VRegister& vd, 3147 const VRegister& vn, 3148 double value) { 3149 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value); 3150 } 3151 3152 3153 void Assembler::fcmlt(const VRegister& vd, 3154 const VRegister& vn, 3155 double value) { 3156 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value); 3157 } 3158 3159 3160 void Assembler::frecpx(const VRegister& vd, 3161 const VRegister& vn) { 3162 VIXL_ASSERT(vd.IsScalar()); 3163 VIXL_ASSERT(AreSameFormat(vd, vn)); 3164 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 3165 Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd)); 3166 } 3167 3168 3169 #define NEON_3SAME_LIST(V) \ 3170 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \ 3171 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \ 3172 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \ 3173 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \ 3174 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \ 3175 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \ 3176 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \ 3177 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \ 3178 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \ 3179 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \ 3180 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \ 3181 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \ 3182 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \ 3183 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \ 3184 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \ 3185 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3186 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3187 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3188 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3189 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \ 3190 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \ 3191 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \ 3192 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3193 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \ 3194 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3195 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \ 3196 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3197 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \ 3198 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \ 3199 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \ 3200 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3201 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \ 3202 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \ 3203 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \ 3204 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \ 3205 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \ 3206 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \ 3207 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \ 3208 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \ 3209 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \ 3210 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \ 3211 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \ 3212 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \ 3213 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \ 3214 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \ 3215 V(uqadd, NEON_UQADD, true) \ 3216 V(sqadd, NEON_SQADD, true) \ 3217 V(uqsub, NEON_UQSUB, true) \ 3218 V(sqsub, NEON_SQSUB, true) \ 3219 V(sqshl, NEON_SQSHL, true) \ 3220 V(uqshl, NEON_UQSHL, true) \ 3221 V(sqrshl, NEON_SQRSHL, true) \ 3222 V(uqrshl, NEON_UQRSHL, true) 3223 3224 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 3225 void Assembler::FN(const VRegister& vd, \ 3226 const VRegister& vn, \ 3227 const VRegister& vm) { \ 3228 VIXL_ASSERT(AS); \ 3229 NEON3Same(vd, vn, vm, OP); \ 3230 } 3231 NEON_3SAME_LIST(DEFINE_ASM_FUNC) 3232 #undef DEFINE_ASM_FUNC 3233 3234 3235 #define NEON_FP3SAME_LIST(V) \ 3236 V(fadd, NEON_FADD, FADD) \ 3237 V(fsub, NEON_FSUB, FSUB) \ 3238 V(fmul, NEON_FMUL, FMUL) \ 3239 V(fdiv, NEON_FDIV, FDIV) \ 3240 V(fmax, NEON_FMAX, FMAX) \ 3241 V(fmaxnm, NEON_FMAXNM, FMAXNM) \ 3242 V(fmin, NEON_FMIN, FMIN) \ 3243 V(fminnm, NEON_FMINNM, FMINNM) \ 3244 V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \ 3245 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \ 3246 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \ 3247 V(fabd, NEON_FABD, NEON_FABD_scalar) \ 3248 V(fmla, NEON_FMLA, 0) \ 3249 V(fmls, NEON_FMLS, 0) \ 3250 V(facge, NEON_FACGE, NEON_FACGE_scalar) \ 3251 V(facgt, NEON_FACGT, NEON_FACGT_scalar) \ 3252 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \ 3253 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \ 3254 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \ 3255 V(faddp, NEON_FADDP, 0) \ 3256 V(fmaxp, NEON_FMAXP, 0) \ 3257 V(fminp, NEON_FMINP, 0) \ 3258 V(fmaxnmp, NEON_FMAXNMP, 0) \ 3259 V(fminnmp, NEON_FMINNMP, 0) 3260 3261 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \ 3262 void Assembler::FN(const VRegister& vd, \ 3263 const VRegister& vn, \ 3264 const VRegister& vm) { \ 3265 Instr op; \ 3266 if ((SCA_OP != 0) && vd.IsScalar()) { \ 3267 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \ 3268 op = SCA_OP; \ 3269 } else { \ 3270 VIXL_ASSERT(vd.IsVector()); \ 3271 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \ 3272 op = VEC_OP; \ 3273 } \ 3274 NEONFP3Same(vd, vn, vm, op); \ 3275 } 3276 NEON_FP3SAME_LIST(DEFINE_ASM_FUNC) 3277 #undef DEFINE_ASM_FUNC 3278 3279 3280 void Assembler::addp(const VRegister& vd, 3281 const VRegister& vn) { 3282 VIXL_ASSERT((vd.Is1D() && vn.Is2D())); 3283 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd)); 3284 } 3285 3286 3287 void Assembler::faddp(const VRegister& vd, 3288 const VRegister& vn) { 3289 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || 3290 (vd.Is1D() && vn.Is2D())); 3291 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd)); 3292 } 3293 3294 3295 void Assembler::fmaxp(const VRegister& vd, 3296 const VRegister& vn) { 3297 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || 3298 (vd.Is1D() && vn.Is2D())); 3299 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd)); 3300 } 3301 3302 3303 void Assembler::fminp(const VRegister& vd, 3304 const VRegister& vn) { 3305 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || 3306 (vd.Is1D() && vn.Is2D())); 3307 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd)); 3308 } 3309 3310 3311 void Assembler::fmaxnmp(const VRegister& vd, 3312 const VRegister& vn) { 3313 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || 3314 (vd.Is1D() && vn.Is2D())); 3315 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd)); 3316 } 3317 3318 3319 void Assembler::fminnmp(const VRegister& vd, 3320 const VRegister& vn) { 3321 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || 3322 (vd.Is1D() && vn.Is2D())); 3323 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd)); 3324 } 3325 3326 3327 void Assembler::orr(const VRegister& vd, 3328 const int imm8, 3329 const int left_shift) { 3330 NEONModifiedImmShiftLsl(vd, imm8, left_shift, 3331 NEONModifiedImmediate_ORR); 3332 } 3333 3334 3335 void Assembler::mov(const VRegister& vd, 3336 const VRegister& vn) { 3337 VIXL_ASSERT(AreSameFormat(vd, vn)); 3338 if (vd.IsD()) { 3339 orr(vd.V8B(), vn.V8B(), vn.V8B()); 3340 } else { 3341 VIXL_ASSERT(vd.IsQ()); 3342 orr(vd.V16B(), vn.V16B(), vn.V16B()); 3343 } 3344 } 3345 3346 3347 void Assembler::bic(const VRegister& vd, 3348 const int imm8, 3349 const int left_shift) { 3350 NEONModifiedImmShiftLsl(vd, imm8, left_shift, 3351 NEONModifiedImmediate_BIC); 3352 } 3353 3354 3355 void Assembler::movi(const VRegister& vd, 3356 const uint64_t imm, 3357 Shift shift, 3358 const int shift_amount) { 3359 VIXL_ASSERT((shift == LSL) || (shift == MSL)); 3360 if (vd.Is2D() || vd.Is1D()) { 3361 VIXL_ASSERT(shift_amount == 0); 3362 int imm8 = 0; 3363 for (int i = 0; i < 8; ++i) { 3364 int byte = (imm >> (i * 8)) & 0xff; 3365 VIXL_ASSERT((byte == 0) || (byte == 0xff)); 3366 if (byte == 0xff) { 3367 imm8 |= (1 << i); 3368 } 3369 } 3370 int q = vd.Is2D() ? NEON_Q : 0; 3371 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI | 3372 ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd)); 3373 } else if (shift == LSL) { 3374 NEONModifiedImmShiftLsl(vd, imm, shift_amount, 3375 NEONModifiedImmediate_MOVI); 3376 } else { 3377 NEONModifiedImmShiftMsl(vd, imm, shift_amount, 3378 NEONModifiedImmediate_MOVI); 3379 } 3380 } 3381 3382 3383 void Assembler::mvn(const VRegister& vd, 3384 const VRegister& vn) { 3385 VIXL_ASSERT(AreSameFormat(vd, vn)); 3386 if (vd.IsD()) { 3387 not_(vd.V8B(), vn.V8B()); 3388 } else { 3389 VIXL_ASSERT(vd.IsQ()); 3390 not_(vd.V16B(), vn.V16B()); 3391 } 3392 } 3393 3394 3395 void Assembler::mvni(const VRegister& vd, 3396 const int imm8, 3397 Shift shift, 3398 const int shift_amount) { 3399 VIXL_ASSERT((shift == LSL) || (shift == MSL)); 3400 if (shift == LSL) { 3401 NEONModifiedImmShiftLsl(vd, imm8, shift_amount, 3402 NEONModifiedImmediate_MVNI); 3403 } else { 3404 NEONModifiedImmShiftMsl(vd, imm8, shift_amount, 3405 NEONModifiedImmediate_MVNI); 3406 } 3407 } 3408 3409 3410 void Assembler::NEONFPByElement(const VRegister& vd, 3411 const VRegister& vn, 3412 const VRegister& vm, 3413 int vm_index, 3414 NEONByIndexedElementOp vop) { 3415 VIXL_ASSERT(AreSameFormat(vd, vn)); 3416 VIXL_ASSERT((vd.Is2S() && vm.Is1S()) || 3417 (vd.Is4S() && vm.Is1S()) || 3418 (vd.Is1S() && vm.Is1S()) || 3419 (vd.Is2D() && vm.Is1D()) || 3420 (vd.Is1D() && vm.Is1D())); 3421 VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) || 3422 (vm.Is1D() && (vm_index < 2))); 3423 3424 Instr op = vop; 3425 int index_num_bits = vm.Is1S() ? 2 : 1; 3426 if (vd.IsScalar()) { 3427 op |= NEON_Q | NEONScalar; 3428 } 3429 3430 Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) | 3431 Rm(vm) | Rn(vn) | Rd(vd)); 3432 } 3433 3434 3435 void Assembler::NEONByElement(const VRegister& vd, 3436 const VRegister& vn, 3437 const VRegister& vm, 3438 int vm_index, 3439 NEONByIndexedElementOp vop) { 3440 VIXL_ASSERT(AreSameFormat(vd, vn)); 3441 VIXL_ASSERT((vd.Is4H() && vm.Is1H()) || 3442 (vd.Is8H() && vm.Is1H()) || 3443 (vd.Is1H() && vm.Is1H()) || 3444 (vd.Is2S() && vm.Is1S()) || 3445 (vd.Is4S() && vm.Is1S()) || 3446 (vd.Is1S() && vm.Is1S())); 3447 VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) || 3448 (vm.Is1S() && (vm_index < 4))); 3449 3450 Instr format, op = vop; 3451 int index_num_bits = vm.Is1H() ? 3 : 2; 3452 if (vd.IsScalar()) { 3453 op |= NEONScalar | NEON_Q; 3454 format = SFormat(vn); 3455 } else { 3456 format = VFormat(vn); 3457 } 3458 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | 3459 Rm(vm) | Rn(vn) | Rd(vd)); 3460 } 3461 3462 3463 void Assembler::NEONByElementL(const VRegister& vd, 3464 const VRegister& vn, 3465 const VRegister& vm, 3466 int vm_index, 3467 NEONByIndexedElementOp vop) { 3468 VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) || 3469 (vd.Is4S() && vn.Is8H() && vm.Is1H()) || 3470 (vd.Is1S() && vn.Is1H() && vm.Is1H()) || 3471 (vd.Is2D() && vn.Is2S() && vm.Is1S()) || 3472 (vd.Is2D() && vn.Is4S() && vm.Is1S()) || 3473 (vd.Is1D() && vn.Is1S() && vm.Is1S())); 3474 3475 VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) || 3476 (vm.Is1S() && (vm_index < 4))); 3477 3478 Instr format, op = vop; 3479 int index_num_bits = vm.Is1H() ? 3 : 2; 3480 if (vd.IsScalar()) { 3481 op |= NEONScalar | NEON_Q; 3482 format = SFormat(vn); 3483 } else { 3484 format = VFormat(vn); 3485 } 3486 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | 3487 Rm(vm) | Rn(vn) | Rd(vd)); 3488 } 3489 3490 3491 #define NEON_BYELEMENT_LIST(V) \ 3492 V(mul, NEON_MUL_byelement, vn.IsVector()) \ 3493 V(mla, NEON_MLA_byelement, vn.IsVector()) \ 3494 V(mls, NEON_MLS_byelement, vn.IsVector()) \ 3495 V(sqdmulh, NEON_SQDMULH_byelement, true) \ 3496 V(sqrdmulh, NEON_SQRDMULH_byelement, true) 3497 3498 3499 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 3500 void Assembler::FN(const VRegister& vd, \ 3501 const VRegister& vn, \ 3502 const VRegister& vm, \ 3503 int vm_index) { \ 3504 VIXL_ASSERT(AS); \ 3505 NEONByElement(vd, vn, vm, vm_index, OP); \ 3506 } 3507 NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC) 3508 #undef DEFINE_ASM_FUNC 3509 3510 3511 #define NEON_FPBYELEMENT_LIST(V) \ 3512 V(fmul, NEON_FMUL_byelement) \ 3513 V(fmla, NEON_FMLA_byelement) \ 3514 V(fmls, NEON_FMLS_byelement) \ 3515 V(fmulx, NEON_FMULX_byelement) 3516 3517 3518 #define DEFINE_ASM_FUNC(FN, OP) \ 3519 void Assembler::FN(const VRegister& vd, \ 3520 const VRegister& vn, \ 3521 const VRegister& vm, \ 3522 int vm_index) { \ 3523 NEONFPByElement(vd, vn, vm, vm_index, OP); \ 3524 } 3525 NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC) 3526 #undef DEFINE_ASM_FUNC 3527 3528 3529 #define NEON_BYELEMENT_LONG_LIST(V) \ 3530 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \ 3531 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \ 3532 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \ 3533 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \ 3534 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \ 3535 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \ 3536 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \ 3537 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \ 3538 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \ 3539 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \ 3540 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \ 3541 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \ 3542 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \ 3543 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \ 3544 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \ 3545 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \ 3546 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \ 3547 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ()) 3548 3549 3550 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 3551 void Assembler::FN(const VRegister& vd, \ 3552 const VRegister& vn, \ 3553 const VRegister& vm, \ 3554 int vm_index) { \ 3555 VIXL_ASSERT(AS); \ 3556 NEONByElementL(vd, vn, vm, vm_index, OP); \ 3557 } 3558 NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC) 3559 #undef DEFINE_ASM_FUNC 3560 3561 3562 void Assembler::suqadd(const VRegister& vd, 3563 const VRegister& vn) { 3564 NEON2RegMisc(vd, vn, NEON_SUQADD); 3565 } 3566 3567 3568 void Assembler::usqadd(const VRegister& vd, 3569 const VRegister& vn) { 3570 NEON2RegMisc(vd, vn, NEON_USQADD); 3571 } 3572 3573 3574 void Assembler::abs(const VRegister& vd, 3575 const VRegister& vn) { 3576 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3577 NEON2RegMisc(vd, vn, NEON_ABS); 3578 } 3579 3580 3581 void Assembler::sqabs(const VRegister& vd, 3582 const VRegister& vn) { 3583 NEON2RegMisc(vd, vn, NEON_SQABS); 3584 } 3585 3586 3587 void Assembler::neg(const VRegister& vd, 3588 const VRegister& vn) { 3589 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 3590 NEON2RegMisc(vd, vn, NEON_NEG); 3591 } 3592 3593 3594 void Assembler::sqneg(const VRegister& vd, 3595 const VRegister& vn) { 3596 NEON2RegMisc(vd, vn, NEON_SQNEG); 3597 } 3598 3599 3600 void Assembler::NEONXtn(const VRegister& vd, 3601 const VRegister& vn, 3602 NEON2RegMiscOp vop) { 3603 Instr format, op = vop; 3604 if (vd.IsScalar()) { 3605 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || 3606 (vd.Is1H() && vn.Is1S()) || 3607 (vd.Is1S() && vn.Is1D())); 3608 op |= NEON_Q | NEONScalar; 3609 format = SFormat(vd); 3610 } else { 3611 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || 3612 (vd.Is4H() && vn.Is4S()) || 3613 (vd.Is2S() && vn.Is2D()) || 3614 (vd.Is16B() && vn.Is8H()) || 3615 (vd.Is8H() && vn.Is4S()) || 3616 (vd.Is4S() && vn.Is2D())); 3617 format = VFormat(vd); 3618 } 3619 Emit(format | op | Rn(vn) | Rd(vd)); 3620 } 3621 3622 3623 void Assembler::xtn(const VRegister& vd, 3624 const VRegister& vn) { 3625 VIXL_ASSERT(vd.IsVector() && vd.IsD()); 3626 NEONXtn(vd, vn, NEON_XTN); 3627 } 3628 3629 3630 void Assembler::xtn2(const VRegister& vd, 3631 const VRegister& vn) { 3632 VIXL_ASSERT(vd.IsVector() && vd.IsQ()); 3633 NEONXtn(vd, vn, NEON_XTN); 3634 } 3635 3636 3637 void Assembler::sqxtn(const VRegister& vd, 3638 const VRegister& vn) { 3639 VIXL_ASSERT(vd.IsScalar() || vd.IsD()); 3640 NEONXtn(vd, vn, NEON_SQXTN); 3641 } 3642 3643 3644 void Assembler::sqxtn2(const VRegister& vd, 3645 const VRegister& vn) { 3646 VIXL_ASSERT(vd.IsVector() && vd.IsQ()); 3647 NEONXtn(vd, vn, NEON_SQXTN); 3648 } 3649 3650 3651 void Assembler::sqxtun(const VRegister& vd, 3652 const VRegister& vn) { 3653 VIXL_ASSERT(vd.IsScalar() || vd.IsD()); 3654 NEONXtn(vd, vn, NEON_SQXTUN); 3655 } 3656 3657 3658 void Assembler::sqxtun2(const VRegister& vd, 3659 const VRegister& vn) { 3660 VIXL_ASSERT(vd.IsVector() && vd.IsQ()); 3661 NEONXtn(vd, vn, NEON_SQXTUN); 3662 } 3663 3664 3665 void Assembler::uqxtn(const VRegister& vd, 3666 const VRegister& vn) { 3667 VIXL_ASSERT(vd.IsScalar() || vd.IsD()); 3668 NEONXtn(vd, vn, NEON_UQXTN); 3669 } 3670 3671 3672 void Assembler::uqxtn2(const VRegister& vd, 3673 const VRegister& vn) { 3674 VIXL_ASSERT(vd.IsVector() && vd.IsQ()); 3675 NEONXtn(vd, vn, NEON_UQXTN); 3676 } 3677 3678 3679 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size". 3680 void Assembler::not_(const VRegister& vd, 3681 const VRegister& vn) { 3682 VIXL_ASSERT(AreSameFormat(vd, vn)); 3683 VIXL_ASSERT(vd.Is8B() || vd.Is16B()); 3684 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd)); 3685 } 3686 3687 3688 void Assembler::rbit(const VRegister& vd, 3689 const VRegister& vn) { 3690 VIXL_ASSERT(AreSameFormat(vd, vn)); 3691 VIXL_ASSERT(vd.Is8B() || vd.Is16B()); 3692 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd)); 3693 } 3694 3695 3696 void Assembler::ext(const VRegister& vd, 3697 const VRegister& vn, 3698 const VRegister& vm, 3699 int index) { 3700 VIXL_ASSERT(AreSameFormat(vd, vn, vm)); 3701 VIXL_ASSERT(vd.Is8B() || vd.Is16B()); 3702 VIXL_ASSERT((0 <= index) && (index < vd.lanes())); 3703 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd)); 3704 } 3705 3706 3707 void Assembler::dup(const VRegister& vd, 3708 const VRegister& vn, 3709 int vn_index) { 3710 Instr q, scalar; 3711 3712 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the 3713 // number of lanes, and T is b, h, s or d. 3714 int lane_size = vn.LaneSizeInBytes(); 3715 NEONFormatField format; 3716 switch (lane_size) { 3717 case 1: format = NEON_16B; break; 3718 case 2: format = NEON_8H; break; 3719 case 4: format = NEON_4S; break; 3720 default: 3721 VIXL_ASSERT(lane_size == 8); 3722 format = NEON_2D; 3723 break; 3724 } 3725 3726 if (vd.IsScalar()) { 3727 q = NEON_Q; 3728 scalar = NEONScalar; 3729 } else { 3730 VIXL_ASSERT(!vd.Is1D()); 3731 q = vd.IsD() ? 0 : NEON_Q; 3732 scalar = 0; 3733 } 3734 Emit(q | scalar | NEON_DUP_ELEMENT | 3735 ImmNEON5(format, vn_index) | Rn(vn) | Rd(vd)); 3736 } 3737 3738 3739 void Assembler::mov(const VRegister& vd, 3740 const VRegister& vn, 3741 int vn_index) { 3742 VIXL_ASSERT(vn.IsScalar()); 3743 dup(vd, vn, vn_index); 3744 } 3745 3746 3747 void Assembler::dup(const VRegister& vd, const Register& rn) { 3748 VIXL_ASSERT(!vd.Is1D()); 3749 VIXL_ASSERT(vd.Is2D() == rn.IsX()); 3750 int q = vd.IsD() ? 0 : NEON_Q; 3751 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd)); 3752 } 3753 3754 3755 void Assembler::ins(const VRegister& vd, 3756 int vd_index, 3757 const VRegister& vn, 3758 int vn_index) { 3759 VIXL_ASSERT(AreSameFormat(vd, vn)); 3760 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the 3761 // number of lanes, and T is b, h, s or d. 3762 int lane_size = vd.LaneSizeInBytes(); 3763 NEONFormatField format; 3764 switch (lane_size) { 3765 case 1: format = NEON_16B; break; 3766 case 2: format = NEON_8H; break; 3767 case 4: format = NEON_4S; break; 3768 default: 3769 VIXL_ASSERT(lane_size == 8); 3770 format = NEON_2D; 3771 break; 3772 } 3773 3774 VIXL_ASSERT((0 <= vd_index) && 3775 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 3776 VIXL_ASSERT((0 <= vn_index) && 3777 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 3778 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) | 3779 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd)); 3780 } 3781 3782 3783 void Assembler::mov(const VRegister& vd, 3784 int vd_index, 3785 const VRegister& vn, 3786 int vn_index) { 3787 ins(vd, vd_index, vn, vn_index); 3788 } 3789 3790 3791 void Assembler::ins(const VRegister& vd, 3792 int vd_index, 3793 const Register& rn) { 3794 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the 3795 // number of lanes, and T is b, h, s or d. 3796 int lane_size = vd.LaneSizeInBytes(); 3797 NEONFormatField format; 3798 switch (lane_size) { 3799 case 1: format = NEON_16B; VIXL_ASSERT(rn.IsW()); break; 3800 case 2: format = NEON_8H; VIXL_ASSERT(rn.IsW()); break; 3801 case 4: format = NEON_4S; VIXL_ASSERT(rn.IsW()); break; 3802 default: 3803 VIXL_ASSERT(lane_size == 8); 3804 VIXL_ASSERT(rn.IsX()); 3805 format = NEON_2D; 3806 break; 3807 } 3808 3809 VIXL_ASSERT((0 <= vd_index) && 3810 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 3811 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd)); 3812 } 3813 3814 3815 void Assembler::mov(const VRegister& vd, 3816 int vd_index, 3817 const Register& rn) { 3818 ins(vd, vd_index, rn); 3819 } 3820 3821 3822 void Assembler::umov(const Register& rd, 3823 const VRegister& vn, 3824 int vn_index) { 3825 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the 3826 // number of lanes, and T is b, h, s or d. 3827 int lane_size = vn.LaneSizeInBytes(); 3828 NEONFormatField format; 3829 Instr q = 0; 3830 switch (lane_size) { 3831 case 1: format = NEON_16B; VIXL_ASSERT(rd.IsW()); break; 3832 case 2: format = NEON_8H; VIXL_ASSERT(rd.IsW()); break; 3833 case 4: format = NEON_4S; VIXL_ASSERT(rd.IsW()); break; 3834 default: 3835 VIXL_ASSERT(lane_size == 8); 3836 VIXL_ASSERT(rd.IsX()); 3837 format = NEON_2D; 3838 q = NEON_Q; 3839 break; 3840 } 3841 3842 VIXL_ASSERT((0 <= vn_index) && 3843 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 3844 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd)); 3845 } 3846 3847 3848 void Assembler::mov(const Register& rd, 3849 const VRegister& vn, 3850 int vn_index) { 3851 VIXL_ASSERT(vn.SizeInBytes() >= 4); 3852 umov(rd, vn, vn_index); 3853 } 3854 3855 3856 void Assembler::smov(const Register& rd, 3857 const VRegister& vn, 3858 int vn_index) { 3859 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the 3860 // number of lanes, and T is b, h, s. 3861 int lane_size = vn.LaneSizeInBytes(); 3862 NEONFormatField format; 3863 Instr q = 0; 3864 VIXL_ASSERT(lane_size != 8); 3865 switch (lane_size) { 3866 case 1: format = NEON_16B; break; 3867 case 2: format = NEON_8H; break; 3868 default: 3869 VIXL_ASSERT(lane_size == 4); 3870 VIXL_ASSERT(rd.IsX()); 3871 format = NEON_4S; 3872 break; 3873 } 3874 q = rd.IsW() ? 0 : NEON_Q; 3875 VIXL_ASSERT((0 <= vn_index) && 3876 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format)))); 3877 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd)); 3878 } 3879 3880 3881 void Assembler::cls(const VRegister& vd, 3882 const VRegister& vn) { 3883 VIXL_ASSERT(AreSameFormat(vd, vn)); 3884 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D()); 3885 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd)); 3886 } 3887 3888 3889 void Assembler::clz(const VRegister& vd, 3890 const VRegister& vn) { 3891 VIXL_ASSERT(AreSameFormat(vd, vn)); 3892 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D()); 3893 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd)); 3894 } 3895 3896 3897 void Assembler::cnt(const VRegister& vd, 3898 const VRegister& vn) { 3899 VIXL_ASSERT(AreSameFormat(vd, vn)); 3900 VIXL_ASSERT(vd.Is8B() || vd.Is16B()); 3901 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd)); 3902 } 3903 3904 3905 void Assembler::rev16(const VRegister& vd, 3906 const VRegister& vn) { 3907 VIXL_ASSERT(AreSameFormat(vd, vn)); 3908 VIXL_ASSERT(vd.Is8B() || vd.Is16B()); 3909 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd)); 3910 } 3911 3912 3913 void Assembler::rev32(const VRegister& vd, 3914 const VRegister& vn) { 3915 VIXL_ASSERT(AreSameFormat(vd, vn)); 3916 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H()); 3917 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd)); 3918 } 3919 3920 3921 void Assembler::rev64(const VRegister& vd, 3922 const VRegister& vn) { 3923 VIXL_ASSERT(AreSameFormat(vd, vn)); 3924 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D()); 3925 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd)); 3926 } 3927 3928 3929 void Assembler::ursqrte(const VRegister& vd, 3930 const VRegister& vn) { 3931 VIXL_ASSERT(AreSameFormat(vd, vn)); 3932 VIXL_ASSERT(vd.Is2S() || vd.Is4S()); 3933 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd)); 3934 } 3935 3936 3937 void Assembler::urecpe(const VRegister& vd, 3938 const VRegister& vn) { 3939 VIXL_ASSERT(AreSameFormat(vd, vn)); 3940 VIXL_ASSERT(vd.Is2S() || vd.Is4S()); 3941 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd)); 3942 } 3943 3944 3945 void Assembler::NEONAddlp(const VRegister& vd, 3946 const VRegister& vn, 3947 NEON2RegMiscOp op) { 3948 VIXL_ASSERT((op == NEON_SADDLP) || 3949 (op == NEON_UADDLP) || 3950 (op == NEON_SADALP) || 3951 (op == NEON_UADALP)); 3952 3953 VIXL_ASSERT((vn.Is8B() && vd.Is4H()) || 3954 (vn.Is4H() && vd.Is2S()) || 3955 (vn.Is2S() && vd.Is1D()) || 3956 (vn.Is16B() && vd.Is8H())|| 3957 (vn.Is8H() && vd.Is4S()) || 3958 (vn.Is4S() && vd.Is2D())); 3959 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd)); 3960 } 3961 3962 3963 void Assembler::saddlp(const VRegister& vd, 3964 const VRegister& vn) { 3965 NEONAddlp(vd, vn, NEON_SADDLP); 3966 } 3967 3968 3969 void Assembler::uaddlp(const VRegister& vd, 3970 const VRegister& vn) { 3971 NEONAddlp(vd, vn, NEON_UADDLP); 3972 } 3973 3974 3975 void Assembler::sadalp(const VRegister& vd, 3976 const VRegister& vn) { 3977 NEONAddlp(vd, vn, NEON_SADALP); 3978 } 3979 3980 3981 void Assembler::uadalp(const VRegister& vd, 3982 const VRegister& vn) { 3983 NEONAddlp(vd, vn, NEON_UADALP); 3984 } 3985 3986 3987 void Assembler::NEONAcrossLanesL(const VRegister& vd, 3988 const VRegister& vn, 3989 NEONAcrossLanesOp op) { 3990 VIXL_ASSERT((vn.Is8B() && vd.Is1H()) || 3991 (vn.Is16B() && vd.Is1H()) || 3992 (vn.Is4H() && vd.Is1S()) || 3993 (vn.Is8H() && vd.Is1S()) || 3994 (vn.Is4S() && vd.Is1D())); 3995 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd)); 3996 } 3997 3998 3999 void Assembler::saddlv(const VRegister& vd, 4000 const VRegister& vn) { 4001 NEONAcrossLanesL(vd, vn, NEON_SADDLV); 4002 } 4003 4004 4005 void Assembler::uaddlv(const VRegister& vd, 4006 const VRegister& vn) { 4007 NEONAcrossLanesL(vd, vn, NEON_UADDLV); 4008 } 4009 4010 4011 void Assembler::NEONAcrossLanes(const VRegister& vd, 4012 const VRegister& vn, 4013 NEONAcrossLanesOp op) { 4014 VIXL_ASSERT((vn.Is8B() && vd.Is1B()) || 4015 (vn.Is16B() && vd.Is1B()) || 4016 (vn.Is4H() && vd.Is1H()) || 4017 (vn.Is8H() && vd.Is1H()) || 4018 (vn.Is4S() && vd.Is1S())); 4019 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) { 4020 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd)); 4021 } else { 4022 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd)); 4023 } 4024 } 4025 4026 4027 #define NEON_ACROSSLANES_LIST(V) \ 4028 V(fmaxv, NEON_FMAXV, vd.Is1S()) \ 4029 V(fminv, NEON_FMINV, vd.Is1S()) \ 4030 V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \ 4031 V(fminnmv, NEON_FMINNMV, vd.Is1S()) \ 4032 V(addv, NEON_ADDV, true) \ 4033 V(smaxv, NEON_SMAXV, true) \ 4034 V(sminv, NEON_SMINV, true) \ 4035 V(umaxv, NEON_UMAXV, true) \ 4036 V(uminv, NEON_UMINV, true) 4037 4038 4039 #define DEFINE_ASM_FUNC(FN, OP, AS) \ 4040 void Assembler::FN(const VRegister& vd, \ 4041 const VRegister& vn) { \ 4042 VIXL_ASSERT(AS); \ 4043 NEONAcrossLanes(vd, vn, OP); \ 4044 } 4045 NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC) 4046 #undef DEFINE_ASM_FUNC 4047 4048 4049 void Assembler::NEONPerm(const VRegister& vd, 4050 const VRegister& vn, 4051 const VRegister& vm, 4052 NEONPermOp op) { 4053 VIXL_ASSERT(AreSameFormat(vd, vn, vm)); 4054 VIXL_ASSERT(!vd.Is1D()); 4055 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd)); 4056 } 4057 4058 4059 void Assembler::trn1(const VRegister& vd, 4060 const VRegister& vn, 4061 const VRegister& vm) { 4062 NEONPerm(vd, vn, vm, NEON_TRN1); 4063 } 4064 4065 4066 void Assembler::trn2(const VRegister& vd, 4067 const VRegister& vn, 4068 const VRegister& vm) { 4069 NEONPerm(vd, vn, vm, NEON_TRN2); 4070 } 4071 4072 4073 void Assembler::uzp1(const VRegister& vd, 4074 const VRegister& vn, 4075 const VRegister& vm) { 4076 NEONPerm(vd, vn, vm, NEON_UZP1); 4077 } 4078 4079 4080 void Assembler::uzp2(const VRegister& vd, 4081 const VRegister& vn, 4082 const VRegister& vm) { 4083 NEONPerm(vd, vn, vm, NEON_UZP2); 4084 } 4085 4086 4087 void Assembler::zip1(const VRegister& vd, 4088 const VRegister& vn, 4089 const VRegister& vm) { 4090 NEONPerm(vd, vn, vm, NEON_ZIP1); 4091 } 4092 4093 4094 void Assembler::zip2(const VRegister& vd, 4095 const VRegister& vn, 4096 const VRegister& vm) { 4097 NEONPerm(vd, vn, vm, NEON_ZIP2); 4098 } 4099 4100 4101 void Assembler::NEONShiftImmediate(const VRegister& vd, 4102 const VRegister& vn, 4103 NEONShiftImmediateOp op, 4104 int immh_immb) { 4105 VIXL_ASSERT(AreSameFormat(vd, vn)); 4106 Instr q, scalar; 4107 if (vn.IsScalar()) { 4108 q = NEON_Q; 4109 scalar = NEONScalar; 4110 } else { 4111 q = vd.IsD() ? 0 : NEON_Q; 4112 scalar = 0; 4113 } 4114 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd)); 4115 } 4116 4117 4118 void Assembler::NEONShiftLeftImmediate(const VRegister& vd, 4119 const VRegister& vn, 4120 int shift, 4121 NEONShiftImmediateOp op) { 4122 int laneSizeInBits = vn.LaneSizeInBits(); 4123 VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits)); 4124 NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16); 4125 } 4126 4127 4128 void Assembler::NEONShiftRightImmediate(const VRegister& vd, 4129 const VRegister& vn, 4130 int shift, 4131 NEONShiftImmediateOp op) { 4132 int laneSizeInBits = vn.LaneSizeInBits(); 4133 VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits)); 4134 NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16); 4135 } 4136 4137 4138 void Assembler::NEONShiftImmediateL(const VRegister& vd, 4139 const VRegister& vn, 4140 int shift, 4141 NEONShiftImmediateOp op) { 4142 int laneSizeInBits = vn.LaneSizeInBits(); 4143 VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits)); 4144 int immh_immb = (laneSizeInBits + shift) << 16; 4145 4146 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || 4147 (vn.Is4H() && vd.Is4S()) || 4148 (vn.Is2S() && vd.Is2D()) || 4149 (vn.Is16B() && vd.Is8H())|| 4150 (vn.Is8H() && vd.Is4S()) || 4151 (vn.Is4S() && vd.Is2D())); 4152 Instr q; 4153 q = vn.IsD() ? 0 : NEON_Q; 4154 Emit(q | op | immh_immb | Rn(vn) | Rd(vd)); 4155 } 4156 4157 4158 void Assembler::NEONShiftImmediateN(const VRegister& vd, 4159 const VRegister& vn, 4160 int shift, 4161 NEONShiftImmediateOp op) { 4162 Instr q, scalar; 4163 int laneSizeInBits = vd.LaneSizeInBits(); 4164 VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits)); 4165 int immh_immb = (2 * laneSizeInBits - shift) << 16; 4166 4167 if (vn.IsScalar()) { 4168 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || 4169 (vd.Is1H() && vn.Is1S()) || 4170 (vd.Is1S() && vn.Is1D())); 4171 q = NEON_Q; 4172 scalar = NEONScalar; 4173 } else { 4174 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || 4175 (vd.Is4H() && vn.Is4S()) || 4176 (vd.Is2S() && vn.Is2D()) || 4177 (vd.Is16B() && vn.Is8H())|| 4178 (vd.Is8H() && vn.Is4S()) || 4179 (vd.Is4S() && vn.Is2D())); 4180 scalar = 0; 4181 q = vd.IsD() ? 0 : NEON_Q; 4182 } 4183 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd)); 4184 } 4185 4186 4187 void Assembler::shl(const VRegister& vd, 4188 const VRegister& vn, 4189 int shift) { 4190 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4191 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL); 4192 } 4193 4194 4195 void Assembler::sli(const VRegister& vd, 4196 const VRegister& vn, 4197 int shift) { 4198 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4199 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI); 4200 } 4201 4202 4203 void Assembler::sqshl(const VRegister& vd, 4204 const VRegister& vn, 4205 int shift) { 4206 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm); 4207 } 4208 4209 4210 void Assembler::sqshlu(const VRegister& vd, 4211 const VRegister& vn, 4212 int shift) { 4213 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU); 4214 } 4215 4216 4217 void Assembler::uqshl(const VRegister& vd, 4218 const VRegister& vn, 4219 int shift) { 4220 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm); 4221 } 4222 4223 4224 void Assembler::sshll(const VRegister& vd, 4225 const VRegister& vn, 4226 int shift) { 4227 VIXL_ASSERT(vn.IsD()); 4228 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL); 4229 } 4230 4231 4232 void Assembler::sshll2(const VRegister& vd, 4233 const VRegister& vn, 4234 int shift) { 4235 VIXL_ASSERT(vn.IsQ()); 4236 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL); 4237 } 4238 4239 4240 void Assembler::sxtl(const VRegister& vd, 4241 const VRegister& vn) { 4242 sshll(vd, vn, 0); 4243 } 4244 4245 4246 void Assembler::sxtl2(const VRegister& vd, 4247 const VRegister& vn) { 4248 sshll2(vd, vn, 0); 4249 } 4250 4251 4252 void Assembler::ushll(const VRegister& vd, 4253 const VRegister& vn, 4254 int shift) { 4255 VIXL_ASSERT(vn.IsD()); 4256 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL); 4257 } 4258 4259 4260 void Assembler::ushll2(const VRegister& vd, 4261 const VRegister& vn, 4262 int shift) { 4263 VIXL_ASSERT(vn.IsQ()); 4264 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL); 4265 } 4266 4267 4268 void Assembler::uxtl(const VRegister& vd, 4269 const VRegister& vn) { 4270 ushll(vd, vn, 0); 4271 } 4272 4273 4274 void Assembler::uxtl2(const VRegister& vd, 4275 const VRegister& vn) { 4276 ushll2(vd, vn, 0); 4277 } 4278 4279 4280 void Assembler::sri(const VRegister& vd, 4281 const VRegister& vn, 4282 int shift) { 4283 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4284 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI); 4285 } 4286 4287 4288 void Assembler::sshr(const VRegister& vd, 4289 const VRegister& vn, 4290 int shift) { 4291 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4292 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR); 4293 } 4294 4295 4296 void Assembler::ushr(const VRegister& vd, 4297 const VRegister& vn, 4298 int shift) { 4299 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4300 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR); 4301 } 4302 4303 4304 void Assembler::srshr(const VRegister& vd, 4305 const VRegister& vn, 4306 int shift) { 4307 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4308 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR); 4309 } 4310 4311 4312 void Assembler::urshr(const VRegister& vd, 4313 const VRegister& vn, 4314 int shift) { 4315 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4316 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR); 4317 } 4318 4319 4320 void Assembler::ssra(const VRegister& vd, 4321 const VRegister& vn, 4322 int shift) { 4323 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4324 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA); 4325 } 4326 4327 4328 void Assembler::usra(const VRegister& vd, 4329 const VRegister& vn, 4330 int shift) { 4331 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4332 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA); 4333 } 4334 4335 4336 void Assembler::srsra(const VRegister& vd, 4337 const VRegister& vn, 4338 int shift) { 4339 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4340 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA); 4341 } 4342 4343 4344 void Assembler::ursra(const VRegister& vd, 4345 const VRegister& vn, 4346 int shift) { 4347 VIXL_ASSERT(vd.IsVector() || vd.Is1D()); 4348 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA); 4349 } 4350 4351 4352 void Assembler::shrn(const VRegister& vd, 4353 const VRegister& vn, 4354 int shift) { 4355 VIXL_ASSERT(vn.IsVector() && vd.IsD()); 4356 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN); 4357 } 4358 4359 4360 void Assembler::shrn2(const VRegister& vd, 4361 const VRegister& vn, 4362 int shift) { 4363 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4364 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN); 4365 } 4366 4367 4368 void Assembler::rshrn(const VRegister& vd, 4369 const VRegister& vn, 4370 int shift) { 4371 VIXL_ASSERT(vn.IsVector() && vd.IsD()); 4372 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN); 4373 } 4374 4375 4376 void Assembler::rshrn2(const VRegister& vd, 4377 const VRegister& vn, 4378 int shift) { 4379 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4380 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN); 4381 } 4382 4383 4384 void Assembler::sqshrn(const VRegister& vd, 4385 const VRegister& vn, 4386 int shift) { 4387 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 4388 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN); 4389 } 4390 4391 4392 void Assembler::sqshrn2(const VRegister& vd, 4393 const VRegister& vn, 4394 int shift) { 4395 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4396 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN); 4397 } 4398 4399 4400 void Assembler::sqrshrn(const VRegister& vd, 4401 const VRegister& vn, 4402 int shift) { 4403 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 4404 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN); 4405 } 4406 4407 4408 void Assembler::sqrshrn2(const VRegister& vd, 4409 const VRegister& vn, 4410 int shift) { 4411 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4412 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN); 4413 } 4414 4415 4416 void Assembler::sqshrun(const VRegister& vd, 4417 const VRegister& vn, 4418 int shift) { 4419 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 4420 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN); 4421 } 4422 4423 4424 void Assembler::sqshrun2(const VRegister& vd, 4425 const VRegister& vn, 4426 int shift) { 4427 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4428 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN); 4429 } 4430 4431 4432 void Assembler::sqrshrun(const VRegister& vd, 4433 const VRegister& vn, 4434 int shift) { 4435 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 4436 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN); 4437 } 4438 4439 4440 void Assembler::sqrshrun2(const VRegister& vd, 4441 const VRegister& vn, 4442 int shift) { 4443 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4444 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN); 4445 } 4446 4447 4448 void Assembler::uqshrn(const VRegister& vd, 4449 const VRegister& vn, 4450 int shift) { 4451 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 4452 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN); 4453 } 4454 4455 4456 void Assembler::uqshrn2(const VRegister& vd, 4457 const VRegister& vn, 4458 int shift) { 4459 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4460 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN); 4461 } 4462 4463 4464 void Assembler::uqrshrn(const VRegister& vd, 4465 const VRegister& vn, 4466 int shift) { 4467 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar())); 4468 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN); 4469 } 4470 4471 4472 void Assembler::uqrshrn2(const VRegister& vd, 4473 const VRegister& vn, 4474 int shift) { 4475 VIXL_ASSERT(vn.IsVector() && vd.IsQ()); 4476 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN); 4477 } 4478 4479 4480 // Note: 4481 // Below, a difference in case for the same letter indicates a 4482 // negated bit. 4483 // If b is 1, then B is 0. 4484 uint32_t Assembler::FP32ToImm8(float imm) { 4485 VIXL_ASSERT(IsImmFP32(imm)); 4486 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000 4487 uint32_t bits = float_to_rawbits(imm); 4488 // bit7: a000.0000 4489 uint32_t bit7 = ((bits >> 31) & 0x1) << 7; 4490 // bit6: 0b00.0000 4491 uint32_t bit6 = ((bits >> 29) & 0x1) << 6; 4492 // bit5_to_0: 00cd.efgh 4493 uint32_t bit5_to_0 = (bits >> 19) & 0x3f; 4494 4495 return bit7 | bit6 | bit5_to_0; 4496 } 4497 4498 4499 Instr Assembler::ImmFP32(float imm) { 4500 return FP32ToImm8(imm) << ImmFP_offset; 4501 } 4502 4503 4504 uint32_t Assembler::FP64ToImm8(double imm) { 4505 VIXL_ASSERT(IsImmFP64(imm)); 4506 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 4507 // 0000.0000.0000.0000.0000.0000.0000.0000 4508 uint64_t bits = double_to_rawbits(imm); 4509 // bit7: a000.0000 4510 uint32_t bit7 = ((bits >> 63) & 0x1) << 7; 4511 // bit6: 0b00.0000 4512 uint32_t bit6 = ((bits >> 61) & 0x1) << 6; 4513 // bit5_to_0: 00cd.efgh 4514 uint32_t bit5_to_0 = (bits >> 48) & 0x3f; 4515 4516 return bit7 | bit6 | bit5_to_0; 4517 } 4518 4519 4520 Instr Assembler::ImmFP64(double imm) { 4521 return FP64ToImm8(imm) << ImmFP_offset; 4522 } 4523 4524 4525 // Code generation helpers. 4526 void Assembler::MoveWide(const Register& rd, 4527 uint64_t imm, 4528 int shift, 4529 MoveWideImmediateOp mov_op) { 4530 // Ignore the top 32 bits of an immediate if we're moving to a W register. 4531 if (rd.Is32Bits()) { 4532 // Check that the top 32 bits are zero (a positive 32-bit number) or top 4533 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits). 4534 VIXL_ASSERT(((imm >> kWRegSize) == 0) || 4535 ((imm >> (kWRegSize - 1)) == 0x1ffffffff)); 4536 imm &= kWRegMask; 4537 } 4538 4539 if (shift >= 0) { 4540 // Explicit shift specified. 4541 VIXL_ASSERT((shift == 0) || (shift == 16) || 4542 (shift == 32) || (shift == 48)); 4543 VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16)); 4544 shift /= 16; 4545 } else { 4546 // Calculate a new immediate and shift combination to encode the immediate 4547 // argument. 4548 shift = 0; 4549 if ((imm & 0xffffffffffff0000) == 0) { 4550 // Nothing to do. 4551 } else if ((imm & 0xffffffff0000ffff) == 0) { 4552 imm >>= 16; 4553 shift = 1; 4554 } else if ((imm & 0xffff0000ffffffff) == 0) { 4555 VIXL_ASSERT(rd.Is64Bits()); 4556 imm >>= 32; 4557 shift = 2; 4558 } else if ((imm & 0x0000ffffffffffff) == 0) { 4559 VIXL_ASSERT(rd.Is64Bits()); 4560 imm >>= 48; 4561 shift = 3; 4562 } 4563 } 4564 4565 VIXL_ASSERT(is_uint16(imm)); 4566 4567 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | 4568 Rd(rd) | ImmMoveWide(imm) | ShiftMoveWide(shift)); 4569 } 4570 4571 4572 void Assembler::AddSub(const Register& rd, 4573 const Register& rn, 4574 const Operand& operand, 4575 FlagsUpdate S, 4576 AddSubOp op) { 4577 VIXL_ASSERT(rd.size() == rn.size()); 4578 if (operand.IsImmediate()) { 4579 int64_t immediate = operand.immediate(); 4580 VIXL_ASSERT(IsImmAddSub(immediate)); 4581 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 4582 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) | 4583 ImmAddSub(immediate) | dest_reg | RnSP(rn)); 4584 } else if (operand.IsShiftedRegister()) { 4585 VIXL_ASSERT(operand.reg().size() == rd.size()); 4586 VIXL_ASSERT(operand.shift() != ROR); 4587 4588 // For instructions of the form: 4589 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ] 4590 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ] 4591 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ] 4592 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ] 4593 // or their 64-bit register equivalents, convert the operand from shifted to 4594 // extended register mode, and emit an add/sub extended instruction. 4595 if (rn.IsSP() || rd.IsSP()) { 4596 VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags))); 4597 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S, 4598 AddSubExtendedFixed | op); 4599 } else { 4600 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op); 4601 } 4602 } else { 4603 VIXL_ASSERT(operand.IsExtendedRegister()); 4604 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op); 4605 } 4606 } 4607 4608 4609 void Assembler::AddSubWithCarry(const Register& rd, 4610 const Register& rn, 4611 const Operand& operand, 4612 FlagsUpdate S, 4613 AddSubWithCarryOp op) { 4614 VIXL_ASSERT(rd.size() == rn.size()); 4615 VIXL_ASSERT(rd.size() == operand.reg().size()); 4616 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 4617 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd)); 4618 } 4619 4620 4621 void Assembler::hlt(int code) { 4622 VIXL_ASSERT(is_uint16(code)); 4623 Emit(HLT | ImmException(code)); 4624 } 4625 4626 4627 void Assembler::brk(int code) { 4628 VIXL_ASSERT(is_uint16(code)); 4629 Emit(BRK | ImmException(code)); 4630 } 4631 4632 4633 void Assembler::svc(int code) { 4634 Emit(SVC | ImmException(code)); 4635 } 4636 4637 4638 // TODO(all): The third parameter should be passed by reference but gcc 4.8.2 4639 // reports a bogus uninitialised warning then. 4640 void Assembler::Logical(const Register& rd, 4641 const Register& rn, 4642 const Operand operand, 4643 LogicalOp op) { 4644 VIXL_ASSERT(rd.size() == rn.size()); 4645 if (operand.IsImmediate()) { 4646 int64_t immediate = operand.immediate(); 4647 unsigned reg_size = rd.size(); 4648 4649 VIXL_ASSERT(immediate != 0); 4650 VIXL_ASSERT(immediate != -1); 4651 VIXL_ASSERT(rd.Is64Bits() || is_uint32(immediate)); 4652 4653 // If the operation is NOT, invert the operation and immediate. 4654 if ((op & NOT) == NOT) { 4655 op = static_cast<LogicalOp>(op & ~NOT); 4656 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask); 4657 } 4658 4659 unsigned n, imm_s, imm_r; 4660 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 4661 // Immediate can be encoded in the instruction. 4662 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 4663 } else { 4664 // This case is handled in the macro assembler. 4665 VIXL_UNREACHABLE(); 4666 } 4667 } else { 4668 VIXL_ASSERT(operand.IsShiftedRegister()); 4669 VIXL_ASSERT(operand.reg().size() == rd.size()); 4670 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed); 4671 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op); 4672 } 4673 } 4674 4675 4676 void Assembler::LogicalImmediate(const Register& rd, 4677 const Register& rn, 4678 unsigned n, 4679 unsigned imm_s, 4680 unsigned imm_r, 4681 LogicalOp op) { 4682 unsigned reg_size = rd.size(); 4683 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd); 4684 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) | 4685 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg | 4686 Rn(rn)); 4687 } 4688 4689 4690 void Assembler::ConditionalCompare(const Register& rn, 4691 const Operand& operand, 4692 StatusFlags nzcv, 4693 Condition cond, 4694 ConditionalCompareOp op) { 4695 Instr ccmpop; 4696 if (operand.IsImmediate()) { 4697 int64_t immediate = operand.immediate(); 4698 VIXL_ASSERT(IsImmConditionalCompare(immediate)); 4699 ccmpop = ConditionalCompareImmediateFixed | op | ImmCondCmp(immediate); 4700 } else { 4701 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0)); 4702 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg()); 4703 } 4704 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv)); 4705 } 4706 4707 4708 void Assembler::DataProcessing1Source(const Register& rd, 4709 const Register& rn, 4710 DataProcessing1SourceOp op) { 4711 VIXL_ASSERT(rd.size() == rn.size()); 4712 Emit(SF(rn) | op | Rn(rn) | Rd(rd)); 4713 } 4714 4715 4716 void Assembler::FPDataProcessing1Source(const VRegister& vd, 4717 const VRegister& vn, 4718 FPDataProcessing1SourceOp op) { 4719 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D()); 4720 Emit(FPType(vn) | op | Rn(vn) | Rd(vd)); 4721 } 4722 4723 4724 void Assembler::FPDataProcessing3Source(const VRegister& vd, 4725 const VRegister& vn, 4726 const VRegister& vm, 4727 const VRegister& va, 4728 FPDataProcessing3SourceOp op) { 4729 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); 4730 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va)); 4731 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va)); 4732 } 4733 4734 4735 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd, 4736 const int imm8, 4737 const int left_shift, 4738 NEONModifiedImmediateOp op) { 4739 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || 4740 vd.Is2S() || vd.Is4S()); 4741 VIXL_ASSERT((left_shift == 0) || (left_shift == 8) || 4742 (left_shift == 16) || (left_shift == 24)); 4743 VIXL_ASSERT(is_uint8(imm8)); 4744 4745 int cmode_1, cmode_2, cmode_3; 4746 if (vd.Is8B() || vd.Is16B()) { 4747 VIXL_ASSERT(op == NEONModifiedImmediate_MOVI); 4748 cmode_1 = 1; 4749 cmode_2 = 1; 4750 cmode_3 = 1; 4751 } else { 4752 cmode_1 = (left_shift >> 3) & 1; 4753 cmode_2 = left_shift >> 4; 4754 cmode_3 = 0; 4755 if (vd.Is4H() || vd.Is8H()) { 4756 VIXL_ASSERT((left_shift == 0) || (left_shift == 8)); 4757 cmode_3 = 1; 4758 } 4759 } 4760 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1); 4761 4762 int q = vd.IsQ() ? NEON_Q : 0; 4763 4764 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd)); 4765 } 4766 4767 4768 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd, 4769 const int imm8, 4770 const int shift_amount, 4771 NEONModifiedImmediateOp op) { 4772 VIXL_ASSERT(vd.Is2S() || vd.Is4S()); 4773 VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16)); 4774 VIXL_ASSERT(is_uint8(imm8)); 4775 4776 int cmode_0 = (shift_amount >> 4) & 1; 4777 int cmode = 0xc | cmode_0; 4778 4779 int q = vd.IsQ() ? NEON_Q : 0; 4780 4781 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd)); 4782 } 4783 4784 4785 void Assembler::EmitShift(const Register& rd, 4786 const Register& rn, 4787 Shift shift, 4788 unsigned shift_amount) { 4789 switch (shift) { 4790 case LSL: 4791 lsl(rd, rn, shift_amount); 4792 break; 4793 case LSR: 4794 lsr(rd, rn, shift_amount); 4795 break; 4796 case ASR: 4797 asr(rd, rn, shift_amount); 4798 break; 4799 case ROR: 4800 ror(rd, rn, shift_amount); 4801 break; 4802 default: 4803 VIXL_UNREACHABLE(); 4804 } 4805 } 4806 4807 4808 void Assembler::EmitExtendShift(const Register& rd, 4809 const Register& rn, 4810 Extend extend, 4811 unsigned left_shift) { 4812 VIXL_ASSERT(rd.size() >= rn.size()); 4813 unsigned reg_size = rd.size(); 4814 // Use the correct size of register. 4815 Register rn_ = Register(rn.code(), rd.size()); 4816 // Bits extracted are high_bit:0. 4817 unsigned high_bit = (8 << (extend & 0x3)) - 1; 4818 // Number of bits left in the result that are not introduced by the shift. 4819 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1); 4820 4821 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) { 4822 switch (extend) { 4823 case UXTB: 4824 case UXTH: 4825 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break; 4826 case SXTB: 4827 case SXTH: 4828 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break; 4829 case UXTX: 4830 case SXTX: { 4831 VIXL_ASSERT(rn.size() == kXRegSize); 4832 // Nothing to extend. Just shift. 4833 lsl(rd, rn_, left_shift); 4834 break; 4835 } 4836 default: VIXL_UNREACHABLE(); 4837 } 4838 } else { 4839 // No need to extend as the extended bits would be shifted away. 4840 lsl(rd, rn_, left_shift); 4841 } 4842 } 4843 4844 4845 void Assembler::DataProcShiftedRegister(const Register& rd, 4846 const Register& rn, 4847 const Operand& operand, 4848 FlagsUpdate S, 4849 Instr op) { 4850 VIXL_ASSERT(operand.IsShiftedRegister()); 4851 VIXL_ASSERT(rn.Is64Bits() || (rn.Is32Bits() && 4852 is_uint5(operand.shift_amount()))); 4853 Emit(SF(rd) | op | Flags(S) | 4854 ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) | 4855 Rm(operand.reg()) | Rn(rn) | Rd(rd)); 4856 } 4857 4858 4859 void Assembler::DataProcExtendedRegister(const Register& rd, 4860 const Register& rn, 4861 const Operand& operand, 4862 FlagsUpdate S, 4863 Instr op) { 4864 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd); 4865 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | 4866 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) | 4867 dest_reg | RnSP(rn)); 4868 } 4869 4870 4871 Instr Assembler::LoadStoreMemOperand(const MemOperand& addr, 4872 unsigned access_size, 4873 LoadStoreScalingOption option) { 4874 Instr base = RnSP(addr.base()); 4875 int64_t offset = addr.offset(); 4876 4877 if (addr.IsImmediateOffset()) { 4878 bool prefer_unscaled = (option == PreferUnscaledOffset) || 4879 (option == RequireUnscaledOffset); 4880 if (prefer_unscaled && IsImmLSUnscaled(offset)) { 4881 // Use the unscaled addressing mode. 4882 return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset); 4883 } 4884 4885 if ((option != RequireUnscaledOffset) && 4886 IsImmLSScaled(offset, access_size)) { 4887 // Use the scaled addressing mode. 4888 return base | LoadStoreUnsignedOffsetFixed | 4889 ImmLSUnsigned(offset >> access_size); 4890 } 4891 4892 if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) { 4893 // Use the unscaled addressing mode. 4894 return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset); 4895 } 4896 } 4897 4898 // All remaining addressing modes are register-offset, pre-indexed or 4899 // post-indexed modes. 4900 VIXL_ASSERT((option != RequireUnscaledOffset) && 4901 (option != RequireScaledOffset)); 4902 4903 if (addr.IsRegisterOffset()) { 4904 Extend ext = addr.extend(); 4905 Shift shift = addr.shift(); 4906 unsigned shift_amount = addr.shift_amount(); 4907 4908 // LSL is encoded in the option field as UXTX. 4909 if (shift == LSL) { 4910 ext = UXTX; 4911 } 4912 4913 // Shifts are encoded in one bit, indicating a left shift by the memory 4914 // access size. 4915 VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size)); 4916 return base | LoadStoreRegisterOffsetFixed | Rm(addr.regoffset()) | 4917 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0); 4918 } 4919 4920 if (addr.IsPreIndex() && IsImmLSUnscaled(offset)) { 4921 return base | LoadStorePreIndexFixed | ImmLS(offset); 4922 } 4923 4924 if (addr.IsPostIndex() && IsImmLSUnscaled(offset)) { 4925 return base | LoadStorePostIndexFixed | ImmLS(offset); 4926 } 4927 4928 // If this point is reached, the MemOperand (addr) cannot be encoded. 4929 VIXL_UNREACHABLE(); 4930 return 0; 4931 } 4932 4933 4934 void Assembler::LoadStore(const CPURegister& rt, 4935 const MemOperand& addr, 4936 LoadStoreOp op, 4937 LoadStoreScalingOption option) { 4938 Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option)); 4939 } 4940 4941 4942 void Assembler::Prefetch(PrefetchOperation op, 4943 const MemOperand& addr, 4944 LoadStoreScalingOption option) { 4945 VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset()); 4946 4947 Instr prfop = ImmPrefetchOperation(op); 4948 Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option)); 4949 } 4950 4951 4952 bool Assembler::IsImmAddSub(int64_t immediate) { 4953 return is_uint12(immediate) || 4954 (is_uint12(immediate >> 12) && ((immediate & 0xfff) == 0)); 4955 } 4956 4957 4958 bool Assembler::IsImmConditionalCompare(int64_t immediate) { 4959 return is_uint5(immediate); 4960 } 4961 4962 4963 bool Assembler::IsImmFP32(float imm) { 4964 // Valid values will have the form: 4965 // aBbb.bbbc.defg.h000.0000.0000.0000.0000 4966 uint32_t bits = float_to_rawbits(imm); 4967 // bits[19..0] are cleared. 4968 if ((bits & 0x7ffff) != 0) { 4969 return false; 4970 } 4971 4972 // bits[29..25] are all set or all cleared. 4973 uint32_t b_pattern = (bits >> 16) & 0x3e00; 4974 if (b_pattern != 0 && b_pattern != 0x3e00) { 4975 return false; 4976 } 4977 4978 // bit[30] and bit[29] are opposite. 4979 if (((bits ^ (bits << 1)) & 0x40000000) == 0) { 4980 return false; 4981 } 4982 4983 return true; 4984 } 4985 4986 4987 bool Assembler::IsImmFP64(double imm) { 4988 // Valid values will have the form: 4989 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 4990 // 0000.0000.0000.0000.0000.0000.0000.0000 4991 uint64_t bits = double_to_rawbits(imm); 4992 // bits[47..0] are cleared. 4993 if ((bits & 0x0000ffffffffffff) != 0) { 4994 return false; 4995 } 4996 4997 // bits[61..54] are all set or all cleared. 4998 uint32_t b_pattern = (bits >> 48) & 0x3fc0; 4999 if ((b_pattern != 0) && (b_pattern != 0x3fc0)) { 5000 return false; 5001 } 5002 5003 // bit[62] and bit[61] are opposite. 5004 if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) { 5005 return false; 5006 } 5007 5008 return true; 5009 } 5010 5011 5012 bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size) { 5013 VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2); 5014 bool offset_is_size_multiple = 5015 (((offset >> access_size) << access_size) == offset); 5016 return offset_is_size_multiple && is_int7(offset >> access_size); 5017 } 5018 5019 5020 bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size) { 5021 VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2); 5022 bool offset_is_size_multiple = 5023 (((offset >> access_size) << access_size) == offset); 5024 return offset_is_size_multiple && is_uint12(offset >> access_size); 5025 } 5026 5027 5028 bool Assembler::IsImmLSUnscaled(int64_t offset) { 5029 return is_int9(offset); 5030 } 5031 5032 5033 // The movn instruction can generate immediates containing an arbitrary 16-bit 5034 // value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff. 5035 bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) { 5036 return IsImmMovz(~imm, reg_size); 5037 } 5038 5039 5040 // The movz instruction can generate immediates containing an arbitrary 16-bit 5041 // value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000. 5042 bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) { 5043 VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize)); 5044 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1); 5045 } 5046 5047 5048 // Test if a given value can be encoded in the immediate field of a logical 5049 // instruction. 5050 // If it can be encoded, the function returns true, and values pointed to by n, 5051 // imm_s and imm_r are updated with immediates encoded in the format required 5052 // by the corresponding fields in the logical instruction. 5053 // If it can not be encoded, the function returns false, and the values pointed 5054 // to by n, imm_s and imm_r are undefined. 5055 bool Assembler::IsImmLogical(uint64_t value, 5056 unsigned width, 5057 unsigned* n, 5058 unsigned* imm_s, 5059 unsigned* imm_r) { 5060 VIXL_ASSERT((width == kWRegSize) || (width == kXRegSize)); 5061 5062 bool negate = false; 5063 5064 // Logical immediates are encoded using parameters n, imm_s and imm_r using 5065 // the following table: 5066 // 5067 // N imms immr size S R 5068 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr) 5069 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr) 5070 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr) 5071 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr) 5072 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr) 5073 // 0 11110s xxxxxr 2 UInt(s) UInt(r) 5074 // (s bits must not be all set) 5075 // 5076 // A pattern is constructed of size bits, where the least significant S+1 bits 5077 // are set. The pattern is rotated right by R, and repeated across a 32 or 5078 // 64-bit value, depending on destination register width. 5079 // 5080 // Put another way: the basic format of a logical immediate is a single 5081 // contiguous stretch of 1 bits, repeated across the whole word at intervals 5082 // given by a power of 2. To identify them quickly, we first locate the 5083 // lowest stretch of 1 bits, then the next 1 bit above that; that combination 5084 // is different for every logical immediate, so it gives us all the 5085 // information we need to identify the only logical immediate that our input 5086 // could be, and then we simply check if that's the value we actually have. 5087 // 5088 // (The rotation parameter does give the possibility of the stretch of 1 bits 5089 // going 'round the end' of the word. To deal with that, we observe that in 5090 // any situation where that happens the bitwise NOT of the value is also a 5091 // valid logical immediate. So we simply invert the input whenever its low bit 5092 // is set, and then we know that the rotated case can't arise.) 5093 5094 if (value & 1) { 5095 // If the low bit is 1, negate the value, and set a flag to remember that we 5096 // did (so that we can adjust the return values appropriately). 5097 negate = true; 5098 value = ~value; 5099 } 5100 5101 if (width == kWRegSize) { 5102 // To handle 32-bit logical immediates, the very easiest thing is to repeat 5103 // the input value twice to make a 64-bit word. The correct encoding of that 5104 // as a logical immediate will also be the correct encoding of the 32-bit 5105 // value. 5106 5107 // Avoid making the assumption that the most-significant 32 bits are zero by 5108 // shifting the value left and duplicating it. 5109 value <<= kWRegSize; 5110 value |= value >> kWRegSize; 5111 } 5112 5113 // The basic analysis idea: imagine our input word looks like this. 5114 // 5115 // 0011111000111110001111100011111000111110001111100011111000111110 5116 // c b a 5117 // |<--d-->| 5118 // 5119 // We find the lowest set bit (as an actual power-of-2 value, not its index) 5120 // and call it a. Then we add a to our original number, which wipes out the 5121 // bottommost stretch of set bits and replaces it with a 1 carried into the 5122 // next zero bit. Then we look for the new lowest set bit, which is in 5123 // position b, and subtract it, so now our number is just like the original 5124 // but with the lowest stretch of set bits completely gone. Now we find the 5125 // lowest set bit again, which is position c in the diagram above. Then we'll 5126 // measure the distance d between bit positions a and c (using CLZ), and that 5127 // tells us that the only valid logical immediate that could possibly be equal 5128 // to this number is the one in which a stretch of bits running from a to just 5129 // below b is replicated every d bits. 5130 uint64_t a = LowestSetBit(value); 5131 uint64_t value_plus_a = value + a; 5132 uint64_t b = LowestSetBit(value_plus_a); 5133 uint64_t value_plus_a_minus_b = value_plus_a - b; 5134 uint64_t c = LowestSetBit(value_plus_a_minus_b); 5135 5136 int d, clz_a, out_n; 5137 uint64_t mask; 5138 5139 if (c != 0) { 5140 // The general case, in which there is more than one stretch of set bits. 5141 // Compute the repeat distance d, and set up a bitmask covering the basic 5142 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all 5143 // of these cases the N bit of the output will be zero. 5144 clz_a = CountLeadingZeros(a, kXRegSize); 5145 int clz_c = CountLeadingZeros(c, kXRegSize); 5146 d = clz_a - clz_c; 5147 mask = ((UINT64_C(1) << d) - 1); 5148 out_n = 0; 5149 } else { 5150 // Handle degenerate cases. 5151 // 5152 // If any of those 'find lowest set bit' operations didn't find a set bit at 5153 // all, then the word will have been zero thereafter, so in particular the 5154 // last lowest_set_bit operation will have returned zero. So we can test for 5155 // all the special case conditions in one go by seeing if c is zero. 5156 if (a == 0) { 5157 // The input was zero (or all 1 bits, which will come to here too after we 5158 // inverted it at the start of the function), for which we just return 5159 // false. 5160 return false; 5161 } else { 5162 // Otherwise, if c was zero but a was not, then there's just one stretch 5163 // of set bits in our word, meaning that we have the trivial case of 5164 // d == 64 and only one 'repetition'. Set up all the same variables as in 5165 // the general case above, and set the N bit in the output. 5166 clz_a = CountLeadingZeros(a, kXRegSize); 5167 d = 64; 5168 mask = ~UINT64_C(0); 5169 out_n = 1; 5170 } 5171 } 5172 5173 // If the repeat period d is not a power of two, it can't be encoded. 5174 if (!IsPowerOf2(d)) { 5175 return false; 5176 } 5177 5178 if (((b - a) & ~mask) != 0) { 5179 // If the bit stretch (b - a) does not fit within the mask derived from the 5180 // repeat period, then fail. 5181 return false; 5182 } 5183 5184 // The only possible option is b - a repeated every d bits. Now we're going to 5185 // actually construct the valid logical immediate derived from that 5186 // specification, and see if it equals our original input. 5187 // 5188 // To repeat a value every d bits, we multiply it by a number of the form 5189 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can 5190 // be derived using a table lookup on CLZ(d). 5191 static const uint64_t multipliers[] = { 5192 0x0000000000000001UL, 5193 0x0000000100000001UL, 5194 0x0001000100010001UL, 5195 0x0101010101010101UL, 5196 0x1111111111111111UL, 5197 0x5555555555555555UL, 5198 }; 5199 uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57]; 5200 uint64_t candidate = (b - a) * multiplier; 5201 5202 if (value != candidate) { 5203 // The candidate pattern doesn't match our input value, so fail. 5204 return false; 5205 } 5206 5207 // We have a match! This is a valid logical immediate, so now we have to 5208 // construct the bits and pieces of the instruction encoding that generates 5209 // it. 5210 5211 // Count the set bits in our basic stretch. The special case of clz(0) == -1 5212 // makes the answer come out right for stretches that reach the very top of 5213 // the word (e.g. numbers like 0xffffc00000000000). 5214 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize); 5215 int s = clz_a - clz_b; 5216 5217 // Decide how many bits to rotate right by, to put the low bit of that basic 5218 // stretch in position a. 5219 int r; 5220 if (negate) { 5221 // If we inverted the input right at the start of this function, here's 5222 // where we compensate: the number of set bits becomes the number of clear 5223 // bits, and the rotation count is based on position b rather than position 5224 // a (since b is the location of the 'lowest' 1 bit after inversion). 5225 s = d - s; 5226 r = (clz_b + 1) & (d - 1); 5227 } else { 5228 r = (clz_a + 1) & (d - 1); 5229 } 5230 5231 // Now we're done, except for having to encode the S output in such a way that 5232 // it gives both the number of set bits and the length of the repeated 5233 // segment. The s field is encoded like this: 5234 // 5235 // imms size S 5236 // ssssss 64 UInt(ssssss) 5237 // 0sssss 32 UInt(sssss) 5238 // 10ssss 16 UInt(ssss) 5239 // 110sss 8 UInt(sss) 5240 // 1110ss 4 UInt(ss) 5241 // 11110s 2 UInt(s) 5242 // 5243 // So we 'or' (-d << 1) with our computed s to form imms. 5244 if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) { 5245 *n = out_n; 5246 *imm_s = ((-d << 1) | (s - 1)) & 0x3f; 5247 *imm_r = r; 5248 } 5249 5250 return true; 5251 } 5252 5253 5254 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) { 5255 VIXL_ASSERT(rt.IsValid()); 5256 if (rt.IsRegister()) { 5257 return rt.Is64Bits() ? LDR_x : LDR_w; 5258 } else { 5259 VIXL_ASSERT(rt.IsVRegister()); 5260 switch (rt.SizeInBits()) { 5261 case kBRegSize: return LDR_b; 5262 case kHRegSize: return LDR_h; 5263 case kSRegSize: return LDR_s; 5264 case kDRegSize: return LDR_d; 5265 default: 5266 VIXL_ASSERT(rt.IsQ()); 5267 return LDR_q; 5268 } 5269 } 5270 } 5271 5272 5273 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) { 5274 VIXL_ASSERT(rt.IsValid()); 5275 if (rt.IsRegister()) { 5276 return rt.Is64Bits() ? STR_x : STR_w; 5277 } else { 5278 VIXL_ASSERT(rt.IsVRegister()); 5279 switch (rt.SizeInBits()) { 5280 case kBRegSize: return STR_b; 5281 case kHRegSize: return STR_h; 5282 case kSRegSize: return STR_s; 5283 case kDRegSize: return STR_d; 5284 default: 5285 VIXL_ASSERT(rt.IsQ()); 5286 return STR_q; 5287 } 5288 } 5289 } 5290 5291 5292 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt, 5293 const CPURegister& rt2) { 5294 VIXL_ASSERT(AreSameSizeAndType(rt, rt2)); 5295 USE(rt2); 5296 if (rt.IsRegister()) { 5297 return rt.Is64Bits() ? STP_x : STP_w; 5298 } else { 5299 VIXL_ASSERT(rt.IsVRegister()); 5300 switch (rt.SizeInBytes()) { 5301 case kSRegSizeInBytes: return STP_s; 5302 case kDRegSizeInBytes: return STP_d; 5303 default: 5304 VIXL_ASSERT(rt.IsQ()); 5305 return STP_q; 5306 } 5307 } 5308 } 5309 5310 5311 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt, 5312 const CPURegister& rt2) { 5313 VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w); 5314 return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) | 5315 LoadStorePairLBit); 5316 } 5317 5318 5319 LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor( 5320 const CPURegister& rt, const CPURegister& rt2) { 5321 VIXL_ASSERT(AreSameSizeAndType(rt, rt2)); 5322 USE(rt2); 5323 if (rt.IsRegister()) { 5324 return rt.Is64Bits() ? STNP_x : STNP_w; 5325 } else { 5326 VIXL_ASSERT(rt.IsVRegister()); 5327 switch (rt.SizeInBytes()) { 5328 case kSRegSizeInBytes: return STNP_s; 5329 case kDRegSizeInBytes: return STNP_d; 5330 default: 5331 VIXL_ASSERT(rt.IsQ()); 5332 return STNP_q; 5333 } 5334 } 5335 } 5336 5337 5338 LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor( 5339 const CPURegister& rt, const CPURegister& rt2) { 5340 VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w); 5341 return static_cast<LoadStorePairNonTemporalOp>( 5342 StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit); 5343 } 5344 5345 5346 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) { 5347 if (rt.IsRegister()) { 5348 return rt.IsX() ? LDR_x_lit : LDR_w_lit; 5349 } else { 5350 VIXL_ASSERT(rt.IsVRegister()); 5351 switch (rt.SizeInBytes()) { 5352 case kSRegSizeInBytes: return LDR_s_lit; 5353 case kDRegSizeInBytes: return LDR_d_lit; 5354 default: 5355 VIXL_ASSERT(rt.IsQ()); 5356 return LDR_q_lit; 5357 } 5358 } 5359 } 5360 5361 5362 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2, 5363 const CPURegister& reg3, const CPURegister& reg4, 5364 const CPURegister& reg5, const CPURegister& reg6, 5365 const CPURegister& reg7, const CPURegister& reg8) { 5366 int number_of_valid_regs = 0; 5367 int number_of_valid_fpregs = 0; 5368 5369 RegList unique_regs = 0; 5370 RegList unique_fpregs = 0; 5371 5372 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8}; 5373 5374 for (unsigned i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) { 5375 if (regs[i].IsRegister()) { 5376 number_of_valid_regs++; 5377 unique_regs |= regs[i].Bit(); 5378 } else if (regs[i].IsVRegister()) { 5379 number_of_valid_fpregs++; 5380 unique_fpregs |= regs[i].Bit(); 5381 } else { 5382 VIXL_ASSERT(!regs[i].IsValid()); 5383 } 5384 } 5385 5386 int number_of_unique_regs = CountSetBits(unique_regs); 5387 int number_of_unique_fpregs = CountSetBits(unique_fpregs); 5388 5389 VIXL_ASSERT(number_of_valid_regs >= number_of_unique_regs); 5390 VIXL_ASSERT(number_of_valid_fpregs >= number_of_unique_fpregs); 5391 5392 return (number_of_valid_regs != number_of_unique_regs) || 5393 (number_of_valid_fpregs != number_of_unique_fpregs); 5394 } 5395 5396 5397 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2, 5398 const CPURegister& reg3, const CPURegister& reg4, 5399 const CPURegister& reg5, const CPURegister& reg6, 5400 const CPURegister& reg7, const CPURegister& reg8) { 5401 VIXL_ASSERT(reg1.IsValid()); 5402 bool match = true; 5403 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1); 5404 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1); 5405 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1); 5406 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1); 5407 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1); 5408 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1); 5409 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1); 5410 return match; 5411 } 5412 5413 5414 bool AreSameFormat(const VRegister& reg1, const VRegister& reg2, 5415 const VRegister& reg3, const VRegister& reg4) { 5416 VIXL_ASSERT(reg1.IsValid()); 5417 bool match = true; 5418 match &= !reg2.IsValid() || reg2.IsSameFormat(reg1); 5419 match &= !reg3.IsValid() || reg3.IsSameFormat(reg1); 5420 match &= !reg4.IsValid() || reg4.IsSameFormat(reg1); 5421 return match; 5422 } 5423 5424 5425 bool AreConsecutive(const VRegister& reg1, const VRegister& reg2, 5426 const VRegister& reg3, const VRegister& reg4) { 5427 VIXL_ASSERT(reg1.IsValid()); 5428 bool match = true; 5429 match &= !reg2.IsValid() || 5430 (reg2.code() == ((reg1.code() + 1) % kNumberOfVRegisters)); 5431 match &= !reg3.IsValid() || 5432 (reg3.code() == ((reg1.code() + 2) % kNumberOfVRegisters)); 5433 match &= !reg4.IsValid() || 5434 (reg4.code() == ((reg1.code() + 3) % kNumberOfVRegisters)); 5435 return match; 5436 } 5437 } // namespace vixl 5438