1 // Copyright 2015, ARM Limited 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // * Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // * Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // * Neither the name of ARM Limited nor the names of its contributors may be 13 // used to endorse or promote products derived from this software without 14 // specific prior written permission. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27 #include <ctype.h> 28 29 #include "vixl/a64/macro-assembler-a64.h" 30 31 namespace vixl { 32 33 34 void Pool::Release() { 35 if (--monitor_ == 0) { 36 // Ensure the pool has not been blocked for too long. 37 VIXL_ASSERT(masm_->CursorOffset() < checkpoint_); 38 } 39 } 40 41 42 void Pool::SetNextCheckpoint(ptrdiff_t checkpoint) { 43 masm_->checkpoint_ = std::min(masm_->checkpoint_, checkpoint); 44 checkpoint_ = checkpoint; 45 } 46 47 48 LiteralPool::LiteralPool(MacroAssembler* masm) 49 : Pool(masm), size_(0), first_use_(-1), 50 recommended_checkpoint_(kNoCheckpointRequired) { 51 } 52 53 54 LiteralPool::~LiteralPool() { 55 VIXL_ASSERT(IsEmpty()); 56 VIXL_ASSERT(!IsBlocked()); 57 for (std::vector<RawLiteral*>::iterator it = deleted_on_destruction_.begin(); 58 it != deleted_on_destruction_.end(); 59 it++) { 60 delete *it; 61 } 62 } 63 64 65 void LiteralPool::Reset() { 66 std::vector<RawLiteral*>::iterator it, end; 67 for (it = entries_.begin(), end = entries_.end(); it != end; ++it) { 68 RawLiteral* literal = *it; 69 if (literal->deletion_policy_ == RawLiteral::kDeletedOnPlacementByPool) { 70 delete literal; 71 } 72 } 73 entries_.clear(); 74 size_ = 0; 75 first_use_ = -1; 76 Pool::Reset(); 77 recommended_checkpoint_ = kNoCheckpointRequired; 78 } 79 80 81 void LiteralPool::CheckEmitFor(size_t amount, EmitOption option) { 82 if (IsEmpty() || IsBlocked()) return; 83 84 ptrdiff_t distance = masm_->CursorOffset() + amount - first_use_; 85 if (distance >= kRecommendedLiteralPoolRange) { 86 Emit(option); 87 } 88 } 89 90 91 void LiteralPool::Emit(EmitOption option) { 92 // There is an issue if we are asked to emit a blocked or empty pool. 93 VIXL_ASSERT(!IsBlocked()); 94 VIXL_ASSERT(!IsEmpty()); 95 96 size_t pool_size = Size(); 97 size_t emit_size = pool_size; 98 if (option == kBranchRequired) emit_size += kInstructionSize; 99 Label end_of_pool; 100 101 VIXL_ASSERT(emit_size % kInstructionSize == 0); 102 InstructionAccurateScope guard(masm_, emit_size / kInstructionSize); 103 if (option == kBranchRequired) masm_->b(&end_of_pool); 104 105 // Marker indicating the size of the literal pool in 32-bit words. 106 VIXL_ASSERT((pool_size % kWRegSizeInBytes) == 0); 107 masm_->ldr(xzr, static_cast<int>(pool_size / kWRegSizeInBytes)); 108 109 // Now populate the literal pool. 110 std::vector<RawLiteral*>::iterator it, end; 111 for (it = entries_.begin(), end = entries_.end(); it != end; ++it) { 112 VIXL_ASSERT((*it)->IsUsed()); 113 masm_->place(*it); 114 } 115 116 if (option == kBranchRequired) masm_->bind(&end_of_pool); 117 118 Reset(); 119 } 120 121 122 void LiteralPool::AddEntry(RawLiteral* literal) { 123 // A literal must be registered immediately before its first use. Here we 124 // cannot control that it is its first use, but we check no code has been 125 // emitted since its last use. 126 VIXL_ASSERT(masm_->CursorOffset() == literal->last_use()); 127 128 UpdateFirstUse(masm_->CursorOffset()); 129 VIXL_ASSERT(masm_->CursorOffset() >= first_use_); 130 entries_.push_back(literal); 131 size_ += literal->size(); 132 } 133 134 135 void LiteralPool::UpdateFirstUse(ptrdiff_t use_position) { 136 first_use_ = std::min(first_use_, use_position); 137 if (first_use_ == -1) { 138 first_use_ = use_position; 139 SetNextRecommendedCheckpoint(NextRecommendedCheckpoint()); 140 SetNextCheckpoint(first_use_ + Instruction::kLoadLiteralRange); 141 } else { 142 VIXL_ASSERT(use_position > first_use_); 143 } 144 } 145 146 147 void VeneerPool::Reset() { 148 Pool::Reset(); 149 unresolved_branches_.Reset(); 150 } 151 152 153 void VeneerPool::Release() { 154 if (--monitor_ == 0) { 155 VIXL_ASSERT(IsEmpty() || 156 masm_->CursorOffset() < unresolved_branches_.FirstLimit()); 157 } 158 } 159 160 161 void VeneerPool::RegisterUnresolvedBranch(ptrdiff_t branch_pos, 162 Label* label, 163 ImmBranchType branch_type) { 164 VIXL_ASSERT(!label->IsBound()); 165 BranchInfo branch_info = BranchInfo(branch_pos, label, branch_type); 166 unresolved_branches_.insert(branch_info); 167 UpdateNextCheckPoint(); 168 // TODO: In debug mode register the label with the assembler to make sure it 169 // is bound with masm Bind and not asm bind. 170 } 171 172 173 void VeneerPool::DeleteUnresolvedBranchInfoForLabel(Label* label) { 174 if (IsEmpty()) { 175 VIXL_ASSERT(checkpoint_ == kNoCheckpointRequired); 176 return; 177 } 178 179 if (label->IsLinked()) { 180 Label::LabelLinksIterator links_it(label); 181 for (; !links_it.Done(); links_it.Advance()) { 182 ptrdiff_t link_offset = *links_it.Current(); 183 Instruction* link = masm_->InstructionAt(link_offset); 184 185 // ADR instructions are not handled. 186 if (BranchTypeUsesVeneers(link->BranchType())) { 187 BranchInfo branch_info(link_offset, label, link->BranchType()); 188 unresolved_branches_.erase(branch_info); 189 } 190 } 191 } 192 193 UpdateNextCheckPoint(); 194 } 195 196 197 bool VeneerPool::ShouldEmitVeneer(int64_t max_reachable_pc, size_t amount) { 198 ptrdiff_t offset = 199 kPoolNonVeneerCodeSize + amount + MaxSize() + OtherPoolsMaxSize(); 200 return (masm_->CursorOffset() + offset) > max_reachable_pc; 201 } 202 203 204 void VeneerPool::CheckEmitFor(size_t amount, EmitOption option) { 205 if (IsEmpty()) return; 206 207 VIXL_ASSERT(masm_->CursorOffset() < unresolved_branches_.FirstLimit()); 208 209 if (IsBlocked()) return; 210 211 if (ShouldEmitVeneers(amount)) { 212 Emit(option, amount); 213 } else { 214 UpdateNextCheckPoint(); 215 } 216 } 217 218 219 void VeneerPool::Emit(EmitOption option, size_t amount) { 220 // There is an issue if we are asked to emit a blocked or empty pool. 221 VIXL_ASSERT(!IsBlocked()); 222 VIXL_ASSERT(!IsEmpty()); 223 224 Label end; 225 if (option == kBranchRequired) { 226 InstructionAccurateScope scope(masm_, 1); 227 masm_->b(&end); 228 } 229 230 // We want to avoid generating veneer pools too often, so generate veneers for 231 // branches that don't immediately require a veneer but will soon go out of 232 // range. 233 static const size_t kVeneerEmissionMargin = 1 * KBytes; 234 235 for (BranchInfoSetIterator it(&unresolved_branches_); !it.Done();) { 236 BranchInfo* branch_info = it.Current(); 237 if (ShouldEmitVeneer(branch_info->max_reachable_pc_, 238 amount + kVeneerEmissionMargin)) { 239 InstructionAccurateScope scope(masm_, kVeneerCodeSize / kInstructionSize); 240 ptrdiff_t branch_pos = branch_info->pc_offset_; 241 Instruction* branch = masm_->InstructionAt(branch_pos); 242 Label* label = branch_info->label_; 243 244 // Patch the branch to point to the current position, and emit a branch 245 // to the label. 246 Instruction* veneer = masm_->GetCursorAddress<Instruction*>(); 247 branch->SetImmPCOffsetTarget(veneer); 248 masm_->b(label); 249 250 // Update the label. The branch patched does not point to it any longer. 251 label->DeleteLink(branch_pos); 252 253 it.DeleteCurrentAndAdvance(); 254 } else { 255 it.AdvanceToNextType(); 256 } 257 } 258 259 UpdateNextCheckPoint(); 260 261 masm_->bind(&end); 262 } 263 264 265 EmissionCheckScope::EmissionCheckScope(MacroAssembler* masm, size_t size) 266 : masm_(masm) { 267 masm_->EnsureEmitFor(size); 268 masm_->BlockPools(); 269 #ifdef VIXL_DEBUG 270 masm_->Bind(&start_); 271 size_ = size; 272 masm_->AcquireBuffer(); 273 #endif 274 } 275 276 277 EmissionCheckScope::~EmissionCheckScope() { 278 #ifdef VIXL_DEBUG 279 masm_->ReleaseBuffer(); 280 VIXL_ASSERT(masm_->SizeOfCodeGeneratedSince(&start_) <= size_); 281 #endif 282 masm_->ReleasePools(); 283 } 284 285 286 MacroAssembler::MacroAssembler(size_t capacity, 287 PositionIndependentCodeOption pic) 288 : Assembler(capacity, pic), 289 #ifdef VIXL_DEBUG 290 allow_macro_instructions_(true), 291 #endif 292 allow_simulator_instructions_(VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE), 293 sp_(sp), 294 tmp_list_(ip0, ip1), 295 fptmp_list_(d31), 296 literal_pool_(this), 297 veneer_pool_(this), 298 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 299 checkpoint_ = NextCheckPoint(); 300 } 301 302 303 MacroAssembler::MacroAssembler(byte * buffer, 304 size_t capacity, 305 PositionIndependentCodeOption pic) 306 : Assembler(buffer, capacity, pic), 307 #ifdef VIXL_DEBUG 308 allow_macro_instructions_(true), 309 #endif 310 allow_simulator_instructions_(VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE), 311 sp_(sp), 312 tmp_list_(ip0, ip1), 313 fptmp_list_(d31), 314 literal_pool_(this), 315 veneer_pool_(this), 316 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 317 checkpoint_ = NextCheckPoint(); 318 } 319 320 321 MacroAssembler::~MacroAssembler() { 322 } 323 324 325 void MacroAssembler::Reset() { 326 Assembler::Reset(); 327 328 VIXL_ASSERT(!literal_pool_.IsBlocked()); 329 literal_pool_.Reset(); 330 veneer_pool_.Reset(); 331 332 checkpoint_ = NextCheckPoint(); 333 } 334 335 336 void MacroAssembler::FinalizeCode() { 337 if (!literal_pool_.IsEmpty()) literal_pool_.Emit(); 338 VIXL_ASSERT(veneer_pool_.IsEmpty()); 339 340 Assembler::FinalizeCode(); 341 } 342 343 344 void MacroAssembler::CheckEmitFor(size_t amount) { 345 ptrdiff_t offset = amount; 346 347 literal_pool_.CheckEmitFor(amount); 348 veneer_pool_.CheckEmitFor(amount); 349 // Ensure there's enough space for the emit, keep in mind the cursor will 350 // have moved if a pool was emitted. 351 if ((CursorOffset() + offset) > BufferEndOffset()) { 352 EnsureSpaceFor(amount); 353 } 354 355 checkpoint_ = NextCheckPoint(); 356 } 357 358 359 int MacroAssembler::MoveImmediateHelper(MacroAssembler* masm, 360 const Register &rd, 361 uint64_t imm) { 362 bool emit_code = (masm != NULL); 363 VIXL_ASSERT(is_uint32(imm) || is_int32(imm) || rd.Is64Bits()); 364 // The worst case for size is mov 64-bit immediate to sp: 365 // * up to 4 instructions to materialise the constant 366 // * 1 instruction to move to sp 367 MacroEmissionCheckScope guard(masm); 368 369 // Immediates on Aarch64 can be produced using an initial value, and zero to 370 // three move keep operations. 371 // 372 // Initial values can be generated with: 373 // 1. 64-bit move zero (movz). 374 // 2. 32-bit move inverted (movn). 375 // 3. 64-bit move inverted. 376 // 4. 32-bit orr immediate. 377 // 5. 64-bit orr immediate. 378 // Move-keep may then be used to modify each of the 16-bit half words. 379 // 380 // The code below supports all five initial value generators, and 381 // applying move-keep operations to move-zero and move-inverted initial 382 // values. 383 384 // Try to move the immediate in one instruction, and if that fails, switch to 385 // using multiple instructions. 386 if (OneInstrMoveImmediateHelper(masm, rd, imm)) { 387 return 1; 388 } else { 389 int instruction_count = 0; 390 unsigned reg_size = rd.size(); 391 392 // Generic immediate case. Imm will be represented by 393 // [imm3, imm2, imm1, imm0], where each imm is 16 bits. 394 // A move-zero or move-inverted is generated for the first non-zero or 395 // non-0xffff immX, and a move-keep for subsequent non-zero immX. 396 397 uint64_t ignored_halfword = 0; 398 bool invert_move = false; 399 // If the number of 0xffff halfwords is greater than the number of 0x0000 400 // halfwords, it's more efficient to use move-inverted. 401 if (CountClearHalfWords(~imm, reg_size) > 402 CountClearHalfWords(imm, reg_size)) { 403 ignored_halfword = 0xffff; 404 invert_move = true; 405 } 406 407 // Mov instructions can't move values into the stack pointer, so set up a 408 // temporary register, if needed. 409 UseScratchRegisterScope temps; 410 Register temp; 411 if (emit_code) { 412 temps.Open(masm); 413 temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd; 414 } 415 416 // Iterate through the halfwords. Use movn/movz for the first non-ignored 417 // halfword, and movk for subsequent halfwords. 418 VIXL_ASSERT((reg_size % 16) == 0); 419 bool first_mov_done = false; 420 for (unsigned i = 0; i < (temp.size() / 16); i++) { 421 uint64_t imm16 = (imm >> (16 * i)) & 0xffff; 422 if (imm16 != ignored_halfword) { 423 if (!first_mov_done) { 424 if (invert_move) { 425 if (emit_code) masm->movn(temp, ~imm16 & 0xffff, 16 * i); 426 instruction_count++; 427 } else { 428 if (emit_code) masm->movz(temp, imm16, 16 * i); 429 instruction_count++; 430 } 431 first_mov_done = true; 432 } else { 433 // Construct a wider constant. 434 if (emit_code) masm->movk(temp, imm16, 16 * i); 435 instruction_count++; 436 } 437 } 438 } 439 440 VIXL_ASSERT(first_mov_done); 441 442 // Move the temporary if the original destination register was the stack 443 // pointer. 444 if (rd.IsSP()) { 445 if (emit_code) masm->mov(rd, temp); 446 instruction_count++; 447 } 448 return instruction_count; 449 } 450 } 451 452 453 bool MacroAssembler::OneInstrMoveImmediateHelper(MacroAssembler* masm, 454 const Register& dst, 455 int64_t imm) { 456 bool emit_code = masm != NULL; 457 unsigned n, imm_s, imm_r; 458 int reg_size = dst.size(); 459 460 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) { 461 // Immediate can be represented in a move zero instruction. Movz can't write 462 // to the stack pointer. 463 if (emit_code) { 464 masm->movz(dst, imm); 465 } 466 return true; 467 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) { 468 // Immediate can be represented in a move negative instruction. Movn can't 469 // write to the stack pointer. 470 if (emit_code) { 471 masm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask)); 472 } 473 return true; 474 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) { 475 // Immediate can be represented in a logical orr instruction. 476 VIXL_ASSERT(!dst.IsZero()); 477 if (emit_code) { 478 masm->LogicalImmediate( 479 dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR); 480 } 481 return true; 482 } 483 return false; 484 } 485 486 487 void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) { 488 VIXL_ASSERT((reg.Is(NoReg) || (type >= kBranchTypeFirstUsingReg)) && 489 ((bit == -1) || (type >= kBranchTypeFirstUsingBit))); 490 if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) { 491 B(static_cast<Condition>(type), label); 492 } else { 493 switch (type) { 494 case always: B(label); break; 495 case never: break; 496 case reg_zero: Cbz(reg, label); break; 497 case reg_not_zero: Cbnz(reg, label); break; 498 case reg_bit_clear: Tbz(reg, bit, label); break; 499 case reg_bit_set: Tbnz(reg, bit, label); break; 500 default: 501 VIXL_UNREACHABLE(); 502 } 503 } 504 } 505 506 507 void MacroAssembler::B(Label* label) { 508 SingleEmissionCheckScope guard(this); 509 b(label); 510 } 511 512 513 void MacroAssembler::B(Label* label, Condition cond) { 514 VIXL_ASSERT(allow_macro_instructions_); 515 VIXL_ASSERT((cond != al) && (cond != nv)); 516 EmissionCheckScope guard(this, 2 * kInstructionSize); 517 518 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 519 Label done; 520 b(&done, InvertCondition(cond)); 521 b(label); 522 bind(&done); 523 } else { 524 if (!label->IsBound()) { 525 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 526 label, 527 CondBranchType); 528 } 529 b(label, cond); 530 } 531 } 532 533 534 void MacroAssembler::Cbnz(const Register& rt, Label* label) { 535 VIXL_ASSERT(allow_macro_instructions_); 536 VIXL_ASSERT(!rt.IsZero()); 537 EmissionCheckScope guard(this, 2 * kInstructionSize); 538 539 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 540 Label done; 541 cbz(rt, &done); 542 b(label); 543 bind(&done); 544 } else { 545 if (!label->IsBound()) { 546 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 547 label, 548 CompareBranchType); 549 } 550 cbnz(rt, label); 551 } 552 } 553 554 555 void MacroAssembler::Cbz(const Register& rt, Label* label) { 556 VIXL_ASSERT(allow_macro_instructions_); 557 VIXL_ASSERT(!rt.IsZero()); 558 EmissionCheckScope guard(this, 2 * kInstructionSize); 559 560 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 561 Label done; 562 cbnz(rt, &done); 563 b(label); 564 bind(&done); 565 } else { 566 if (!label->IsBound()) { 567 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 568 label, 569 CompareBranchType); 570 } 571 cbz(rt, label); 572 } 573 } 574 575 576 void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) { 577 VIXL_ASSERT(allow_macro_instructions_); 578 VIXL_ASSERT(!rt.IsZero()); 579 EmissionCheckScope guard(this, 2 * kInstructionSize); 580 581 if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) { 582 Label done; 583 tbz(rt, bit_pos, &done); 584 b(label); 585 bind(&done); 586 } else { 587 if (!label->IsBound()) { 588 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 589 label, 590 TestBranchType); 591 } 592 tbnz(rt, bit_pos, label); 593 } 594 } 595 596 597 void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) { 598 VIXL_ASSERT(allow_macro_instructions_); 599 VIXL_ASSERT(!rt.IsZero()); 600 EmissionCheckScope guard(this, 2 * kInstructionSize); 601 602 if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) { 603 Label done; 604 tbnz(rt, bit_pos, &done); 605 b(label); 606 bind(&done); 607 } else { 608 if (!label->IsBound()) { 609 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 610 label, 611 TestBranchType); 612 } 613 tbz(rt, bit_pos, label); 614 } 615 } 616 617 618 void MacroAssembler::Bind(Label* label) { 619 VIXL_ASSERT(allow_macro_instructions_); 620 veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label); 621 bind(label); 622 } 623 624 625 // Bind a label to a specified offset from the start of the buffer. 626 void MacroAssembler::BindToOffset(Label* label, ptrdiff_t offset) { 627 VIXL_ASSERT(allow_macro_instructions_); 628 veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label); 629 Assembler::BindToOffset(label, offset); 630 } 631 632 633 void MacroAssembler::And(const Register& rd, 634 const Register& rn, 635 const Operand& operand) { 636 VIXL_ASSERT(allow_macro_instructions_); 637 LogicalMacro(rd, rn, operand, AND); 638 } 639 640 641 void MacroAssembler::Ands(const Register& rd, 642 const Register& rn, 643 const Operand& operand) { 644 VIXL_ASSERT(allow_macro_instructions_); 645 LogicalMacro(rd, rn, operand, ANDS); 646 } 647 648 649 void MacroAssembler::Tst(const Register& rn, 650 const Operand& operand) { 651 VIXL_ASSERT(allow_macro_instructions_); 652 Ands(AppropriateZeroRegFor(rn), rn, operand); 653 } 654 655 656 void MacroAssembler::Bic(const Register& rd, 657 const Register& rn, 658 const Operand& operand) { 659 VIXL_ASSERT(allow_macro_instructions_); 660 LogicalMacro(rd, rn, operand, BIC); 661 } 662 663 664 void MacroAssembler::Bics(const Register& rd, 665 const Register& rn, 666 const Operand& operand) { 667 VIXL_ASSERT(allow_macro_instructions_); 668 LogicalMacro(rd, rn, operand, BICS); 669 } 670 671 672 void MacroAssembler::Orr(const Register& rd, 673 const Register& rn, 674 const Operand& operand) { 675 VIXL_ASSERT(allow_macro_instructions_); 676 LogicalMacro(rd, rn, operand, ORR); 677 } 678 679 680 void MacroAssembler::Orn(const Register& rd, 681 const Register& rn, 682 const Operand& operand) { 683 VIXL_ASSERT(allow_macro_instructions_); 684 LogicalMacro(rd, rn, operand, ORN); 685 } 686 687 688 void MacroAssembler::Eor(const Register& rd, 689 const Register& rn, 690 const Operand& operand) { 691 VIXL_ASSERT(allow_macro_instructions_); 692 LogicalMacro(rd, rn, operand, EOR); 693 } 694 695 696 void MacroAssembler::Eon(const Register& rd, 697 const Register& rn, 698 const Operand& operand) { 699 VIXL_ASSERT(allow_macro_instructions_); 700 LogicalMacro(rd, rn, operand, EON); 701 } 702 703 704 void MacroAssembler::LogicalMacro(const Register& rd, 705 const Register& rn, 706 const Operand& operand, 707 LogicalOp op) { 708 // The worst case for size is logical immediate to sp: 709 // * up to 4 instructions to materialise the constant 710 // * 1 instruction to do the operation 711 // * 1 instruction to move to sp 712 MacroEmissionCheckScope guard(this); 713 UseScratchRegisterScope temps(this); 714 715 if (operand.IsImmediate()) { 716 int64_t immediate = operand.immediate(); 717 unsigned reg_size = rd.size(); 718 719 // If the operation is NOT, invert the operation and immediate. 720 if ((op & NOT) == NOT) { 721 op = static_cast<LogicalOp>(op & ~NOT); 722 immediate = ~immediate; 723 } 724 725 // Ignore the top 32 bits of an immediate if we're moving to a W register. 726 if (rd.Is32Bits()) { 727 // Check that the top 32 bits are consistent. 728 VIXL_ASSERT(((immediate >> kWRegSize) == 0) || 729 ((immediate >> kWRegSize) == -1)); 730 immediate &= kWRegMask; 731 } 732 733 VIXL_ASSERT(rd.Is64Bits() || is_uint32(immediate)); 734 735 // Special cases for all set or all clear immediates. 736 if (immediate == 0) { 737 switch (op) { 738 case AND: 739 Mov(rd, 0); 740 return; 741 case ORR: 742 VIXL_FALLTHROUGH(); 743 case EOR: 744 Mov(rd, rn); 745 return; 746 case ANDS: 747 VIXL_FALLTHROUGH(); 748 case BICS: 749 break; 750 default: 751 VIXL_UNREACHABLE(); 752 } 753 } else if ((rd.Is64Bits() && (immediate == -1)) || 754 (rd.Is32Bits() && (immediate == 0xffffffff))) { 755 switch (op) { 756 case AND: 757 Mov(rd, rn); 758 return; 759 case ORR: 760 Mov(rd, immediate); 761 return; 762 case EOR: 763 Mvn(rd, rn); 764 return; 765 case ANDS: 766 VIXL_FALLTHROUGH(); 767 case BICS: 768 break; 769 default: 770 VIXL_UNREACHABLE(); 771 } 772 } 773 774 unsigned n, imm_s, imm_r; 775 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 776 // Immediate can be encoded in the instruction. 777 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 778 } else { 779 // Immediate can't be encoded: synthesize using move immediate. 780 Register temp = temps.AcquireSameSizeAs(rn); 781 Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate); 782 783 if (rd.Is(sp)) { 784 // If rd is the stack pointer we cannot use it as the destination 785 // register so we use the temp register as an intermediate again. 786 Logical(temp, rn, imm_operand, op); 787 Mov(sp, temp); 788 } else { 789 Logical(rd, rn, imm_operand, op); 790 } 791 } 792 } else if (operand.IsExtendedRegister()) { 793 VIXL_ASSERT(operand.reg().size() <= rd.size()); 794 // Add/sub extended supports shift <= 4. We want to support exactly the 795 // same modes here. 796 VIXL_ASSERT(operand.shift_amount() <= 4); 797 VIXL_ASSERT(operand.reg().Is64Bits() || 798 ((operand.extend() != UXTX) && (operand.extend() != SXTX))); 799 800 temps.Exclude(operand.reg()); 801 Register temp = temps.AcquireSameSizeAs(rn); 802 EmitExtendShift(temp, operand.reg(), operand.extend(), 803 operand.shift_amount()); 804 Logical(rd, rn, Operand(temp), op); 805 } else { 806 // The operand can be encoded in the instruction. 807 VIXL_ASSERT(operand.IsShiftedRegister()); 808 Logical(rd, rn, operand, op); 809 } 810 } 811 812 813 void MacroAssembler::Mov(const Register& rd, 814 const Operand& operand, 815 DiscardMoveMode discard_mode) { 816 VIXL_ASSERT(allow_macro_instructions_); 817 // The worst case for size is mov immediate with up to 4 instructions. 818 MacroEmissionCheckScope guard(this); 819 820 if (operand.IsImmediate()) { 821 // Call the macro assembler for generic immediates. 822 Mov(rd, operand.immediate()); 823 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) { 824 // Emit a shift instruction if moving a shifted register. This operation 825 // could also be achieved using an orr instruction (like orn used by Mvn), 826 // but using a shift instruction makes the disassembly clearer. 827 EmitShift(rd, operand.reg(), operand.shift(), operand.shift_amount()); 828 } else if (operand.IsExtendedRegister()) { 829 // Emit an extend instruction if moving an extended register. This handles 830 // extend with post-shift operations, too. 831 EmitExtendShift(rd, operand.reg(), operand.extend(), 832 operand.shift_amount()); 833 } else { 834 // Otherwise, emit a register move only if the registers are distinct, or 835 // if they are not X registers. 836 // 837 // Note that mov(w0, w0) is not a no-op because it clears the top word of 838 // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W 839 // registers is not required to clear the top word of the X register. In 840 // this case, the instruction is discarded. 841 // 842 // If the sp is an operand, add #0 is emitted, otherwise, orr #0. 843 if (!rd.Is(operand.reg()) || (rd.Is32Bits() && 844 (discard_mode == kDontDiscardForSameWReg))) { 845 mov(rd, operand.reg()); 846 } 847 } 848 } 849 850 851 void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) { 852 VIXL_ASSERT(is_uint16(imm)); 853 int byte1 = (imm & 0xff); 854 int byte2 = ((imm >> 8) & 0xff); 855 if (byte1 == byte2) { 856 movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1); 857 } else if (byte1 == 0) { 858 movi(vd, byte2, LSL, 8); 859 } else if (byte2 == 0) { 860 movi(vd, byte1); 861 } else if (byte1 == 0xff) { 862 mvni(vd, ~byte2 & 0xff, LSL, 8); 863 } else if (byte2 == 0xff) { 864 mvni(vd, ~byte1 & 0xff); 865 } else { 866 UseScratchRegisterScope temps(this); 867 Register temp = temps.AcquireW(); 868 movz(temp, imm); 869 dup(vd, temp); 870 } 871 } 872 873 874 void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) { 875 VIXL_ASSERT(is_uint32(imm)); 876 877 uint8_t bytes[sizeof(imm)]; 878 memcpy(bytes, &imm, sizeof(imm)); 879 880 // All bytes are either 0x00 or 0xff. 881 { 882 bool all0orff = true; 883 for (int i = 0; i < 4; ++i) { 884 if ((bytes[i] != 0) && (bytes[i] != 0xff)) { 885 all0orff = false; 886 break; 887 } 888 } 889 890 if (all0orff == true) { 891 movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm)); 892 return; 893 } 894 } 895 896 // Of the 4 bytes, only one byte is non-zero. 897 for (int i = 0; i < 4; i++) { 898 if ((imm & (0xff << (i * 8))) == imm) { 899 movi(vd, bytes[i], LSL, i * 8); 900 return; 901 } 902 } 903 904 // Of the 4 bytes, only one byte is not 0xff. 905 for (int i = 0; i < 4; i++) { 906 uint32_t mask = ~(0xff << (i * 8)); 907 if ((imm & mask) == mask) { 908 mvni(vd, ~bytes[i] & 0xff, LSL, i * 8); 909 return; 910 } 911 } 912 913 // Immediate is of the form 0x00MMFFFF. 914 if ((imm & 0xff00ffff) == 0x0000ffff) { 915 movi(vd, bytes[2], MSL, 16); 916 return; 917 } 918 919 // Immediate is of the form 0x0000MMFF. 920 if ((imm & 0xffff00ff) == 0x000000ff) { 921 movi(vd, bytes[1], MSL, 8); 922 return; 923 } 924 925 // Immediate is of the form 0xFFMM0000. 926 if ((imm & 0xff00ffff) == 0xff000000) { 927 mvni(vd, ~bytes[2] & 0xff, MSL, 16); 928 return; 929 } 930 // Immediate is of the form 0xFFFFMM00. 931 if ((imm & 0xffff00ff) == 0xffff0000) { 932 mvni(vd, ~bytes[1] & 0xff, MSL, 8); 933 return; 934 } 935 936 // Top and bottom 16-bits are equal. 937 if (((imm >> 16) & 0xffff) == (imm & 0xffff)) { 938 Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff); 939 return; 940 } 941 942 // Default case. 943 { 944 UseScratchRegisterScope temps(this); 945 Register temp = temps.AcquireW(); 946 Mov(temp, imm); 947 dup(vd, temp); 948 } 949 } 950 951 952 void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) { 953 // All bytes are either 0x00 or 0xff. 954 { 955 bool all0orff = true; 956 for (int i = 0; i < 8; ++i) { 957 int byteval = (imm >> (i * 8)) & 0xff; 958 if (byteval != 0 && byteval != 0xff) { 959 all0orff = false; 960 break; 961 } 962 } 963 if (all0orff == true) { 964 movi(vd, imm); 965 return; 966 } 967 } 968 969 // Top and bottom 32-bits are equal. 970 if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) { 971 Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff); 972 return; 973 } 974 975 // Default case. 976 { 977 UseScratchRegisterScope temps(this); 978 Register temp = temps.AcquireX(); 979 Mov(temp, imm); 980 if (vd.Is1D()) { 981 mov(vd.D(), 0, temp); 982 } else { 983 dup(vd.V2D(), temp); 984 } 985 } 986 } 987 988 989 void MacroAssembler::Movi(const VRegister& vd, 990 uint64_t imm, 991 Shift shift, 992 int shift_amount) { 993 VIXL_ASSERT(allow_macro_instructions_); 994 MacroEmissionCheckScope guard(this); 995 if (shift_amount != 0 || shift != LSL) { 996 movi(vd, imm, shift, shift_amount); 997 } else if (vd.Is8B() || vd.Is16B()) { 998 // 8-bit immediate. 999 VIXL_ASSERT(is_uint8(imm)); 1000 movi(vd, imm); 1001 } else if (vd.Is4H() || vd.Is8H()) { 1002 // 16-bit immediate. 1003 Movi16bitHelper(vd, imm); 1004 } else if (vd.Is2S() || vd.Is4S()) { 1005 // 32-bit immediate. 1006 Movi32bitHelper(vd, imm); 1007 } else { 1008 // 64-bit immediate. 1009 Movi64bitHelper(vd, imm); 1010 } 1011 } 1012 1013 1014 void MacroAssembler::Movi(const VRegister& vd, 1015 uint64_t hi, 1016 uint64_t lo) { 1017 // TODO: Move 128-bit values in a more efficient way. 1018 VIXL_ASSERT(vd.Is128Bits()); 1019 UseScratchRegisterScope temps(this); 1020 Movi(vd.V2D(), lo); 1021 Register temp = temps.AcquireX(); 1022 Mov(temp, hi); 1023 Ins(vd.V2D(), 1, temp); 1024 } 1025 1026 1027 void MacroAssembler::Mvn(const Register& rd, const Operand& operand) { 1028 VIXL_ASSERT(allow_macro_instructions_); 1029 // The worst case for size is mvn immediate with up to 4 instructions. 1030 MacroEmissionCheckScope guard(this); 1031 1032 if (operand.IsImmediate()) { 1033 // Call the macro assembler for generic immediates. 1034 Mvn(rd, operand.immediate()); 1035 } else if (operand.IsExtendedRegister()) { 1036 UseScratchRegisterScope temps(this); 1037 temps.Exclude(operand.reg()); 1038 1039 // Emit two instructions for the extend case. This differs from Mov, as 1040 // the extend and invert can't be achieved in one instruction. 1041 Register temp = temps.AcquireSameSizeAs(rd); 1042 EmitExtendShift(temp, operand.reg(), operand.extend(), 1043 operand.shift_amount()); 1044 mvn(rd, Operand(temp)); 1045 } else { 1046 // Otherwise, register and shifted register cases can be handled by the 1047 // assembler directly, using orn. 1048 mvn(rd, operand); 1049 } 1050 } 1051 1052 1053 void MacroAssembler::Mov(const Register& rd, uint64_t imm) { 1054 VIXL_ASSERT(allow_macro_instructions_); 1055 MoveImmediateHelper(this, rd, imm); 1056 } 1057 1058 1059 void MacroAssembler::Ccmp(const Register& rn, 1060 const Operand& operand, 1061 StatusFlags nzcv, 1062 Condition cond) { 1063 VIXL_ASSERT(allow_macro_instructions_); 1064 if (operand.IsImmediate() && (operand.immediate() < 0)) { 1065 ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMN); 1066 } else { 1067 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP); 1068 } 1069 } 1070 1071 1072 void MacroAssembler::Ccmn(const Register& rn, 1073 const Operand& operand, 1074 StatusFlags nzcv, 1075 Condition cond) { 1076 VIXL_ASSERT(allow_macro_instructions_); 1077 if (operand.IsImmediate() && (operand.immediate() < 0)) { 1078 ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMP); 1079 } else { 1080 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN); 1081 } 1082 } 1083 1084 1085 void MacroAssembler::ConditionalCompareMacro(const Register& rn, 1086 const Operand& operand, 1087 StatusFlags nzcv, 1088 Condition cond, 1089 ConditionalCompareOp op) { 1090 VIXL_ASSERT((cond != al) && (cond != nv)); 1091 // The worst case for size is ccmp immediate: 1092 // * up to 4 instructions to materialise the constant 1093 // * 1 instruction for ccmp 1094 MacroEmissionCheckScope guard(this); 1095 1096 if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) || 1097 (operand.IsImmediate() && IsImmConditionalCompare(operand.immediate()))) { 1098 // The immediate can be encoded in the instruction, or the operand is an 1099 // unshifted register: call the assembler. 1100 ConditionalCompare(rn, operand, nzcv, cond, op); 1101 } else { 1102 UseScratchRegisterScope temps(this); 1103 // The operand isn't directly supported by the instruction: perform the 1104 // operation on a temporary register. 1105 Register temp = temps.AcquireSameSizeAs(rn); 1106 Mov(temp, operand); 1107 ConditionalCompare(rn, temp, nzcv, cond, op); 1108 } 1109 } 1110 1111 1112 void MacroAssembler::Csel(const Register& rd, 1113 const Register& rn, 1114 const Operand& operand, 1115 Condition cond) { 1116 VIXL_ASSERT(allow_macro_instructions_); 1117 VIXL_ASSERT(!rd.IsZero()); 1118 VIXL_ASSERT(!rn.IsZero()); 1119 VIXL_ASSERT((cond != al) && (cond != nv)); 1120 // The worst case for size is csel immediate: 1121 // * up to 4 instructions to materialise the constant 1122 // * 1 instruction for csel 1123 MacroEmissionCheckScope guard(this); 1124 1125 if (operand.IsImmediate()) { 1126 // Immediate argument. Handle special cases of 0, 1 and -1 using zero 1127 // register. 1128 int64_t imm = operand.immediate(); 1129 Register zr = AppropriateZeroRegFor(rn); 1130 if (imm == 0) { 1131 csel(rd, rn, zr, cond); 1132 } else if (imm == 1) { 1133 csinc(rd, rn, zr, cond); 1134 } else if (imm == -1) { 1135 csinv(rd, rn, zr, cond); 1136 } else { 1137 UseScratchRegisterScope temps(this); 1138 Register temp = temps.AcquireSameSizeAs(rn); 1139 Mov(temp, operand.immediate()); 1140 csel(rd, rn, temp, cond); 1141 } 1142 } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) { 1143 // Unshifted register argument. 1144 csel(rd, rn, operand.reg(), cond); 1145 } else { 1146 // All other arguments. 1147 UseScratchRegisterScope temps(this); 1148 Register temp = temps.AcquireSameSizeAs(rn); 1149 Mov(temp, operand); 1150 csel(rd, rn, temp, cond); 1151 } 1152 } 1153 1154 1155 void MacroAssembler::Add(const Register& rd, 1156 const Register& rn, 1157 const Operand& operand, 1158 FlagsUpdate S) { 1159 VIXL_ASSERT(allow_macro_instructions_); 1160 if (operand.IsImmediate() && (operand.immediate() < 0) && 1161 IsImmAddSub(-operand.immediate())) { 1162 AddSubMacro(rd, rn, -operand.immediate(), S, SUB); 1163 } else { 1164 AddSubMacro(rd, rn, operand, S, ADD); 1165 } 1166 } 1167 1168 1169 void MacroAssembler::Adds(const Register& rd, 1170 const Register& rn, 1171 const Operand& operand) { 1172 Add(rd, rn, operand, SetFlags); 1173 } 1174 1175 1176 void MacroAssembler::Sub(const Register& rd, 1177 const Register& rn, 1178 const Operand& operand, 1179 FlagsUpdate S) { 1180 VIXL_ASSERT(allow_macro_instructions_); 1181 if (operand.IsImmediate() && (operand.immediate() < 0) && 1182 IsImmAddSub(-operand.immediate())) { 1183 AddSubMacro(rd, rn, -operand.immediate(), S, ADD); 1184 } else { 1185 AddSubMacro(rd, rn, operand, S, SUB); 1186 } 1187 } 1188 1189 1190 void MacroAssembler::Subs(const Register& rd, 1191 const Register& rn, 1192 const Operand& operand) { 1193 Sub(rd, rn, operand, SetFlags); 1194 } 1195 1196 1197 void MacroAssembler::Cmn(const Register& rn, const Operand& operand) { 1198 VIXL_ASSERT(allow_macro_instructions_); 1199 Adds(AppropriateZeroRegFor(rn), rn, operand); 1200 } 1201 1202 1203 void MacroAssembler::Cmp(const Register& rn, const Operand& operand) { 1204 VIXL_ASSERT(allow_macro_instructions_); 1205 Subs(AppropriateZeroRegFor(rn), rn, operand); 1206 } 1207 1208 1209 void MacroAssembler::Fcmp(const FPRegister& fn, double value, 1210 FPTrapFlags trap) { 1211 VIXL_ASSERT(allow_macro_instructions_); 1212 // The worst case for size is: 1213 // * 1 to materialise the constant, using literal pool if necessary 1214 // * 1 instruction for fcmp{e} 1215 MacroEmissionCheckScope guard(this); 1216 if (value != 0.0) { 1217 UseScratchRegisterScope temps(this); 1218 FPRegister tmp = temps.AcquireSameSizeAs(fn); 1219 Fmov(tmp, value); 1220 FPCompareMacro(fn, tmp, trap); 1221 } else { 1222 FPCompareMacro(fn, value, trap); 1223 } 1224 } 1225 1226 1227 void MacroAssembler::Fcmpe(const FPRegister& fn, double value) { 1228 Fcmp(fn, value, EnableTrap); 1229 } 1230 1231 1232 void MacroAssembler::Fmov(VRegister vd, double imm) { 1233 VIXL_ASSERT(allow_macro_instructions_); 1234 // Floating point immediates are loaded through the literal pool. 1235 MacroEmissionCheckScope guard(this); 1236 1237 if (vd.Is1S() || vd.Is2S() || vd.Is4S()) { 1238 Fmov(vd, static_cast<float>(imm)); 1239 return; 1240 } 1241 1242 VIXL_ASSERT(vd.Is1D() || vd.Is2D()); 1243 if (IsImmFP64(imm)) { 1244 fmov(vd, imm); 1245 } else { 1246 uint64_t rawbits = double_to_rawbits(imm); 1247 if (vd.IsScalar()) { 1248 if (rawbits == 0) { 1249 fmov(vd, xzr); 1250 } else { 1251 ldr(vd, 1252 new Literal<double>(imm, 1253 &literal_pool_, 1254 RawLiteral::kDeletedOnPlacementByPool)); 1255 } 1256 } else { 1257 // TODO: consider NEON support for load literal. 1258 Movi(vd, rawbits); 1259 } 1260 } 1261 } 1262 1263 1264 void MacroAssembler::Fmov(VRegister vd, float imm) { 1265 VIXL_ASSERT(allow_macro_instructions_); 1266 // Floating point immediates are loaded through the literal pool. 1267 MacroEmissionCheckScope guard(this); 1268 1269 if (vd.Is1D() || vd.Is2D()) { 1270 Fmov(vd, static_cast<double>(imm)); 1271 return; 1272 } 1273 1274 VIXL_ASSERT(vd.Is1S() || vd.Is2S() || vd.Is4S()); 1275 if (IsImmFP32(imm)) { 1276 fmov(vd, imm); 1277 } else { 1278 uint32_t rawbits = float_to_rawbits(imm); 1279 if (vd.IsScalar()) { 1280 if (rawbits == 0) { 1281 fmov(vd, wzr); 1282 } else { 1283 ldr(vd, 1284 new Literal<float>(imm, 1285 &literal_pool_, 1286 RawLiteral::kDeletedOnPlacementByPool)); 1287 } 1288 } else { 1289 // TODO: consider NEON support for load literal. 1290 Movi(vd, rawbits); 1291 } 1292 } 1293 } 1294 1295 1296 1297 void MacroAssembler::Neg(const Register& rd, 1298 const Operand& operand) { 1299 VIXL_ASSERT(allow_macro_instructions_); 1300 if (operand.IsImmediate()) { 1301 Mov(rd, -operand.immediate()); 1302 } else { 1303 Sub(rd, AppropriateZeroRegFor(rd), operand); 1304 } 1305 } 1306 1307 1308 void MacroAssembler::Negs(const Register& rd, 1309 const Operand& operand) { 1310 VIXL_ASSERT(allow_macro_instructions_); 1311 Subs(rd, AppropriateZeroRegFor(rd), operand); 1312 } 1313 1314 1315 bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst, 1316 int64_t imm) { 1317 return OneInstrMoveImmediateHelper(this, dst, imm); 1318 } 1319 1320 1321 Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst, 1322 int64_t imm) { 1323 int reg_size = dst.size(); 1324 1325 // Encode the immediate in a single move instruction, if possible. 1326 if (TryOneInstrMoveImmediate(dst, imm)) { 1327 // The move was successful; nothing to do here. 1328 } else { 1329 // Pre-shift the immediate to the least-significant bits of the register. 1330 int shift_low = CountTrailingZeros(imm, reg_size); 1331 int64_t imm_low = imm >> shift_low; 1332 1333 // Pre-shift the immediate to the most-significant bits of the register, 1334 // inserting set bits in the least-significant bits. 1335 int shift_high = CountLeadingZeros(imm, reg_size); 1336 int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1); 1337 1338 if (TryOneInstrMoveImmediate(dst, imm_low)) { 1339 // The new immediate has been moved into the destination's low bits: 1340 // return a new leftward-shifting operand. 1341 return Operand(dst, LSL, shift_low); 1342 } else if (TryOneInstrMoveImmediate(dst, imm_high)) { 1343 // The new immediate has been moved into the destination's high bits: 1344 // return a new rightward-shifting operand. 1345 return Operand(dst, LSR, shift_high); 1346 } else { 1347 Mov(dst, imm); 1348 } 1349 } 1350 return Operand(dst); 1351 } 1352 1353 1354 void MacroAssembler::ComputeAddress(const Register& dst, 1355 const MemOperand& mem_op) { 1356 // We cannot handle pre-indexing or post-indexing. 1357 VIXL_ASSERT(mem_op.addrmode() == Offset); 1358 Register base = mem_op.base(); 1359 if (mem_op.IsImmediateOffset()) { 1360 Add(dst, base, mem_op.offset()); 1361 } else { 1362 VIXL_ASSERT(mem_op.IsRegisterOffset()); 1363 Register reg_offset = mem_op.regoffset(); 1364 Shift shift = mem_op.shift(); 1365 Extend extend = mem_op.extend(); 1366 if (shift == NO_SHIFT) { 1367 VIXL_ASSERT(extend != NO_EXTEND); 1368 Add(dst, base, Operand(reg_offset, extend, mem_op.shift_amount())); 1369 } else { 1370 VIXL_ASSERT(extend == NO_EXTEND); 1371 Add(dst, base, Operand(reg_offset, shift, mem_op.shift_amount())); 1372 } 1373 } 1374 } 1375 1376 1377 void MacroAssembler::AddSubMacro(const Register& rd, 1378 const Register& rn, 1379 const Operand& operand, 1380 FlagsUpdate S, 1381 AddSubOp op) { 1382 // Worst case is add/sub immediate: 1383 // * up to 4 instructions to materialise the constant 1384 // * 1 instruction for add/sub 1385 MacroEmissionCheckScope guard(this); 1386 1387 if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() && 1388 (S == LeaveFlags)) { 1389 // The instruction would be a nop. Avoid generating useless code. 1390 return; 1391 } 1392 1393 if ((operand.IsImmediate() && !IsImmAddSub(operand.immediate())) || 1394 (rn.IsZero() && !operand.IsShiftedRegister()) || 1395 (operand.IsShiftedRegister() && (operand.shift() == ROR))) { 1396 UseScratchRegisterScope temps(this); 1397 Register temp = temps.AcquireSameSizeAs(rn); 1398 if (operand.IsImmediate()) { 1399 Operand imm_operand = 1400 MoveImmediateForShiftedOp(temp, operand.immediate()); 1401 AddSub(rd, rn, imm_operand, S, op); 1402 } else { 1403 Mov(temp, operand); 1404 AddSub(rd, rn, temp, S, op); 1405 } 1406 } else { 1407 AddSub(rd, rn, operand, S, op); 1408 } 1409 } 1410 1411 1412 void MacroAssembler::Adc(const Register& rd, 1413 const Register& rn, 1414 const Operand& operand) { 1415 VIXL_ASSERT(allow_macro_instructions_); 1416 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC); 1417 } 1418 1419 1420 void MacroAssembler::Adcs(const Register& rd, 1421 const Register& rn, 1422 const Operand& operand) { 1423 VIXL_ASSERT(allow_macro_instructions_); 1424 AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC); 1425 } 1426 1427 1428 void MacroAssembler::Sbc(const Register& rd, 1429 const Register& rn, 1430 const Operand& operand) { 1431 VIXL_ASSERT(allow_macro_instructions_); 1432 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC); 1433 } 1434 1435 1436 void MacroAssembler::Sbcs(const Register& rd, 1437 const Register& rn, 1438 const Operand& operand) { 1439 VIXL_ASSERT(allow_macro_instructions_); 1440 AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC); 1441 } 1442 1443 1444 void MacroAssembler::Ngc(const Register& rd, 1445 const Operand& operand) { 1446 VIXL_ASSERT(allow_macro_instructions_); 1447 Register zr = AppropriateZeroRegFor(rd); 1448 Sbc(rd, zr, operand); 1449 } 1450 1451 1452 void MacroAssembler::Ngcs(const Register& rd, 1453 const Operand& operand) { 1454 VIXL_ASSERT(allow_macro_instructions_); 1455 Register zr = AppropriateZeroRegFor(rd); 1456 Sbcs(rd, zr, operand); 1457 } 1458 1459 1460 void MacroAssembler::AddSubWithCarryMacro(const Register& rd, 1461 const Register& rn, 1462 const Operand& operand, 1463 FlagsUpdate S, 1464 AddSubWithCarryOp op) { 1465 VIXL_ASSERT(rd.size() == rn.size()); 1466 // Worst case is addc/subc immediate: 1467 // * up to 4 instructions to materialise the constant 1468 // * 1 instruction for add/sub 1469 MacroEmissionCheckScope guard(this); 1470 UseScratchRegisterScope temps(this); 1471 1472 if (operand.IsImmediate() || 1473 (operand.IsShiftedRegister() && (operand.shift() == ROR))) { 1474 // Add/sub with carry (immediate or ROR shifted register.) 1475 Register temp = temps.AcquireSameSizeAs(rn); 1476 Mov(temp, operand); 1477 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1478 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) { 1479 // Add/sub with carry (shifted register). 1480 VIXL_ASSERT(operand.reg().size() == rd.size()); 1481 VIXL_ASSERT(operand.shift() != ROR); 1482 VIXL_ASSERT(is_uintn(rd.size() == kXRegSize ? kXRegSizeLog2 : kWRegSizeLog2, 1483 operand.shift_amount())); 1484 temps.Exclude(operand.reg()); 1485 Register temp = temps.AcquireSameSizeAs(rn); 1486 EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount()); 1487 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1488 } else if (operand.IsExtendedRegister()) { 1489 // Add/sub with carry (extended register). 1490 VIXL_ASSERT(operand.reg().size() <= rd.size()); 1491 // Add/sub extended supports a shift <= 4. We want to support exactly the 1492 // same modes. 1493 VIXL_ASSERT(operand.shift_amount() <= 4); 1494 VIXL_ASSERT(operand.reg().Is64Bits() || 1495 ((operand.extend() != UXTX) && (operand.extend() != SXTX))); 1496 temps.Exclude(operand.reg()); 1497 Register temp = temps.AcquireSameSizeAs(rn); 1498 EmitExtendShift(temp, operand.reg(), operand.extend(), 1499 operand.shift_amount()); 1500 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1501 } else { 1502 // The addressing mode is directly supported by the instruction. 1503 AddSubWithCarry(rd, rn, operand, S, op); 1504 } 1505 } 1506 1507 1508 #define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \ 1509 void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \ 1510 VIXL_ASSERT(allow_macro_instructions_); \ 1511 LoadStoreMacro(REG, addr, OP); \ 1512 } 1513 LS_MACRO_LIST(DEFINE_FUNCTION) 1514 #undef DEFINE_FUNCTION 1515 1516 1517 void MacroAssembler::LoadStoreMacro(const CPURegister& rt, 1518 const MemOperand& addr, 1519 LoadStoreOp op) { 1520 // Worst case is ldr/str pre/post index: 1521 // * 1 instruction for ldr/str 1522 // * up to 4 instructions to materialise the constant 1523 // * 1 instruction to update the base 1524 MacroEmissionCheckScope guard(this); 1525 1526 int64_t offset = addr.offset(); 1527 unsigned access_size = CalcLSDataSize(op); 1528 1529 // Check if an immediate offset fits in the immediate field of the 1530 // appropriate instruction. If not, emit two instructions to perform 1531 // the operation. 1532 if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, access_size) && 1533 !IsImmLSUnscaled(offset)) { 1534 // Immediate offset that can't be encoded using unsigned or unscaled 1535 // addressing modes. 1536 UseScratchRegisterScope temps(this); 1537 Register temp = temps.AcquireSameSizeAs(addr.base()); 1538 Mov(temp, addr.offset()); 1539 LoadStore(rt, MemOperand(addr.base(), temp), op); 1540 } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) { 1541 // Post-index beyond unscaled addressing range. 1542 LoadStore(rt, MemOperand(addr.base()), op); 1543 Add(addr.base(), addr.base(), Operand(offset)); 1544 } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) { 1545 // Pre-index beyond unscaled addressing range. 1546 Add(addr.base(), addr.base(), Operand(offset)); 1547 LoadStore(rt, MemOperand(addr.base()), op); 1548 } else { 1549 // Encodable in one load/store instruction. 1550 LoadStore(rt, addr, op); 1551 } 1552 } 1553 1554 1555 #define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \ 1556 void MacroAssembler::FN(const REGTYPE REG, \ 1557 const REGTYPE REG2, \ 1558 const MemOperand& addr) { \ 1559 VIXL_ASSERT(allow_macro_instructions_); \ 1560 LoadStorePairMacro(REG, REG2, addr, OP); \ 1561 } 1562 LSPAIR_MACRO_LIST(DEFINE_FUNCTION) 1563 #undef DEFINE_FUNCTION 1564 1565 void MacroAssembler::LoadStorePairMacro(const CPURegister& rt, 1566 const CPURegister& rt2, 1567 const MemOperand& addr, 1568 LoadStorePairOp op) { 1569 // TODO(all): Should we support register offset for load-store-pair? 1570 VIXL_ASSERT(!addr.IsRegisterOffset()); 1571 // Worst case is ldp/stp immediate: 1572 // * 1 instruction for ldp/stp 1573 // * up to 4 instructions to materialise the constant 1574 // * 1 instruction to update the base 1575 MacroEmissionCheckScope guard(this); 1576 1577 int64_t offset = addr.offset(); 1578 unsigned access_size = CalcLSPairDataSize(op); 1579 1580 // Check if the offset fits in the immediate field of the appropriate 1581 // instruction. If not, emit two instructions to perform the operation. 1582 if (IsImmLSPair(offset, access_size)) { 1583 // Encodable in one load/store pair instruction. 1584 LoadStorePair(rt, rt2, addr, op); 1585 } else { 1586 Register base = addr.base(); 1587 if (addr.IsImmediateOffset()) { 1588 UseScratchRegisterScope temps(this); 1589 Register temp = temps.AcquireSameSizeAs(base); 1590 Add(temp, base, offset); 1591 LoadStorePair(rt, rt2, MemOperand(temp), op); 1592 } else if (addr.IsPostIndex()) { 1593 LoadStorePair(rt, rt2, MemOperand(base), op); 1594 Add(base, base, offset); 1595 } else { 1596 VIXL_ASSERT(addr.IsPreIndex()); 1597 Add(base, base, offset); 1598 LoadStorePair(rt, rt2, MemOperand(base), op); 1599 } 1600 } 1601 } 1602 1603 1604 void MacroAssembler::Prfm(PrefetchOperation op, const MemOperand& addr) { 1605 MacroEmissionCheckScope guard(this); 1606 1607 // There are no pre- or post-index modes for prfm. 1608 VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsRegisterOffset()); 1609 1610 // The access size is implicitly 8 bytes for all prefetch operations. 1611 unsigned size = kXRegSizeInBytesLog2; 1612 1613 // Check if an immediate offset fits in the immediate field of the 1614 // appropriate instruction. If not, emit two instructions to perform 1615 // the operation. 1616 if (addr.IsImmediateOffset() && !IsImmLSScaled(addr.offset(), size) && 1617 !IsImmLSUnscaled(addr.offset())) { 1618 // Immediate offset that can't be encoded using unsigned or unscaled 1619 // addressing modes. 1620 UseScratchRegisterScope temps(this); 1621 Register temp = temps.AcquireSameSizeAs(addr.base()); 1622 Mov(temp, addr.offset()); 1623 Prefetch(op, MemOperand(addr.base(), temp)); 1624 } else { 1625 // Simple register-offsets are encodable in one instruction. 1626 Prefetch(op, addr); 1627 } 1628 } 1629 1630 1631 void MacroAssembler::Push(const CPURegister& src0, const CPURegister& src1, 1632 const CPURegister& src2, const CPURegister& src3) { 1633 VIXL_ASSERT(allow_macro_instructions_); 1634 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3)); 1635 VIXL_ASSERT(src0.IsValid()); 1636 1637 int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid(); 1638 int size = src0.SizeInBytes(); 1639 1640 PrepareForPush(count, size); 1641 PushHelper(count, size, src0, src1, src2, src3); 1642 } 1643 1644 1645 void MacroAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1, 1646 const CPURegister& dst2, const CPURegister& dst3) { 1647 // It is not valid to pop into the same register more than once in one 1648 // instruction, not even into the zero register. 1649 VIXL_ASSERT(allow_macro_instructions_); 1650 VIXL_ASSERT(!AreAliased(dst0, dst1, dst2, dst3)); 1651 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3)); 1652 VIXL_ASSERT(dst0.IsValid()); 1653 1654 int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid(); 1655 int size = dst0.SizeInBytes(); 1656 1657 PrepareForPop(count, size); 1658 PopHelper(count, size, dst0, dst1, dst2, dst3); 1659 } 1660 1661 1662 void MacroAssembler::PushCPURegList(CPURegList registers) { 1663 VIXL_ASSERT(!registers.Overlaps(*TmpList())); 1664 VIXL_ASSERT(!registers.Overlaps(*FPTmpList())); 1665 VIXL_ASSERT(allow_macro_instructions_); 1666 1667 int reg_size = registers.RegisterSizeInBytes(); 1668 PrepareForPush(registers.Count(), reg_size); 1669 1670 // Bump the stack pointer and store two registers at the bottom. 1671 int size = registers.TotalSizeInBytes(); 1672 const CPURegister& bottom_0 = registers.PopLowestIndex(); 1673 const CPURegister& bottom_1 = registers.PopLowestIndex(); 1674 if (bottom_0.IsValid() && bottom_1.IsValid()) { 1675 Stp(bottom_0, bottom_1, MemOperand(StackPointer(), -size, PreIndex)); 1676 } else if (bottom_0.IsValid()) { 1677 Str(bottom_0, MemOperand(StackPointer(), -size, PreIndex)); 1678 } 1679 1680 int offset = 2 * reg_size; 1681 while (!registers.IsEmpty()) { 1682 const CPURegister& src0 = registers.PopLowestIndex(); 1683 const CPURegister& src1 = registers.PopLowestIndex(); 1684 if (src1.IsValid()) { 1685 Stp(src0, src1, MemOperand(StackPointer(), offset)); 1686 } else { 1687 Str(src0, MemOperand(StackPointer(), offset)); 1688 } 1689 offset += 2 * reg_size; 1690 } 1691 } 1692 1693 1694 void MacroAssembler::PopCPURegList(CPURegList registers) { 1695 VIXL_ASSERT(!registers.Overlaps(*TmpList())); 1696 VIXL_ASSERT(!registers.Overlaps(*FPTmpList())); 1697 VIXL_ASSERT(allow_macro_instructions_); 1698 1699 int reg_size = registers.RegisterSizeInBytes(); 1700 PrepareForPop(registers.Count(), reg_size); 1701 1702 1703 int size = registers.TotalSizeInBytes(); 1704 const CPURegister& bottom_0 = registers.PopLowestIndex(); 1705 const CPURegister& bottom_1 = registers.PopLowestIndex(); 1706 1707 int offset = 2 * reg_size; 1708 while (!registers.IsEmpty()) { 1709 const CPURegister& dst0 = registers.PopLowestIndex(); 1710 const CPURegister& dst1 = registers.PopLowestIndex(); 1711 if (dst1.IsValid()) { 1712 Ldp(dst0, dst1, MemOperand(StackPointer(), offset)); 1713 } else { 1714 Ldr(dst0, MemOperand(StackPointer(), offset)); 1715 } 1716 offset += 2 * reg_size; 1717 } 1718 1719 // Load the two registers at the bottom and drop the stack pointer. 1720 if (bottom_0.IsValid() && bottom_1.IsValid()) { 1721 Ldp(bottom_0, bottom_1, MemOperand(StackPointer(), size, PostIndex)); 1722 } else if (bottom_0.IsValid()) { 1723 Ldr(bottom_0, MemOperand(StackPointer(), size, PostIndex)); 1724 } 1725 } 1726 1727 1728 void MacroAssembler::PushMultipleTimes(int count, Register src) { 1729 VIXL_ASSERT(allow_macro_instructions_); 1730 int size = src.SizeInBytes(); 1731 1732 PrepareForPush(count, size); 1733 // Push up to four registers at a time if possible because if the current 1734 // stack pointer is sp and the register size is 32, registers must be pushed 1735 // in blocks of four in order to maintain the 16-byte alignment for sp. 1736 while (count >= 4) { 1737 PushHelper(4, size, src, src, src, src); 1738 count -= 4; 1739 } 1740 if (count >= 2) { 1741 PushHelper(2, size, src, src, NoReg, NoReg); 1742 count -= 2; 1743 } 1744 if (count == 1) { 1745 PushHelper(1, size, src, NoReg, NoReg, NoReg); 1746 count -= 1; 1747 } 1748 VIXL_ASSERT(count == 0); 1749 } 1750 1751 1752 void MacroAssembler::PushHelper(int count, int size, 1753 const CPURegister& src0, 1754 const CPURegister& src1, 1755 const CPURegister& src2, 1756 const CPURegister& src3) { 1757 // Ensure that we don't unintentionally modify scratch or debug registers. 1758 // Worst case for size is 2 stp. 1759 InstructionAccurateScope scope(this, 2, 1760 InstructionAccurateScope::kMaximumSize); 1761 1762 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3)); 1763 VIXL_ASSERT(size == src0.SizeInBytes()); 1764 1765 // When pushing multiple registers, the store order is chosen such that 1766 // Push(a, b) is equivalent to Push(a) followed by Push(b). 1767 switch (count) { 1768 case 1: 1769 VIXL_ASSERT(src1.IsNone() && src2.IsNone() && src3.IsNone()); 1770 str(src0, MemOperand(StackPointer(), -1 * size, PreIndex)); 1771 break; 1772 case 2: 1773 VIXL_ASSERT(src2.IsNone() && src3.IsNone()); 1774 stp(src1, src0, MemOperand(StackPointer(), -2 * size, PreIndex)); 1775 break; 1776 case 3: 1777 VIXL_ASSERT(src3.IsNone()); 1778 stp(src2, src1, MemOperand(StackPointer(), -3 * size, PreIndex)); 1779 str(src0, MemOperand(StackPointer(), 2 * size)); 1780 break; 1781 case 4: 1782 // Skip over 4 * size, then fill in the gap. This allows four W registers 1783 // to be pushed using sp, whilst maintaining 16-byte alignment for sp at 1784 // all times. 1785 stp(src3, src2, MemOperand(StackPointer(), -4 * size, PreIndex)); 1786 stp(src1, src0, MemOperand(StackPointer(), 2 * size)); 1787 break; 1788 default: 1789 VIXL_UNREACHABLE(); 1790 } 1791 } 1792 1793 1794 void MacroAssembler::PopHelper(int count, int size, 1795 const CPURegister& dst0, 1796 const CPURegister& dst1, 1797 const CPURegister& dst2, 1798 const CPURegister& dst3) { 1799 // Ensure that we don't unintentionally modify scratch or debug registers. 1800 // Worst case for size is 2 ldp. 1801 InstructionAccurateScope scope(this, 2, 1802 InstructionAccurateScope::kMaximumSize); 1803 1804 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3)); 1805 VIXL_ASSERT(size == dst0.SizeInBytes()); 1806 1807 // When popping multiple registers, the load order is chosen such that 1808 // Pop(a, b) is equivalent to Pop(a) followed by Pop(b). 1809 switch (count) { 1810 case 1: 1811 VIXL_ASSERT(dst1.IsNone() && dst2.IsNone() && dst3.IsNone()); 1812 ldr(dst0, MemOperand(StackPointer(), 1 * size, PostIndex)); 1813 break; 1814 case 2: 1815 VIXL_ASSERT(dst2.IsNone() && dst3.IsNone()); 1816 ldp(dst0, dst1, MemOperand(StackPointer(), 2 * size, PostIndex)); 1817 break; 1818 case 3: 1819 VIXL_ASSERT(dst3.IsNone()); 1820 ldr(dst2, MemOperand(StackPointer(), 2 * size)); 1821 ldp(dst0, dst1, MemOperand(StackPointer(), 3 * size, PostIndex)); 1822 break; 1823 case 4: 1824 // Load the higher addresses first, then load the lower addresses and skip 1825 // the whole block in the second instruction. This allows four W registers 1826 // to be popped using sp, whilst maintaining 16-byte alignment for sp at 1827 // all times. 1828 ldp(dst2, dst3, MemOperand(StackPointer(), 2 * size)); 1829 ldp(dst0, dst1, MemOperand(StackPointer(), 4 * size, PostIndex)); 1830 break; 1831 default: 1832 VIXL_UNREACHABLE(); 1833 } 1834 } 1835 1836 1837 void MacroAssembler::PrepareForPush(int count, int size) { 1838 if (sp.Is(StackPointer())) { 1839 // If the current stack pointer is sp, then it must be aligned to 16 bytes 1840 // on entry and the total size of the specified registers must also be a 1841 // multiple of 16 bytes. 1842 VIXL_ASSERT((count * size) % 16 == 0); 1843 } else { 1844 // Even if the current stack pointer is not the system stack pointer (sp), 1845 // the system stack pointer will still be modified in order to comply with 1846 // ABI rules about accessing memory below the system stack pointer. 1847 BumpSystemStackPointer(count * size); 1848 } 1849 } 1850 1851 1852 void MacroAssembler::PrepareForPop(int count, int size) { 1853 USE(count, size); 1854 if (sp.Is(StackPointer())) { 1855 // If the current stack pointer is sp, then it must be aligned to 16 bytes 1856 // on entry and the total size of the specified registers must also be a 1857 // multiple of 16 bytes. 1858 VIXL_ASSERT((count * size) % 16 == 0); 1859 } 1860 } 1861 1862 void MacroAssembler::Poke(const Register& src, const Operand& offset) { 1863 VIXL_ASSERT(allow_macro_instructions_); 1864 if (offset.IsImmediate()) { 1865 VIXL_ASSERT(offset.immediate() >= 0); 1866 } 1867 1868 Str(src, MemOperand(StackPointer(), offset)); 1869 } 1870 1871 1872 void MacroAssembler::Peek(const Register& dst, const Operand& offset) { 1873 VIXL_ASSERT(allow_macro_instructions_); 1874 if (offset.IsImmediate()) { 1875 VIXL_ASSERT(offset.immediate() >= 0); 1876 } 1877 1878 Ldr(dst, MemOperand(StackPointer(), offset)); 1879 } 1880 1881 1882 void MacroAssembler::Claim(const Operand& size) { 1883 VIXL_ASSERT(allow_macro_instructions_); 1884 1885 if (size.IsZero()) { 1886 return; 1887 } 1888 1889 if (size.IsImmediate()) { 1890 VIXL_ASSERT(size.immediate() > 0); 1891 if (sp.Is(StackPointer())) { 1892 VIXL_ASSERT((size.immediate() % 16) == 0); 1893 } 1894 } 1895 1896 if (!sp.Is(StackPointer())) { 1897 BumpSystemStackPointer(size); 1898 } 1899 1900 Sub(StackPointer(), StackPointer(), size); 1901 } 1902 1903 1904 void MacroAssembler::Drop(const Operand& size) { 1905 VIXL_ASSERT(allow_macro_instructions_); 1906 1907 if (size.IsZero()) { 1908 return; 1909 } 1910 1911 if (size.IsImmediate()) { 1912 VIXL_ASSERT(size.immediate() > 0); 1913 if (sp.Is(StackPointer())) { 1914 VIXL_ASSERT((size.immediate() % 16) == 0); 1915 } 1916 } 1917 1918 Add(StackPointer(), StackPointer(), size); 1919 } 1920 1921 1922 void MacroAssembler::PushCalleeSavedRegisters() { 1923 // Ensure that the macro-assembler doesn't use any scratch registers. 1924 // 10 stp will be emitted. 1925 // TODO(all): Should we use GetCalleeSaved and SavedFP. 1926 InstructionAccurateScope scope(this, 10); 1927 1928 // This method must not be called unless the current stack pointer is sp. 1929 VIXL_ASSERT(sp.Is(StackPointer())); 1930 1931 MemOperand tos(sp, -2 * static_cast<int>(kXRegSizeInBytes), PreIndex); 1932 1933 stp(x29, x30, tos); 1934 stp(x27, x28, tos); 1935 stp(x25, x26, tos); 1936 stp(x23, x24, tos); 1937 stp(x21, x22, tos); 1938 stp(x19, x20, tos); 1939 1940 stp(d14, d15, tos); 1941 stp(d12, d13, tos); 1942 stp(d10, d11, tos); 1943 stp(d8, d9, tos); 1944 } 1945 1946 1947 void MacroAssembler::PopCalleeSavedRegisters() { 1948 // Ensure that the macro-assembler doesn't use any scratch registers. 1949 // 10 ldp will be emitted. 1950 // TODO(all): Should we use GetCalleeSaved and SavedFP. 1951 InstructionAccurateScope scope(this, 10); 1952 1953 // This method must not be called unless the current stack pointer is sp. 1954 VIXL_ASSERT(sp.Is(StackPointer())); 1955 1956 MemOperand tos(sp, 2 * kXRegSizeInBytes, PostIndex); 1957 1958 ldp(d8, d9, tos); 1959 ldp(d10, d11, tos); 1960 ldp(d12, d13, tos); 1961 ldp(d14, d15, tos); 1962 1963 ldp(x19, x20, tos); 1964 ldp(x21, x22, tos); 1965 ldp(x23, x24, tos); 1966 ldp(x25, x26, tos); 1967 ldp(x27, x28, tos); 1968 ldp(x29, x30, tos); 1969 } 1970 1971 void MacroAssembler::LoadCPURegList(CPURegList registers, 1972 const MemOperand& src) { 1973 LoadStoreCPURegListHelper(kLoad, registers, src); 1974 } 1975 1976 void MacroAssembler::StoreCPURegList(CPURegList registers, 1977 const MemOperand& dst) { 1978 LoadStoreCPURegListHelper(kStore, registers, dst); 1979 } 1980 1981 1982 void MacroAssembler::LoadStoreCPURegListHelper(LoadStoreCPURegListAction op, 1983 CPURegList registers, 1984 const MemOperand& mem) { 1985 // We do not handle pre-indexing or post-indexing. 1986 VIXL_ASSERT(!(mem.IsPreIndex() || mem.IsPostIndex())); 1987 VIXL_ASSERT(!registers.Overlaps(tmp_list_)); 1988 VIXL_ASSERT(!registers.Overlaps(fptmp_list_)); 1989 VIXL_ASSERT(!registers.IncludesAliasOf(sp)); 1990 1991 UseScratchRegisterScope temps(this); 1992 1993 MemOperand loc = BaseMemOperandForLoadStoreCPURegList(registers, 1994 mem, 1995 &temps); 1996 1997 while (registers.Count() >= 2) { 1998 const CPURegister& dst0 = registers.PopLowestIndex(); 1999 const CPURegister& dst1 = registers.PopLowestIndex(); 2000 if (op == kStore) { 2001 Stp(dst0, dst1, loc); 2002 } else { 2003 VIXL_ASSERT(op == kLoad); 2004 Ldp(dst0, dst1, loc); 2005 } 2006 loc.AddOffset(2 * registers.RegisterSizeInBytes()); 2007 } 2008 if (!registers.IsEmpty()) { 2009 if (op == kStore) { 2010 Str(registers.PopLowestIndex(), loc); 2011 } else { 2012 VIXL_ASSERT(op == kLoad); 2013 Ldr(registers.PopLowestIndex(), loc); 2014 } 2015 } 2016 } 2017 2018 MemOperand MacroAssembler::BaseMemOperandForLoadStoreCPURegList( 2019 const CPURegList& registers, 2020 const MemOperand& mem, 2021 UseScratchRegisterScope* scratch_scope) { 2022 // If necessary, pre-compute the base address for the accesses. 2023 if (mem.IsRegisterOffset()) { 2024 Register reg_base = scratch_scope->AcquireX(); 2025 ComputeAddress(reg_base, mem); 2026 return MemOperand(reg_base); 2027 2028 } else if (mem.IsImmediateOffset()) { 2029 int reg_size = registers.RegisterSizeInBytes(); 2030 int total_size = registers.TotalSizeInBytes(); 2031 int64_t min_offset = mem.offset(); 2032 int64_t max_offset = mem.offset() + std::max(0, total_size - 2 * reg_size); 2033 if ((registers.Count() >= 2) && 2034 (!Assembler::IsImmLSPair(min_offset, WhichPowerOf2(reg_size)) || 2035 !Assembler::IsImmLSPair(max_offset, WhichPowerOf2(reg_size)))) { 2036 Register reg_base = scratch_scope->AcquireX(); 2037 ComputeAddress(reg_base, mem); 2038 return MemOperand(reg_base); 2039 } 2040 } 2041 2042 return mem; 2043 } 2044 2045 void MacroAssembler::BumpSystemStackPointer(const Operand& space) { 2046 VIXL_ASSERT(!sp.Is(StackPointer())); 2047 // TODO: Several callers rely on this not using scratch registers, so we use 2048 // the assembler directly here. However, this means that large immediate 2049 // values of 'space' cannot be handled. 2050 InstructionAccurateScope scope(this, 1); 2051 sub(sp, StackPointer(), space); 2052 } 2053 2054 2055 // TODO(all): Fix printf for NEON registers, and resolve whether we should be 2056 // using FPRegister or VRegister here. 2057 2058 // This is the main Printf implementation. All callee-saved registers are 2059 // preserved, but NZCV and the caller-saved registers may be clobbered. 2060 void MacroAssembler::PrintfNoPreserve(const char * format, 2061 const CPURegister& arg0, 2062 const CPURegister& arg1, 2063 const CPURegister& arg2, 2064 const CPURegister& arg3) { 2065 // We cannot handle a caller-saved stack pointer. It doesn't make much sense 2066 // in most cases anyway, so this restriction shouldn't be too serious. 2067 VIXL_ASSERT(!kCallerSaved.IncludesAliasOf(StackPointer())); 2068 2069 // The provided arguments, and their proper PCS registers. 2070 CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3}; 2071 CPURegister pcs[kPrintfMaxArgCount]; 2072 2073 int arg_count = kPrintfMaxArgCount; 2074 2075 // The PCS varargs registers for printf. Note that x0 is used for the printf 2076 // format string. 2077 static const CPURegList kPCSVarargs = 2078 CPURegList(CPURegister::kRegister, kXRegSize, 1, arg_count); 2079 static const CPURegList kPCSVarargsFP = 2080 CPURegList(CPURegister::kVRegister, kDRegSize, 0, arg_count - 1); 2081 2082 // We can use caller-saved registers as scratch values, except for the 2083 // arguments and the PCS registers where they might need to go. 2084 UseScratchRegisterScope temps(this); 2085 temps.Include(kCallerSaved); 2086 temps.Include(kCallerSavedV); 2087 temps.Exclude(kPCSVarargs); 2088 temps.Exclude(kPCSVarargsFP); 2089 temps.Exclude(arg0, arg1, arg2, arg3); 2090 2091 // Copies of the arg lists that we can iterate through. 2092 CPURegList pcs_varargs = kPCSVarargs; 2093 CPURegList pcs_varargs_fp = kPCSVarargsFP; 2094 2095 // Place the arguments. There are lots of clever tricks and optimizations we 2096 // could use here, but Printf is a debug tool so instead we just try to keep 2097 // it simple: Move each input that isn't already in the right place to a 2098 // scratch register, then move everything back. 2099 for (unsigned i = 0; i < kPrintfMaxArgCount; i++) { 2100 // Work out the proper PCS register for this argument. 2101 if (args[i].IsRegister()) { 2102 pcs[i] = pcs_varargs.PopLowestIndex().X(); 2103 // We might only need a W register here. We need to know the size of the 2104 // argument so we can properly encode it for the simulator call. 2105 if (args[i].Is32Bits()) pcs[i] = pcs[i].W(); 2106 } else if (args[i].IsVRegister()) { 2107 // In C, floats are always cast to doubles for varargs calls. 2108 pcs[i] = pcs_varargs_fp.PopLowestIndex().D(); 2109 } else { 2110 VIXL_ASSERT(args[i].IsNone()); 2111 arg_count = i; 2112 break; 2113 } 2114 2115 // If the argument is already in the right place, leave it where it is. 2116 if (args[i].Aliases(pcs[i])) continue; 2117 2118 // Otherwise, if the argument is in a PCS argument register, allocate an 2119 // appropriate scratch register and then move it out of the way. 2120 if (kPCSVarargs.IncludesAliasOf(args[i]) || 2121 kPCSVarargsFP.IncludesAliasOf(args[i])) { 2122 if (args[i].IsRegister()) { 2123 Register old_arg = Register(args[i]); 2124 Register new_arg = temps.AcquireSameSizeAs(old_arg); 2125 Mov(new_arg, old_arg); 2126 args[i] = new_arg; 2127 } else { 2128 FPRegister old_arg = FPRegister(args[i]); 2129 FPRegister new_arg = temps.AcquireSameSizeAs(old_arg); 2130 Fmov(new_arg, old_arg); 2131 args[i] = new_arg; 2132 } 2133 } 2134 } 2135 2136 // Do a second pass to move values into their final positions and perform any 2137 // conversions that may be required. 2138 for (int i = 0; i < arg_count; i++) { 2139 VIXL_ASSERT(pcs[i].type() == args[i].type()); 2140 if (pcs[i].IsRegister()) { 2141 Mov(Register(pcs[i]), Register(args[i]), kDiscardForSameWReg); 2142 } else { 2143 VIXL_ASSERT(pcs[i].IsVRegister()); 2144 if (pcs[i].size() == args[i].size()) { 2145 Fmov(FPRegister(pcs[i]), FPRegister(args[i])); 2146 } else { 2147 Fcvt(FPRegister(pcs[i]), FPRegister(args[i])); 2148 } 2149 } 2150 } 2151 2152 // Load the format string into x0, as per the procedure-call standard. 2153 // 2154 // To make the code as portable as possible, the format string is encoded 2155 // directly in the instruction stream. It might be cleaner to encode it in a 2156 // literal pool, but since Printf is usually used for debugging, it is 2157 // beneficial for it to be minimally dependent on other features. 2158 temps.Exclude(x0); 2159 Label format_address; 2160 Adr(x0, &format_address); 2161 2162 // Emit the format string directly in the instruction stream. 2163 { 2164 BlockPoolsScope scope(this); 2165 // Data emitted: 2166 // branch 2167 // strlen(format) + 1 (includes null termination) 2168 // padding to next instruction 2169 // unreachable 2170 EmissionCheckScope guard( 2171 this, 2172 AlignUp(strlen(format) + 1, kInstructionSize) + 2 * kInstructionSize); 2173 Label after_data; 2174 B(&after_data); 2175 Bind(&format_address); 2176 EmitString(format); 2177 Unreachable(); 2178 Bind(&after_data); 2179 } 2180 2181 // We don't pass any arguments on the stack, but we still need to align the C 2182 // stack pointer to a 16-byte boundary for PCS compliance. 2183 if (!sp.Is(StackPointer())) { 2184 Bic(sp, StackPointer(), 0xf); 2185 } 2186 2187 // Actually call printf. This part needs special handling for the simulator, 2188 // since the system printf function will use a different instruction set and 2189 // the procedure-call standard will not be compatible. 2190 if (allow_simulator_instructions_) { 2191 InstructionAccurateScope scope(this, kPrintfLength / kInstructionSize); 2192 hlt(kPrintfOpcode); 2193 dc32(arg_count); // kPrintfArgCountOffset 2194 2195 // Determine the argument pattern. 2196 uint32_t arg_pattern_list = 0; 2197 for (int i = 0; i < arg_count; i++) { 2198 uint32_t arg_pattern; 2199 if (pcs[i].IsRegister()) { 2200 arg_pattern = pcs[i].Is32Bits() ? kPrintfArgW : kPrintfArgX; 2201 } else { 2202 VIXL_ASSERT(pcs[i].Is64Bits()); 2203 arg_pattern = kPrintfArgD; 2204 } 2205 VIXL_ASSERT(arg_pattern < (1 << kPrintfArgPatternBits)); 2206 arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i)); 2207 } 2208 dc32(arg_pattern_list); // kPrintfArgPatternListOffset 2209 } else { 2210 Register tmp = temps.AcquireX(); 2211 Mov(tmp, reinterpret_cast<uintptr_t>(printf)); 2212 Blr(tmp); 2213 } 2214 } 2215 2216 2217 void MacroAssembler::Printf(const char * format, 2218 CPURegister arg0, 2219 CPURegister arg1, 2220 CPURegister arg2, 2221 CPURegister arg3) { 2222 // We can only print sp if it is the current stack pointer. 2223 if (!sp.Is(StackPointer())) { 2224 VIXL_ASSERT(!sp.Aliases(arg0)); 2225 VIXL_ASSERT(!sp.Aliases(arg1)); 2226 VIXL_ASSERT(!sp.Aliases(arg2)); 2227 VIXL_ASSERT(!sp.Aliases(arg3)); 2228 } 2229 2230 // Make sure that the macro assembler doesn't try to use any of our arguments 2231 // as scratch registers. 2232 UseScratchRegisterScope exclude_all(this); 2233 exclude_all.ExcludeAll(); 2234 2235 // Preserve all caller-saved registers as well as NZCV. 2236 // If sp is the stack pointer, PushCPURegList asserts that the size of each 2237 // list is a multiple of 16 bytes. 2238 PushCPURegList(kCallerSaved); 2239 PushCPURegList(kCallerSavedV); 2240 2241 { UseScratchRegisterScope temps(this); 2242 // We can use caller-saved registers as scratch values (except for argN). 2243 temps.Include(kCallerSaved); 2244 temps.Include(kCallerSavedV); 2245 temps.Exclude(arg0, arg1, arg2, arg3); 2246 2247 // If any of the arguments are the current stack pointer, allocate a new 2248 // register for them, and adjust the value to compensate for pushing the 2249 // caller-saved registers. 2250 bool arg0_sp = StackPointer().Aliases(arg0); 2251 bool arg1_sp = StackPointer().Aliases(arg1); 2252 bool arg2_sp = StackPointer().Aliases(arg2); 2253 bool arg3_sp = StackPointer().Aliases(arg3); 2254 if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) { 2255 // Allocate a register to hold the original stack pointer value, to pass 2256 // to PrintfNoPreserve as an argument. 2257 Register arg_sp = temps.AcquireX(); 2258 Add(arg_sp, StackPointer(), 2259 kCallerSaved.TotalSizeInBytes() + kCallerSavedV.TotalSizeInBytes()); 2260 if (arg0_sp) arg0 = Register(arg_sp.code(), arg0.size()); 2261 if (arg1_sp) arg1 = Register(arg_sp.code(), arg1.size()); 2262 if (arg2_sp) arg2 = Register(arg_sp.code(), arg2.size()); 2263 if (arg3_sp) arg3 = Register(arg_sp.code(), arg3.size()); 2264 } 2265 2266 // Preserve NZCV. 2267 Register tmp = temps.AcquireX(); 2268 Mrs(tmp, NZCV); 2269 Push(tmp, xzr); 2270 temps.Release(tmp); 2271 2272 PrintfNoPreserve(format, arg0, arg1, arg2, arg3); 2273 2274 // Restore NZCV. 2275 tmp = temps.AcquireX(); 2276 Pop(xzr, tmp); 2277 Msr(NZCV, tmp); 2278 temps.Release(tmp); 2279 } 2280 2281 PopCPURegList(kCallerSavedV); 2282 PopCPURegList(kCallerSaved); 2283 } 2284 2285 void MacroAssembler::Trace(TraceParameters parameters, TraceCommand command) { 2286 VIXL_ASSERT(allow_macro_instructions_); 2287 2288 if (allow_simulator_instructions_) { 2289 // The arguments to the trace pseudo instruction need to be contiguous in 2290 // memory, so make sure we don't try to emit a literal pool. 2291 InstructionAccurateScope scope(this, kTraceLength / kInstructionSize); 2292 2293 Label start; 2294 bind(&start); 2295 2296 // Refer to simulator-a64.h for a description of the marker and its 2297 // arguments. 2298 hlt(kTraceOpcode); 2299 2300 VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceParamsOffset); 2301 dc32(parameters); 2302 2303 VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceCommandOffset); 2304 dc32(command); 2305 } else { 2306 // Emit nothing on real hardware. 2307 USE(parameters, command); 2308 } 2309 } 2310 2311 2312 void MacroAssembler::Log(TraceParameters parameters) { 2313 VIXL_ASSERT(allow_macro_instructions_); 2314 2315 if (allow_simulator_instructions_) { 2316 // The arguments to the log pseudo instruction need to be contiguous in 2317 // memory, so make sure we don't try to emit a literal pool. 2318 InstructionAccurateScope scope(this, kLogLength / kInstructionSize); 2319 2320 Label start; 2321 bind(&start); 2322 2323 // Refer to simulator-a64.h for a description of the marker and its 2324 // arguments. 2325 hlt(kLogOpcode); 2326 2327 VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kLogParamsOffset); 2328 dc32(parameters); 2329 } else { 2330 // Emit nothing on real hardware. 2331 USE(parameters); 2332 } 2333 } 2334 2335 2336 void MacroAssembler::EnableInstrumentation() { 2337 VIXL_ASSERT(!isprint(InstrumentStateEnable)); 2338 InstructionAccurateScope scope(this, 1); 2339 movn(xzr, InstrumentStateEnable); 2340 } 2341 2342 2343 void MacroAssembler::DisableInstrumentation() { 2344 VIXL_ASSERT(!isprint(InstrumentStateDisable)); 2345 InstructionAccurateScope scope(this, 1); 2346 movn(xzr, InstrumentStateDisable); 2347 } 2348 2349 2350 void MacroAssembler::AnnotateInstrumentation(const char* marker_name) { 2351 VIXL_ASSERT(strlen(marker_name) == 2); 2352 2353 // We allow only printable characters in the marker names. Unprintable 2354 // characters are reserved for controlling features of the instrumentation. 2355 VIXL_ASSERT(isprint(marker_name[0]) && isprint(marker_name[1])); 2356 2357 InstructionAccurateScope scope(this, 1); 2358 movn(xzr, (marker_name[1] << 8) | marker_name[0]); 2359 } 2360 2361 2362 void UseScratchRegisterScope::Open(MacroAssembler* masm) { 2363 VIXL_ASSERT(!initialised_); 2364 available_ = masm->TmpList(); 2365 availablefp_ = masm->FPTmpList(); 2366 old_available_ = available_->list(); 2367 old_availablefp_ = availablefp_->list(); 2368 VIXL_ASSERT(available_->type() == CPURegister::kRegister); 2369 VIXL_ASSERT(availablefp_->type() == CPURegister::kVRegister); 2370 #ifdef VIXL_DEBUG 2371 initialised_ = true; 2372 #endif 2373 } 2374 2375 2376 void UseScratchRegisterScope::Close() { 2377 if (available_) { 2378 available_->set_list(old_available_); 2379 available_ = NULL; 2380 } 2381 if (availablefp_) { 2382 availablefp_->set_list(old_availablefp_); 2383 availablefp_ = NULL; 2384 } 2385 #ifdef VIXL_DEBUG 2386 initialised_ = false; 2387 #endif 2388 } 2389 2390 2391 UseScratchRegisterScope::UseScratchRegisterScope(MacroAssembler* masm) { 2392 #ifdef VIXL_DEBUG 2393 initialised_ = false; 2394 #endif 2395 Open(masm); 2396 } 2397 2398 // This allows deferred (and optional) initialisation of the scope. 2399 UseScratchRegisterScope::UseScratchRegisterScope() 2400 : available_(NULL), availablefp_(NULL), 2401 old_available_(0), old_availablefp_(0) { 2402 #ifdef VIXL_DEBUG 2403 initialised_ = false; 2404 #endif 2405 } 2406 2407 UseScratchRegisterScope::~UseScratchRegisterScope() { 2408 Close(); 2409 } 2410 2411 2412 bool UseScratchRegisterScope::IsAvailable(const CPURegister& reg) const { 2413 return available_->IncludesAliasOf(reg) || availablefp_->IncludesAliasOf(reg); 2414 } 2415 2416 2417 Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) { 2418 int code = AcquireNextAvailable(available_).code(); 2419 return Register(code, reg.size()); 2420 } 2421 2422 2423 FPRegister UseScratchRegisterScope::AcquireSameSizeAs(const FPRegister& reg) { 2424 int code = AcquireNextAvailable(availablefp_).code(); 2425 return FPRegister(code, reg.size()); 2426 } 2427 2428 2429 void UseScratchRegisterScope::Release(const CPURegister& reg) { 2430 VIXL_ASSERT(initialised_); 2431 if (reg.IsRegister()) { 2432 ReleaseByCode(available_, reg.code()); 2433 } else if (reg.IsFPRegister()) { 2434 ReleaseByCode(availablefp_, reg.code()); 2435 } else { 2436 VIXL_ASSERT(reg.IsNone()); 2437 } 2438 } 2439 2440 2441 void UseScratchRegisterScope::Include(const CPURegList& list) { 2442 VIXL_ASSERT(initialised_); 2443 if (list.type() == CPURegister::kRegister) { 2444 // Make sure that neither sp nor xzr are included the list. 2445 IncludeByRegList(available_, list.list() & ~(xzr.Bit() | sp.Bit())); 2446 } else { 2447 VIXL_ASSERT(list.type() == CPURegister::kVRegister); 2448 IncludeByRegList(availablefp_, list.list()); 2449 } 2450 } 2451 2452 2453 void UseScratchRegisterScope::Include(const Register& reg1, 2454 const Register& reg2, 2455 const Register& reg3, 2456 const Register& reg4) { 2457 VIXL_ASSERT(initialised_); 2458 RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2459 // Make sure that neither sp nor xzr are included the list. 2460 include &= ~(xzr.Bit() | sp.Bit()); 2461 2462 IncludeByRegList(available_, include); 2463 } 2464 2465 2466 void UseScratchRegisterScope::Include(const FPRegister& reg1, 2467 const FPRegister& reg2, 2468 const FPRegister& reg3, 2469 const FPRegister& reg4) { 2470 RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2471 IncludeByRegList(availablefp_, include); 2472 } 2473 2474 2475 void UseScratchRegisterScope::Exclude(const CPURegList& list) { 2476 if (list.type() == CPURegister::kRegister) { 2477 ExcludeByRegList(available_, list.list()); 2478 } else { 2479 VIXL_ASSERT(list.type() == CPURegister::kVRegister); 2480 ExcludeByRegList(availablefp_, list.list()); 2481 } 2482 } 2483 2484 2485 void UseScratchRegisterScope::Exclude(const Register& reg1, 2486 const Register& reg2, 2487 const Register& reg3, 2488 const Register& reg4) { 2489 RegList exclude = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2490 ExcludeByRegList(available_, exclude); 2491 } 2492 2493 2494 void UseScratchRegisterScope::Exclude(const FPRegister& reg1, 2495 const FPRegister& reg2, 2496 const FPRegister& reg3, 2497 const FPRegister& reg4) { 2498 RegList excludefp = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2499 ExcludeByRegList(availablefp_, excludefp); 2500 } 2501 2502 2503 void UseScratchRegisterScope::Exclude(const CPURegister& reg1, 2504 const CPURegister& reg2, 2505 const CPURegister& reg3, 2506 const CPURegister& reg4) { 2507 RegList exclude = 0; 2508 RegList excludefp = 0; 2509 2510 const CPURegister regs[] = {reg1, reg2, reg3, reg4}; 2511 2512 for (unsigned i = 0; i < (sizeof(regs) / sizeof(regs[0])); i++) { 2513 if (regs[i].IsRegister()) { 2514 exclude |= regs[i].Bit(); 2515 } else if (regs[i].IsFPRegister()) { 2516 excludefp |= regs[i].Bit(); 2517 } else { 2518 VIXL_ASSERT(regs[i].IsNone()); 2519 } 2520 } 2521 2522 ExcludeByRegList(available_, exclude); 2523 ExcludeByRegList(availablefp_, excludefp); 2524 } 2525 2526 2527 void UseScratchRegisterScope::ExcludeAll() { 2528 ExcludeByRegList(available_, available_->list()); 2529 ExcludeByRegList(availablefp_, availablefp_->list()); 2530 } 2531 2532 2533 CPURegister UseScratchRegisterScope::AcquireNextAvailable( 2534 CPURegList* available) { 2535 VIXL_CHECK(!available->IsEmpty()); 2536 CPURegister result = available->PopLowestIndex(); 2537 VIXL_ASSERT(!AreAliased(result, xzr, sp)); 2538 return result; 2539 } 2540 2541 2542 void UseScratchRegisterScope::ReleaseByCode(CPURegList* available, int code) { 2543 ReleaseByRegList(available, static_cast<RegList>(1) << code); 2544 } 2545 2546 2547 void UseScratchRegisterScope::ReleaseByRegList(CPURegList* available, 2548 RegList regs) { 2549 available->set_list(available->list() | regs); 2550 } 2551 2552 2553 void UseScratchRegisterScope::IncludeByRegList(CPURegList* available, 2554 RegList regs) { 2555 available->set_list(available->list() | regs); 2556 } 2557 2558 2559 void UseScratchRegisterScope::ExcludeByRegList(CPURegList* available, 2560 RegList exclude) { 2561 available->set_list(available->list() & ~exclude); 2562 } 2563 2564 } // namespace vixl 2565