1 // Copyright 2015, VIXL authors 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // * Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // * Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // * Neither the name of ARM Limited nor the names of its contributors may be 13 // used to endorse or promote products derived from this software without 14 // specific prior written permission. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27 #include <cctype> 28 29 #include "macro-assembler-aarch64.h" 30 31 namespace vixl { 32 namespace aarch64 { 33 34 35 void Pool::Release() { 36 if (--monitor_ == 0) { 37 // Ensure the pool has not been blocked for too long. 38 VIXL_ASSERT(masm_->GetCursorOffset() < checkpoint_); 39 } 40 } 41 42 43 void Pool::SetNextCheckpoint(ptrdiff_t checkpoint) { 44 masm_->checkpoint_ = std::min(masm_->checkpoint_, checkpoint); 45 checkpoint_ = checkpoint; 46 } 47 48 49 LiteralPool::LiteralPool(MacroAssembler* masm) 50 : Pool(masm), 51 size_(0), 52 first_use_(-1), 53 recommended_checkpoint_(kNoCheckpointRequired) {} 54 55 56 LiteralPool::~LiteralPool() { 57 VIXL_ASSERT(IsEmpty()); 58 VIXL_ASSERT(!IsBlocked()); 59 for (std::vector<RawLiteral*>::iterator it = deleted_on_destruction_.begin(); 60 it != deleted_on_destruction_.end(); 61 it++) { 62 delete *it; 63 } 64 } 65 66 67 void LiteralPool::Reset() { 68 std::vector<RawLiteral *>::iterator it, end; 69 for (it = entries_.begin(), end = entries_.end(); it != end; ++it) { 70 RawLiteral* literal = *it; 71 if (literal->deletion_policy_ == RawLiteral::kDeletedOnPlacementByPool) { 72 delete literal; 73 } 74 } 75 entries_.clear(); 76 size_ = 0; 77 first_use_ = -1; 78 Pool::Reset(); 79 recommended_checkpoint_ = kNoCheckpointRequired; 80 } 81 82 83 void LiteralPool::CheckEmitFor(size_t amount, EmitOption option) { 84 if (IsEmpty() || IsBlocked()) return; 85 86 ptrdiff_t distance = masm_->GetCursorOffset() + amount - first_use_; 87 if (distance >= kRecommendedLiteralPoolRange) { 88 Emit(option); 89 } 90 } 91 92 93 void LiteralPool::CheckEmitForBranch(size_t range) { 94 if (IsEmpty() || IsBlocked()) return; 95 if (GetMaxSize() >= range) Emit(); 96 } 97 98 // We use a subclass to access the protected `ExactAssemblyScope` constructor 99 // giving us control over the pools. This allows us to use this scope within 100 // code emitting pools without creating a circular dependency. 101 // We keep the constructor private to restrict usage of this helper class. 102 class ExactAssemblyScopeWithoutPoolsCheck : public ExactAssemblyScope { 103 private: 104 ExactAssemblyScopeWithoutPoolsCheck(MacroAssembler* masm, size_t size) 105 : ExactAssemblyScope(masm, 106 size, 107 ExactAssemblyScope::kExactSize, 108 ExactAssemblyScope::kIgnorePools) {} 109 110 friend void LiteralPool::Emit(LiteralPool::EmitOption); 111 friend void VeneerPool::Emit(VeneerPool::EmitOption, size_t); 112 }; 113 114 115 void LiteralPool::Emit(EmitOption option) { 116 // There is an issue if we are asked to emit a blocked or empty pool. 117 VIXL_ASSERT(!IsBlocked()); 118 VIXL_ASSERT(!IsEmpty()); 119 120 size_t pool_size = GetSize(); 121 size_t emit_size = pool_size; 122 if (option == kBranchRequired) emit_size += kInstructionSize; 123 Label end_of_pool; 124 125 VIXL_ASSERT(emit_size % kInstructionSize == 0); 126 { 127 CodeBufferCheckScope guard(masm_, 128 emit_size, 129 CodeBufferCheckScope::kCheck, 130 CodeBufferCheckScope::kExactSize); 131 #ifdef VIXL_DEBUG 132 // Also explicitly disallow usage of the `MacroAssembler` here. 133 masm_->SetAllowMacroInstructions(false); 134 #endif 135 if (option == kBranchRequired) { 136 ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize); 137 masm_->b(&end_of_pool); 138 } 139 140 { 141 // Marker indicating the size of the literal pool in 32-bit words. 142 VIXL_ASSERT((pool_size % kWRegSizeInBytes) == 0); 143 ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize); 144 masm_->ldr(xzr, static_cast<int>(pool_size / kWRegSizeInBytes)); 145 } 146 147 // Now populate the literal pool. 148 std::vector<RawLiteral *>::iterator it, end; 149 for (it = entries_.begin(), end = entries_.end(); it != end; ++it) { 150 VIXL_ASSERT((*it)->IsUsed()); 151 masm_->place(*it); 152 } 153 154 if (option == kBranchRequired) masm_->bind(&end_of_pool); 155 #ifdef VIXL_DEBUG 156 masm_->SetAllowMacroInstructions(true); 157 #endif 158 } 159 160 Reset(); 161 } 162 163 164 void LiteralPool::AddEntry(RawLiteral* literal) { 165 // A literal must be registered immediately before its first use. Here we 166 // cannot control that it is its first use, but we check no code has been 167 // emitted since its last use. 168 VIXL_ASSERT(masm_->GetCursorOffset() == literal->GetLastUse()); 169 170 UpdateFirstUse(masm_->GetCursorOffset()); 171 VIXL_ASSERT(masm_->GetCursorOffset() >= first_use_); 172 entries_.push_back(literal); 173 size_ += literal->GetSize(); 174 } 175 176 177 void LiteralPool::UpdateFirstUse(ptrdiff_t use_position) { 178 first_use_ = std::min(first_use_, use_position); 179 if (first_use_ == -1) { 180 first_use_ = use_position; 181 SetNextRecommendedCheckpoint(GetNextRecommendedCheckpoint()); 182 SetNextCheckpoint(first_use_ + Instruction::kLoadLiteralRange); 183 } else { 184 VIXL_ASSERT(use_position > first_use_); 185 } 186 } 187 188 189 void VeneerPool::Reset() { 190 Pool::Reset(); 191 unresolved_branches_.Reset(); 192 } 193 194 195 void VeneerPool::Release() { 196 if (--monitor_ == 0) { 197 VIXL_ASSERT(IsEmpty() || 198 masm_->GetCursorOffset() < 199 unresolved_branches_.GetFirstLimit()); 200 } 201 } 202 203 204 void VeneerPool::RegisterUnresolvedBranch(ptrdiff_t branch_pos, 205 Label* label, 206 ImmBranchType branch_type) { 207 VIXL_ASSERT(!label->IsBound()); 208 BranchInfo branch_info = BranchInfo(branch_pos, label, branch_type); 209 unresolved_branches_.insert(branch_info); 210 UpdateNextCheckPoint(); 211 // TODO: In debug mode register the label with the assembler to make sure it 212 // is bound with masm Bind and not asm bind. 213 } 214 215 216 void VeneerPool::DeleteUnresolvedBranchInfoForLabel(Label* label) { 217 if (IsEmpty()) { 218 VIXL_ASSERT(checkpoint_ == kNoCheckpointRequired); 219 return; 220 } 221 222 if (label->IsLinked()) { 223 Label::LabelLinksIterator links_it(label); 224 for (; !links_it.Done(); links_it.Advance()) { 225 ptrdiff_t link_offset = *links_it.Current(); 226 Instruction* link = masm_->GetInstructionAt(link_offset); 227 228 // ADR instructions are not handled. 229 if (BranchTypeUsesVeneers(link->GetBranchType())) { 230 BranchInfo branch_info(link_offset, label, link->GetBranchType()); 231 unresolved_branches_.erase(branch_info); 232 } 233 } 234 } 235 236 UpdateNextCheckPoint(); 237 } 238 239 240 bool VeneerPool::ShouldEmitVeneer(int64_t first_unreacheable_pc, 241 size_t amount) { 242 ptrdiff_t offset = 243 kPoolNonVeneerCodeSize + amount + GetMaxSize() + GetOtherPoolsMaxSize(); 244 return (masm_->GetCursorOffset() + offset) > first_unreacheable_pc; 245 } 246 247 248 void VeneerPool::CheckEmitFor(size_t amount, EmitOption option) { 249 if (IsEmpty()) return; 250 251 VIXL_ASSERT(masm_->GetCursorOffset() + kPoolNonVeneerCodeSize < 252 unresolved_branches_.GetFirstLimit()); 253 254 if (IsBlocked()) return; 255 256 if (ShouldEmitVeneers(amount)) { 257 Emit(option, amount); 258 } else { 259 UpdateNextCheckPoint(); 260 } 261 } 262 263 264 void VeneerPool::Emit(EmitOption option, size_t amount) { 265 // There is an issue if we are asked to emit a blocked or empty pool. 266 VIXL_ASSERT(!IsBlocked()); 267 VIXL_ASSERT(!IsEmpty()); 268 269 Label end; 270 if (option == kBranchRequired) { 271 ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize); 272 masm_->b(&end); 273 } 274 275 // We want to avoid generating veneer pools too often, so generate veneers for 276 // branches that don't immediately require a veneer but will soon go out of 277 // range. 278 static const size_t kVeneerEmissionMargin = 1 * KBytes; 279 280 for (BranchInfoSetIterator it(&unresolved_branches_); !it.Done();) { 281 BranchInfo* branch_info = it.Current(); 282 if (ShouldEmitVeneer(branch_info->first_unreacheable_pc_, 283 amount + kVeneerEmissionMargin)) { 284 CodeBufferCheckScope scope(masm_, 285 kVeneerCodeSize, 286 CodeBufferCheckScope::kCheck, 287 CodeBufferCheckScope::kExactSize); 288 ptrdiff_t branch_pos = branch_info->pc_offset_; 289 Instruction* branch = masm_->GetInstructionAt(branch_pos); 290 Label* label = branch_info->label_; 291 292 // Patch the branch to point to the current position, and emit a branch 293 // to the label. 294 Instruction* veneer = masm_->GetCursorAddress<Instruction*>(); 295 branch->SetImmPCOffsetTarget(veneer); 296 { 297 ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize); 298 masm_->b(label); 299 } 300 301 // Update the label. The branch patched does not point to it any longer. 302 label->DeleteLink(branch_pos); 303 304 it.DeleteCurrentAndAdvance(); 305 } else { 306 it.AdvanceToNextType(); 307 } 308 } 309 310 UpdateNextCheckPoint(); 311 312 masm_->bind(&end); 313 } 314 315 316 MacroAssembler::MacroAssembler(PositionIndependentCodeOption pic) 317 : Assembler(pic), 318 #ifdef VIXL_DEBUG 319 allow_macro_instructions_(true), 320 #endif 321 generate_simulator_code_(VIXL_AARCH64_GENERATE_SIMULATOR_CODE), 322 sp_(sp), 323 tmp_list_(ip0, ip1), 324 fptmp_list_(d31), 325 current_scratch_scope_(NULL), 326 literal_pool_(this), 327 veneer_pool_(this), 328 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 329 checkpoint_ = GetNextCheckPoint(); 330 #ifndef VIXL_DEBUG 331 USE(allow_macro_instructions_); 332 #endif 333 } 334 335 336 MacroAssembler::MacroAssembler(size_t capacity, 337 PositionIndependentCodeOption pic) 338 : Assembler(capacity, pic), 339 #ifdef VIXL_DEBUG 340 allow_macro_instructions_(true), 341 #endif 342 generate_simulator_code_(VIXL_AARCH64_GENERATE_SIMULATOR_CODE), 343 sp_(sp), 344 tmp_list_(ip0, ip1), 345 fptmp_list_(d31), 346 current_scratch_scope_(NULL), 347 literal_pool_(this), 348 veneer_pool_(this), 349 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 350 checkpoint_ = GetNextCheckPoint(); 351 } 352 353 354 MacroAssembler::MacroAssembler(byte* buffer, 355 size_t capacity, 356 PositionIndependentCodeOption pic) 357 : Assembler(buffer, capacity, pic), 358 #ifdef VIXL_DEBUG 359 allow_macro_instructions_(true), 360 #endif 361 generate_simulator_code_(VIXL_AARCH64_GENERATE_SIMULATOR_CODE), 362 sp_(sp), 363 tmp_list_(ip0, ip1), 364 fptmp_list_(d31), 365 current_scratch_scope_(NULL), 366 literal_pool_(this), 367 veneer_pool_(this), 368 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 369 checkpoint_ = GetNextCheckPoint(); 370 } 371 372 373 MacroAssembler::~MacroAssembler() {} 374 375 376 void MacroAssembler::Reset() { 377 Assembler::Reset(); 378 379 VIXL_ASSERT(!literal_pool_.IsBlocked()); 380 literal_pool_.Reset(); 381 veneer_pool_.Reset(); 382 383 checkpoint_ = GetNextCheckPoint(); 384 } 385 386 387 void MacroAssembler::FinalizeCode(FinalizeOption option) { 388 if (!literal_pool_.IsEmpty()) { 389 // The user may decide to emit more code after Finalize, emit a branch if 390 // that's the case. 391 literal_pool_.Emit(option == kUnreachable ? Pool::kNoBranchRequired 392 : Pool::kBranchRequired); 393 } 394 VIXL_ASSERT(veneer_pool_.IsEmpty()); 395 396 Assembler::FinalizeCode(); 397 } 398 399 400 void MacroAssembler::CheckEmitFor(size_t amount) { 401 CheckEmitPoolsFor(amount); 402 GetBuffer()->EnsureSpaceFor(amount); 403 } 404 405 406 void MacroAssembler::CheckEmitPoolsFor(size_t amount) { 407 literal_pool_.CheckEmitFor(amount); 408 veneer_pool_.CheckEmitFor(amount); 409 checkpoint_ = GetNextCheckPoint(); 410 } 411 412 413 int MacroAssembler::MoveImmediateHelper(MacroAssembler* masm, 414 const Register& rd, 415 uint64_t imm) { 416 bool emit_code = (masm != NULL); 417 VIXL_ASSERT(IsUint32(imm) || IsInt32(imm) || rd.Is64Bits()); 418 // The worst case for size is mov 64-bit immediate to sp: 419 // * up to 4 instructions to materialise the constant 420 // * 1 instruction to move to sp 421 MacroEmissionCheckScope guard(masm); 422 423 // Immediates on Aarch64 can be produced using an initial value, and zero to 424 // three move keep operations. 425 // 426 // Initial values can be generated with: 427 // 1. 64-bit move zero (movz). 428 // 2. 32-bit move inverted (movn). 429 // 3. 64-bit move inverted. 430 // 4. 32-bit orr immediate. 431 // 5. 64-bit orr immediate. 432 // Move-keep may then be used to modify each of the 16-bit half words. 433 // 434 // The code below supports all five initial value generators, and 435 // applying move-keep operations to move-zero and move-inverted initial 436 // values. 437 438 // Try to move the immediate in one instruction, and if that fails, switch to 439 // using multiple instructions. 440 if (OneInstrMoveImmediateHelper(masm, rd, imm)) { 441 return 1; 442 } else { 443 int instruction_count = 0; 444 unsigned reg_size = rd.GetSizeInBits(); 445 446 // Generic immediate case. Imm will be represented by 447 // [imm3, imm2, imm1, imm0], where each imm is 16 bits. 448 // A move-zero or move-inverted is generated for the first non-zero or 449 // non-0xffff immX, and a move-keep for subsequent non-zero immX. 450 451 uint64_t ignored_halfword = 0; 452 bool invert_move = false; 453 // If the number of 0xffff halfwords is greater than the number of 0x0000 454 // halfwords, it's more efficient to use move-inverted. 455 if (CountClearHalfWords(~imm, reg_size) > 456 CountClearHalfWords(imm, reg_size)) { 457 ignored_halfword = 0xffff; 458 invert_move = true; 459 } 460 461 // Mov instructions can't move values into the stack pointer, so set up a 462 // temporary register, if needed. 463 UseScratchRegisterScope temps; 464 Register temp; 465 if (emit_code) { 466 temps.Open(masm); 467 temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd; 468 } 469 470 // Iterate through the halfwords. Use movn/movz for the first non-ignored 471 // halfword, and movk for subsequent halfwords. 472 VIXL_ASSERT((reg_size % 16) == 0); 473 bool first_mov_done = false; 474 for (unsigned i = 0; i < (reg_size / 16); i++) { 475 uint64_t imm16 = (imm >> (16 * i)) & 0xffff; 476 if (imm16 != ignored_halfword) { 477 if (!first_mov_done) { 478 if (invert_move) { 479 if (emit_code) masm->movn(temp, ~imm16 & 0xffff, 16 * i); 480 instruction_count++; 481 } else { 482 if (emit_code) masm->movz(temp, imm16, 16 * i); 483 instruction_count++; 484 } 485 first_mov_done = true; 486 } else { 487 // Construct a wider constant. 488 if (emit_code) masm->movk(temp, imm16, 16 * i); 489 instruction_count++; 490 } 491 } 492 } 493 494 VIXL_ASSERT(first_mov_done); 495 496 // Move the temporary if the original destination register was the stack 497 // pointer. 498 if (rd.IsSP()) { 499 if (emit_code) masm->mov(rd, temp); 500 instruction_count++; 501 } 502 return instruction_count; 503 } 504 } 505 506 507 bool MacroAssembler::OneInstrMoveImmediateHelper(MacroAssembler* masm, 508 const Register& dst, 509 int64_t imm) { 510 bool emit_code = masm != NULL; 511 unsigned n, imm_s, imm_r; 512 int reg_size = dst.GetSizeInBits(); 513 514 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) { 515 // Immediate can be represented in a move zero instruction. Movz can't write 516 // to the stack pointer. 517 if (emit_code) { 518 masm->movz(dst, imm); 519 } 520 return true; 521 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) { 522 // Immediate can be represented in a move negative instruction. Movn can't 523 // write to the stack pointer. 524 if (emit_code) { 525 masm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask)); 526 } 527 return true; 528 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) { 529 // Immediate can be represented in a logical orr instruction. 530 VIXL_ASSERT(!dst.IsZero()); 531 if (emit_code) { 532 masm->LogicalImmediate(dst, 533 AppropriateZeroRegFor(dst), 534 n, 535 imm_s, 536 imm_r, 537 ORR); 538 } 539 return true; 540 } 541 return false; 542 } 543 544 545 void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) { 546 VIXL_ASSERT((reg.Is(NoReg) || (type >= kBranchTypeFirstUsingReg)) && 547 ((bit == -1) || (type >= kBranchTypeFirstUsingBit))); 548 if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) { 549 B(static_cast<Condition>(type), label); 550 } else { 551 switch (type) { 552 case always: 553 B(label); 554 break; 555 case never: 556 break; 557 case reg_zero: 558 Cbz(reg, label); 559 break; 560 case reg_not_zero: 561 Cbnz(reg, label); 562 break; 563 case reg_bit_clear: 564 Tbz(reg, bit, label); 565 break; 566 case reg_bit_set: 567 Tbnz(reg, bit, label); 568 break; 569 default: 570 VIXL_UNREACHABLE(); 571 } 572 } 573 } 574 575 576 void MacroAssembler::B(Label* label) { 577 // We don't need to check the size of the literal pool, because the size of 578 // the literal pool is already bounded by the literal range, which is smaller 579 // than the range of this branch. 580 VIXL_ASSERT(Instruction::GetImmBranchForwardRange(UncondBranchType) > 581 Instruction::kLoadLiteralRange); 582 SingleEmissionCheckScope guard(this); 583 b(label); 584 } 585 586 587 void MacroAssembler::B(Label* label, Condition cond) { 588 // We don't need to check the size of the literal pool, because the size of 589 // the literal pool is already bounded by the literal range, which is smaller 590 // than the range of this branch. 591 VIXL_ASSERT(Instruction::GetImmBranchForwardRange(CondBranchType) > 592 Instruction::kLoadLiteralRange); 593 VIXL_ASSERT(allow_macro_instructions_); 594 VIXL_ASSERT((cond != al) && (cond != nv)); 595 EmissionCheckScope guard(this, 2 * kInstructionSize); 596 597 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 598 Label done; 599 b(&done, InvertCondition(cond)); 600 b(label); 601 bind(&done); 602 } else { 603 if (!label->IsBound()) { 604 veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(), 605 label, 606 CondBranchType); 607 } 608 b(label, cond); 609 } 610 } 611 612 613 void MacroAssembler::Cbnz(const Register& rt, Label* label) { 614 // We don't need to check the size of the literal pool, because the size of 615 // the literal pool is already bounded by the literal range, which is smaller 616 // than the range of this branch. 617 VIXL_ASSERT(Instruction::GetImmBranchForwardRange(CompareBranchType) > 618 Instruction::kLoadLiteralRange); 619 VIXL_ASSERT(allow_macro_instructions_); 620 VIXL_ASSERT(!rt.IsZero()); 621 EmissionCheckScope guard(this, 2 * kInstructionSize); 622 623 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 624 Label done; 625 cbz(rt, &done); 626 b(label); 627 bind(&done); 628 } else { 629 if (!label->IsBound()) { 630 veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(), 631 label, 632 CompareBranchType); 633 } 634 cbnz(rt, label); 635 } 636 } 637 638 639 void MacroAssembler::Cbz(const Register& rt, Label* label) { 640 // We don't need to check the size of the literal pool, because the size of 641 // the literal pool is already bounded by the literal range, which is smaller 642 // than the range of this branch. 643 VIXL_ASSERT(Instruction::GetImmBranchForwardRange(CompareBranchType) > 644 Instruction::kLoadLiteralRange); 645 VIXL_ASSERT(allow_macro_instructions_); 646 VIXL_ASSERT(!rt.IsZero()); 647 EmissionCheckScope guard(this, 2 * kInstructionSize); 648 649 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 650 Label done; 651 cbnz(rt, &done); 652 b(label); 653 bind(&done); 654 } else { 655 if (!label->IsBound()) { 656 veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(), 657 label, 658 CompareBranchType); 659 } 660 cbz(rt, label); 661 } 662 } 663 664 665 void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) { 666 // This is to avoid a situation where emitting a veneer for a TBZ/TBNZ branch 667 // can become impossible because we emit the literal pool first. 668 literal_pool_.CheckEmitForBranch( 669 Instruction::GetImmBranchForwardRange(TestBranchType)); 670 VIXL_ASSERT(allow_macro_instructions_); 671 VIXL_ASSERT(!rt.IsZero()); 672 EmissionCheckScope guard(this, 2 * kInstructionSize); 673 674 if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) { 675 Label done; 676 tbz(rt, bit_pos, &done); 677 b(label); 678 bind(&done); 679 } else { 680 if (!label->IsBound()) { 681 veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(), 682 label, 683 TestBranchType); 684 } 685 tbnz(rt, bit_pos, label); 686 } 687 } 688 689 690 void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) { 691 // This is to avoid a situation where emitting a veneer for a TBZ/TBNZ branch 692 // can become impossible because we emit the literal pool first. 693 literal_pool_.CheckEmitForBranch( 694 Instruction::GetImmBranchForwardRange(TestBranchType)); 695 VIXL_ASSERT(allow_macro_instructions_); 696 VIXL_ASSERT(!rt.IsZero()); 697 EmissionCheckScope guard(this, 2 * kInstructionSize); 698 699 if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) { 700 Label done; 701 tbnz(rt, bit_pos, &done); 702 b(label); 703 bind(&done); 704 } else { 705 if (!label->IsBound()) { 706 veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(), 707 label, 708 TestBranchType); 709 } 710 tbz(rt, bit_pos, label); 711 } 712 } 713 714 715 void MacroAssembler::Bind(Label* label) { 716 VIXL_ASSERT(allow_macro_instructions_); 717 veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label); 718 bind(label); 719 } 720 721 722 // Bind a label to a specified offset from the start of the buffer. 723 void MacroAssembler::BindToOffset(Label* label, ptrdiff_t offset) { 724 VIXL_ASSERT(allow_macro_instructions_); 725 veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label); 726 Assembler::BindToOffset(label, offset); 727 } 728 729 730 void MacroAssembler::And(const Register& rd, 731 const Register& rn, 732 const Operand& operand) { 733 VIXL_ASSERT(allow_macro_instructions_); 734 LogicalMacro(rd, rn, operand, AND); 735 } 736 737 738 void MacroAssembler::Ands(const Register& rd, 739 const Register& rn, 740 const Operand& operand) { 741 VIXL_ASSERT(allow_macro_instructions_); 742 LogicalMacro(rd, rn, operand, ANDS); 743 } 744 745 746 void MacroAssembler::Tst(const Register& rn, const Operand& operand) { 747 VIXL_ASSERT(allow_macro_instructions_); 748 Ands(AppropriateZeroRegFor(rn), rn, operand); 749 } 750 751 752 void MacroAssembler::Bic(const Register& rd, 753 const Register& rn, 754 const Operand& operand) { 755 VIXL_ASSERT(allow_macro_instructions_); 756 LogicalMacro(rd, rn, operand, BIC); 757 } 758 759 760 void MacroAssembler::Bics(const Register& rd, 761 const Register& rn, 762 const Operand& operand) { 763 VIXL_ASSERT(allow_macro_instructions_); 764 LogicalMacro(rd, rn, operand, BICS); 765 } 766 767 768 void MacroAssembler::Orr(const Register& rd, 769 const Register& rn, 770 const Operand& operand) { 771 VIXL_ASSERT(allow_macro_instructions_); 772 LogicalMacro(rd, rn, operand, ORR); 773 } 774 775 776 void MacroAssembler::Orn(const Register& rd, 777 const Register& rn, 778 const Operand& operand) { 779 VIXL_ASSERT(allow_macro_instructions_); 780 LogicalMacro(rd, rn, operand, ORN); 781 } 782 783 784 void MacroAssembler::Eor(const Register& rd, 785 const Register& rn, 786 const Operand& operand) { 787 VIXL_ASSERT(allow_macro_instructions_); 788 LogicalMacro(rd, rn, operand, EOR); 789 } 790 791 792 void MacroAssembler::Eon(const Register& rd, 793 const Register& rn, 794 const Operand& operand) { 795 VIXL_ASSERT(allow_macro_instructions_); 796 LogicalMacro(rd, rn, operand, EON); 797 } 798 799 800 void MacroAssembler::LogicalMacro(const Register& rd, 801 const Register& rn, 802 const Operand& operand, 803 LogicalOp op) { 804 // The worst case for size is logical immediate to sp: 805 // * up to 4 instructions to materialise the constant 806 // * 1 instruction to do the operation 807 // * 1 instruction to move to sp 808 MacroEmissionCheckScope guard(this); 809 UseScratchRegisterScope temps(this); 810 811 if (operand.IsImmediate()) { 812 uint64_t immediate = operand.GetImmediate(); 813 unsigned reg_size = rd.GetSizeInBits(); 814 815 // If the operation is NOT, invert the operation and immediate. 816 if ((op & NOT) == NOT) { 817 op = static_cast<LogicalOp>(op & ~NOT); 818 immediate = ~immediate; 819 } 820 821 // Ignore the top 32 bits of an immediate if we're moving to a W register. 822 if (rd.Is32Bits()) { 823 // Check that the top 32 bits are consistent. 824 VIXL_ASSERT(((immediate >> kWRegSize) == 0) || 825 ((immediate >> kWRegSize) == 0xffffffff)); 826 immediate &= kWRegMask; 827 } 828 829 VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate)); 830 831 // Special cases for all set or all clear immediates. 832 if (immediate == 0) { 833 switch (op) { 834 case AND: 835 Mov(rd, 0); 836 return; 837 case ORR: 838 VIXL_FALLTHROUGH(); 839 case EOR: 840 Mov(rd, rn); 841 return; 842 case ANDS: 843 VIXL_FALLTHROUGH(); 844 case BICS: 845 break; 846 default: 847 VIXL_UNREACHABLE(); 848 } 849 } else if ((rd.Is64Bits() && (immediate == UINT64_C(0xffffffffffffffff))) || 850 (rd.Is32Bits() && (immediate == UINT64_C(0x00000000ffffffff)))) { 851 switch (op) { 852 case AND: 853 Mov(rd, rn); 854 return; 855 case ORR: 856 Mov(rd, immediate); 857 return; 858 case EOR: 859 Mvn(rd, rn); 860 return; 861 case ANDS: 862 VIXL_FALLTHROUGH(); 863 case BICS: 864 break; 865 default: 866 VIXL_UNREACHABLE(); 867 } 868 } 869 870 unsigned n, imm_s, imm_r; 871 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 872 // Immediate can be encoded in the instruction. 873 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 874 } else { 875 // Immediate can't be encoded: synthesize using move immediate. 876 Register temp = temps.AcquireSameSizeAs(rn); 877 878 // If the left-hand input is the stack pointer, we can't pre-shift the 879 // immediate, as the encoding won't allow the subsequent post shift. 880 PreShiftImmMode mode = rn.IsSP() ? kNoShift : kAnyShift; 881 Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode); 882 883 if (rd.Is(sp)) { 884 // If rd is the stack pointer we cannot use it as the destination 885 // register so we use the temp register as an intermediate again. 886 Logical(temp, rn, imm_operand, op); 887 Mov(sp, temp); 888 } else { 889 Logical(rd, rn, imm_operand, op); 890 } 891 } 892 } else if (operand.IsExtendedRegister()) { 893 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() <= rd.GetSizeInBits()); 894 // Add/sub extended supports shift <= 4. We want to support exactly the 895 // same modes here. 896 VIXL_ASSERT(operand.GetShiftAmount() <= 4); 897 VIXL_ASSERT( 898 operand.GetRegister().Is64Bits() || 899 ((operand.GetExtend() != UXTX) && (operand.GetExtend() != SXTX))); 900 901 temps.Exclude(operand.GetRegister()); 902 Register temp = temps.AcquireSameSizeAs(rn); 903 EmitExtendShift(temp, 904 operand.GetRegister(), 905 operand.GetExtend(), 906 operand.GetShiftAmount()); 907 Logical(rd, rn, Operand(temp), op); 908 } else { 909 // The operand can be encoded in the instruction. 910 VIXL_ASSERT(operand.IsShiftedRegister()); 911 Logical(rd, rn, operand, op); 912 } 913 } 914 915 916 void MacroAssembler::Mov(const Register& rd, 917 const Operand& operand, 918 DiscardMoveMode discard_mode) { 919 VIXL_ASSERT(allow_macro_instructions_); 920 // The worst case for size is mov immediate with up to 4 instructions. 921 MacroEmissionCheckScope guard(this); 922 923 if (operand.IsImmediate()) { 924 // Call the macro assembler for generic immediates. 925 Mov(rd, operand.GetImmediate()); 926 } else if (operand.IsShiftedRegister() && (operand.GetShiftAmount() != 0)) { 927 // Emit a shift instruction if moving a shifted register. This operation 928 // could also be achieved using an orr instruction (like orn used by Mvn), 929 // but using a shift instruction makes the disassembly clearer. 930 EmitShift(rd, 931 operand.GetRegister(), 932 operand.GetShift(), 933 operand.GetShiftAmount()); 934 } else if (operand.IsExtendedRegister()) { 935 // Emit an extend instruction if moving an extended register. This handles 936 // extend with post-shift operations, too. 937 EmitExtendShift(rd, 938 operand.GetRegister(), 939 operand.GetExtend(), 940 operand.GetShiftAmount()); 941 } else { 942 Mov(rd, operand.GetRegister(), discard_mode); 943 } 944 } 945 946 947 void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) { 948 VIXL_ASSERT(IsUint16(imm)); 949 int byte1 = (imm & 0xff); 950 int byte2 = ((imm >> 8) & 0xff); 951 if (byte1 == byte2) { 952 movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1); 953 } else if (byte1 == 0) { 954 movi(vd, byte2, LSL, 8); 955 } else if (byte2 == 0) { 956 movi(vd, byte1); 957 } else if (byte1 == 0xff) { 958 mvni(vd, ~byte2 & 0xff, LSL, 8); 959 } else if (byte2 == 0xff) { 960 mvni(vd, ~byte1 & 0xff); 961 } else { 962 UseScratchRegisterScope temps(this); 963 Register temp = temps.AcquireW(); 964 movz(temp, imm); 965 dup(vd, temp); 966 } 967 } 968 969 970 void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) { 971 VIXL_ASSERT(IsUint32(imm)); 972 973 uint8_t bytes[sizeof(imm)]; 974 memcpy(bytes, &imm, sizeof(imm)); 975 976 // All bytes are either 0x00 or 0xff. 977 { 978 bool all0orff = true; 979 for (int i = 0; i < 4; ++i) { 980 if ((bytes[i] != 0) && (bytes[i] != 0xff)) { 981 all0orff = false; 982 break; 983 } 984 } 985 986 if (all0orff == true) { 987 movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm)); 988 return; 989 } 990 } 991 992 // Of the 4 bytes, only one byte is non-zero. 993 for (int i = 0; i < 4; i++) { 994 if ((imm & (0xff << (i * 8))) == imm) { 995 movi(vd, bytes[i], LSL, i * 8); 996 return; 997 } 998 } 999 1000 // Of the 4 bytes, only one byte is not 0xff. 1001 for (int i = 0; i < 4; i++) { 1002 uint32_t mask = ~(0xff << (i * 8)); 1003 if ((imm & mask) == mask) { 1004 mvni(vd, ~bytes[i] & 0xff, LSL, i * 8); 1005 return; 1006 } 1007 } 1008 1009 // Immediate is of the form 0x00MMFFFF. 1010 if ((imm & 0xff00ffff) == 0x0000ffff) { 1011 movi(vd, bytes[2], MSL, 16); 1012 return; 1013 } 1014 1015 // Immediate is of the form 0x0000MMFF. 1016 if ((imm & 0xffff00ff) == 0x000000ff) { 1017 movi(vd, bytes[1], MSL, 8); 1018 return; 1019 } 1020 1021 // Immediate is of the form 0xFFMM0000. 1022 if ((imm & 0xff00ffff) == 0xff000000) { 1023 mvni(vd, ~bytes[2] & 0xff, MSL, 16); 1024 return; 1025 } 1026 // Immediate is of the form 0xFFFFMM00. 1027 if ((imm & 0xffff00ff) == 0xffff0000) { 1028 mvni(vd, ~bytes[1] & 0xff, MSL, 8); 1029 return; 1030 } 1031 1032 // Top and bottom 16-bits are equal. 1033 if (((imm >> 16) & 0xffff) == (imm & 0xffff)) { 1034 Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff); 1035 return; 1036 } 1037 1038 // Default case. 1039 { 1040 UseScratchRegisterScope temps(this); 1041 Register temp = temps.AcquireW(); 1042 Mov(temp, imm); 1043 dup(vd, temp); 1044 } 1045 } 1046 1047 1048 void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) { 1049 // All bytes are either 0x00 or 0xff. 1050 { 1051 bool all0orff = true; 1052 for (int i = 0; i < 8; ++i) { 1053 int byteval = (imm >> (i * 8)) & 0xff; 1054 if (byteval != 0 && byteval != 0xff) { 1055 all0orff = false; 1056 break; 1057 } 1058 } 1059 if (all0orff == true) { 1060 movi(vd, imm); 1061 return; 1062 } 1063 } 1064 1065 // Top and bottom 32-bits are equal. 1066 if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) { 1067 Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff); 1068 return; 1069 } 1070 1071 // Default case. 1072 { 1073 UseScratchRegisterScope temps(this); 1074 Register temp = temps.AcquireX(); 1075 Mov(temp, imm); 1076 if (vd.Is1D()) { 1077 mov(vd.D(), 0, temp); 1078 } else { 1079 dup(vd.V2D(), temp); 1080 } 1081 } 1082 } 1083 1084 1085 void MacroAssembler::Movi(const VRegister& vd, 1086 uint64_t imm, 1087 Shift shift, 1088 int shift_amount) { 1089 VIXL_ASSERT(allow_macro_instructions_); 1090 MacroEmissionCheckScope guard(this); 1091 if (shift_amount != 0 || shift != LSL) { 1092 movi(vd, imm, shift, shift_amount); 1093 } else if (vd.Is8B() || vd.Is16B()) { 1094 // 8-bit immediate. 1095 VIXL_ASSERT(IsUint8(imm)); 1096 movi(vd, imm); 1097 } else if (vd.Is4H() || vd.Is8H()) { 1098 // 16-bit immediate. 1099 Movi16bitHelper(vd, imm); 1100 } else if (vd.Is2S() || vd.Is4S()) { 1101 // 32-bit immediate. 1102 Movi32bitHelper(vd, imm); 1103 } else { 1104 // 64-bit immediate. 1105 Movi64bitHelper(vd, imm); 1106 } 1107 } 1108 1109 1110 void MacroAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) { 1111 // TODO: Move 128-bit values in a more efficient way. 1112 VIXL_ASSERT(vd.Is128Bits()); 1113 UseScratchRegisterScope temps(this); 1114 Movi(vd.V2D(), lo); 1115 Register temp = temps.AcquireX(); 1116 Mov(temp, hi); 1117 Ins(vd.V2D(), 1, temp); 1118 } 1119 1120 1121 void MacroAssembler::Mvn(const Register& rd, const Operand& operand) { 1122 VIXL_ASSERT(allow_macro_instructions_); 1123 // The worst case for size is mvn immediate with up to 4 instructions. 1124 MacroEmissionCheckScope guard(this); 1125 1126 if (operand.IsImmediate()) { 1127 // Call the macro assembler for generic immediates. 1128 Mvn(rd, operand.GetImmediate()); 1129 } else if (operand.IsExtendedRegister()) { 1130 UseScratchRegisterScope temps(this); 1131 temps.Exclude(operand.GetRegister()); 1132 1133 // Emit two instructions for the extend case. This differs from Mov, as 1134 // the extend and invert can't be achieved in one instruction. 1135 Register temp = temps.AcquireSameSizeAs(rd); 1136 EmitExtendShift(temp, 1137 operand.GetRegister(), 1138 operand.GetExtend(), 1139 operand.GetShiftAmount()); 1140 mvn(rd, Operand(temp)); 1141 } else { 1142 // Otherwise, register and shifted register cases can be handled by the 1143 // assembler directly, using orn. 1144 mvn(rd, operand); 1145 } 1146 } 1147 1148 1149 void MacroAssembler::Mov(const Register& rd, uint64_t imm) { 1150 VIXL_ASSERT(allow_macro_instructions_); 1151 MoveImmediateHelper(this, rd, imm); 1152 } 1153 1154 1155 void MacroAssembler::Ccmp(const Register& rn, 1156 const Operand& operand, 1157 StatusFlags nzcv, 1158 Condition cond) { 1159 VIXL_ASSERT(allow_macro_instructions_); 1160 if (operand.IsImmediate() && (operand.GetImmediate() < 0)) { 1161 ConditionalCompareMacro(rn, -operand.GetImmediate(), nzcv, cond, CCMN); 1162 } else { 1163 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP); 1164 } 1165 } 1166 1167 1168 void MacroAssembler::Ccmn(const Register& rn, 1169 const Operand& operand, 1170 StatusFlags nzcv, 1171 Condition cond) { 1172 VIXL_ASSERT(allow_macro_instructions_); 1173 if (operand.IsImmediate() && (operand.GetImmediate() < 0)) { 1174 ConditionalCompareMacro(rn, -operand.GetImmediate(), nzcv, cond, CCMP); 1175 } else { 1176 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN); 1177 } 1178 } 1179 1180 1181 void MacroAssembler::ConditionalCompareMacro(const Register& rn, 1182 const Operand& operand, 1183 StatusFlags nzcv, 1184 Condition cond, 1185 ConditionalCompareOp op) { 1186 VIXL_ASSERT((cond != al) && (cond != nv)); 1187 // The worst case for size is ccmp immediate: 1188 // * up to 4 instructions to materialise the constant 1189 // * 1 instruction for ccmp 1190 MacroEmissionCheckScope guard(this); 1191 1192 if ((operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0)) || 1193 (operand.IsImmediate() && 1194 IsImmConditionalCompare(operand.GetImmediate()))) { 1195 // The immediate can be encoded in the instruction, or the operand is an 1196 // unshifted register: call the assembler. 1197 ConditionalCompare(rn, operand, nzcv, cond, op); 1198 } else { 1199 UseScratchRegisterScope temps(this); 1200 // The operand isn't directly supported by the instruction: perform the 1201 // operation on a temporary register. 1202 Register temp = temps.AcquireSameSizeAs(rn); 1203 Mov(temp, operand); 1204 ConditionalCompare(rn, temp, nzcv, cond, op); 1205 } 1206 } 1207 1208 1209 void MacroAssembler::CselHelper(MacroAssembler* masm, 1210 const Register& rd, 1211 Operand left, 1212 Operand right, 1213 Condition cond, 1214 bool* should_synthesise_left, 1215 bool* should_synthesise_right) { 1216 bool emit_code = (masm != NULL); 1217 1218 VIXL_ASSERT(!emit_code || masm->allow_macro_instructions_); 1219 VIXL_ASSERT((cond != al) && (cond != nv)); 1220 VIXL_ASSERT(!rd.IsZero() && !rd.IsSP()); 1221 VIXL_ASSERT(left.IsImmediate() || !left.GetRegister().IsSP()); 1222 VIXL_ASSERT(right.IsImmediate() || !right.GetRegister().IsSP()); 1223 1224 if (should_synthesise_left != NULL) *should_synthesise_left = false; 1225 if (should_synthesise_right != NULL) *should_synthesise_right = false; 1226 1227 // The worst case for size occurs when the inputs are two non encodable 1228 // constants: 1229 // * up to 4 instructions to materialise the left constant 1230 // * up to 4 instructions to materialise the right constant 1231 // * 1 instruction for csel 1232 EmissionCheckScope guard(masm, 9 * kInstructionSize); 1233 UseScratchRegisterScope temps; 1234 if (masm != NULL) { 1235 temps.Open(masm); 1236 } 1237 1238 // Try to handle cases where both inputs are immediates. 1239 bool left_is_immediate = left.IsImmediate() || left.IsZero(); 1240 bool right_is_immediate = right.IsImmediate() || right.IsZero(); 1241 if (left_is_immediate && right_is_immediate && 1242 CselSubHelperTwoImmediates(masm, 1243 rd, 1244 left.GetEquivalentImmediate(), 1245 right.GetEquivalentImmediate(), 1246 cond, 1247 should_synthesise_left, 1248 should_synthesise_right)) { 1249 return; 1250 } 1251 1252 // Handle cases where one of the two inputs is -1, 0, or 1. 1253 bool left_is_small_immediate = 1254 left_is_immediate && ((-1 <= left.GetEquivalentImmediate()) && 1255 (left.GetEquivalentImmediate() <= 1)); 1256 bool right_is_small_immediate = 1257 right_is_immediate && ((-1 <= right.GetEquivalentImmediate()) && 1258 (right.GetEquivalentImmediate() <= 1)); 1259 if (right_is_small_immediate || left_is_small_immediate) { 1260 bool swapped_inputs = false; 1261 if (!right_is_small_immediate) { 1262 std::swap(left, right); 1263 cond = InvertCondition(cond); 1264 swapped_inputs = true; 1265 } 1266 CselSubHelperRightSmallImmediate(masm, 1267 &temps, 1268 rd, 1269 left, 1270 right, 1271 cond, 1272 swapped_inputs ? should_synthesise_right 1273 : should_synthesise_left); 1274 return; 1275 } 1276 1277 // Otherwise both inputs need to be available in registers. Synthesise them 1278 // if necessary and emit the `csel`. 1279 if (!left.IsPlainRegister()) { 1280 if (emit_code) { 1281 Register temp = temps.AcquireSameSizeAs(rd); 1282 masm->Mov(temp, left); 1283 left = temp; 1284 } 1285 if (should_synthesise_left != NULL) *should_synthesise_left = true; 1286 } 1287 if (!right.IsPlainRegister()) { 1288 if (emit_code) { 1289 Register temp = temps.AcquireSameSizeAs(rd); 1290 masm->Mov(temp, right); 1291 right = temp; 1292 } 1293 if (should_synthesise_right != NULL) *should_synthesise_right = true; 1294 } 1295 if (emit_code) { 1296 VIXL_ASSERT(left.IsPlainRegister() && right.IsPlainRegister()); 1297 if (left.GetRegister().Is(right.GetRegister())) { 1298 masm->Mov(rd, left.GetRegister()); 1299 } else { 1300 masm->csel(rd, left.GetRegister(), right.GetRegister(), cond); 1301 } 1302 } 1303 } 1304 1305 1306 bool MacroAssembler::CselSubHelperTwoImmediates(MacroAssembler* masm, 1307 const Register& rd, 1308 int64_t left, 1309 int64_t right, 1310 Condition cond, 1311 bool* should_synthesise_left, 1312 bool* should_synthesise_right) { 1313 bool emit_code = (masm != NULL); 1314 if (should_synthesise_left != NULL) *should_synthesise_left = false; 1315 if (should_synthesise_right != NULL) *should_synthesise_right = false; 1316 1317 if (left == right) { 1318 if (emit_code) masm->Mov(rd, left); 1319 return true; 1320 } else if (left == -right) { 1321 if (should_synthesise_right != NULL) *should_synthesise_right = true; 1322 if (emit_code) { 1323 masm->Mov(rd, right); 1324 masm->Cneg(rd, rd, cond); 1325 } 1326 return true; 1327 } 1328 1329 if (CselSubHelperTwoOrderedImmediates(masm, rd, left, right, cond)) { 1330 return true; 1331 } else { 1332 std::swap(left, right); 1333 if (CselSubHelperTwoOrderedImmediates(masm, 1334 rd, 1335 left, 1336 right, 1337 InvertCondition(cond))) { 1338 return true; 1339 } 1340 } 1341 1342 // TODO: Handle more situations. For example handle `csel rd, #5, #6, cond` 1343 // with `cinc`. 1344 return false; 1345 } 1346 1347 1348 bool MacroAssembler::CselSubHelperTwoOrderedImmediates(MacroAssembler* masm, 1349 const Register& rd, 1350 int64_t left, 1351 int64_t right, 1352 Condition cond) { 1353 bool emit_code = (masm != NULL); 1354 1355 if ((left == 1) && (right == 0)) { 1356 if (emit_code) masm->cset(rd, cond); 1357 return true; 1358 } else if ((left == -1) && (right == 0)) { 1359 if (emit_code) masm->csetm(rd, cond); 1360 return true; 1361 } 1362 return false; 1363 } 1364 1365 1366 void MacroAssembler::CselSubHelperRightSmallImmediate( 1367 MacroAssembler* masm, 1368 UseScratchRegisterScope* temps, 1369 const Register& rd, 1370 const Operand& left, 1371 const Operand& right, 1372 Condition cond, 1373 bool* should_synthesise_left) { 1374 bool emit_code = (masm != NULL); 1375 VIXL_ASSERT((right.IsImmediate() || right.IsZero()) && 1376 (-1 <= right.GetEquivalentImmediate()) && 1377 (right.GetEquivalentImmediate() <= 1)); 1378 Register left_register; 1379 1380 if (left.IsPlainRegister()) { 1381 left_register = left.GetRegister(); 1382 } else { 1383 if (emit_code) { 1384 left_register = temps->AcquireSameSizeAs(rd); 1385 masm->Mov(left_register, left); 1386 } 1387 if (should_synthesise_left != NULL) *should_synthesise_left = true; 1388 } 1389 if (emit_code) { 1390 int64_t imm = right.GetEquivalentImmediate(); 1391 Register zr = AppropriateZeroRegFor(rd); 1392 if (imm == 0) { 1393 masm->csel(rd, left_register, zr, cond); 1394 } else if (imm == 1) { 1395 masm->csinc(rd, left_register, zr, cond); 1396 } else { 1397 VIXL_ASSERT(imm == -1); 1398 masm->csinv(rd, left_register, zr, cond); 1399 } 1400 } 1401 } 1402 1403 1404 void MacroAssembler::Add(const Register& rd, 1405 const Register& rn, 1406 const Operand& operand, 1407 FlagsUpdate S) { 1408 VIXL_ASSERT(allow_macro_instructions_); 1409 if (operand.IsImmediate() && (operand.GetImmediate() < 0) && 1410 IsImmAddSub(-operand.GetImmediate())) { 1411 AddSubMacro(rd, rn, -operand.GetImmediate(), S, SUB); 1412 } else { 1413 AddSubMacro(rd, rn, operand, S, ADD); 1414 } 1415 } 1416 1417 1418 void MacroAssembler::Adds(const Register& rd, 1419 const Register& rn, 1420 const Operand& operand) { 1421 Add(rd, rn, operand, SetFlags); 1422 } 1423 1424 1425 void MacroAssembler::Sub(const Register& rd, 1426 const Register& rn, 1427 const Operand& operand, 1428 FlagsUpdate S) { 1429 VIXL_ASSERT(allow_macro_instructions_); 1430 if (operand.IsImmediate() && (operand.GetImmediate() < 0) && 1431 IsImmAddSub(-operand.GetImmediate())) { 1432 AddSubMacro(rd, rn, -operand.GetImmediate(), S, ADD); 1433 } else { 1434 AddSubMacro(rd, rn, operand, S, SUB); 1435 } 1436 } 1437 1438 1439 void MacroAssembler::Subs(const Register& rd, 1440 const Register& rn, 1441 const Operand& operand) { 1442 Sub(rd, rn, operand, SetFlags); 1443 } 1444 1445 1446 void MacroAssembler::Cmn(const Register& rn, const Operand& operand) { 1447 VIXL_ASSERT(allow_macro_instructions_); 1448 Adds(AppropriateZeroRegFor(rn), rn, operand); 1449 } 1450 1451 1452 void MacroAssembler::Cmp(const Register& rn, const Operand& operand) { 1453 VIXL_ASSERT(allow_macro_instructions_); 1454 Subs(AppropriateZeroRegFor(rn), rn, operand); 1455 } 1456 1457 1458 void MacroAssembler::Fcmp(const FPRegister& fn, 1459 double value, 1460 FPTrapFlags trap) { 1461 VIXL_ASSERT(allow_macro_instructions_); 1462 // The worst case for size is: 1463 // * 1 to materialise the constant, using literal pool if necessary 1464 // * 1 instruction for fcmp{e} 1465 MacroEmissionCheckScope guard(this); 1466 if (value != 0.0) { 1467 UseScratchRegisterScope temps(this); 1468 FPRegister tmp = temps.AcquireSameSizeAs(fn); 1469 Fmov(tmp, value); 1470 FPCompareMacro(fn, tmp, trap); 1471 } else { 1472 FPCompareMacro(fn, value, trap); 1473 } 1474 } 1475 1476 1477 void MacroAssembler::Fcmpe(const FPRegister& fn, double value) { 1478 Fcmp(fn, value, EnableTrap); 1479 } 1480 1481 1482 void MacroAssembler::Fmov(VRegister vd, double imm) { 1483 VIXL_ASSERT(allow_macro_instructions_); 1484 // Floating point immediates are loaded through the literal pool. 1485 MacroEmissionCheckScope guard(this); 1486 1487 if (vd.Is1H() || vd.Is4H() || vd.Is8H()) { 1488 Fmov(vd, Float16(imm)); 1489 return; 1490 } 1491 1492 if (vd.Is1S() || vd.Is2S() || vd.Is4S()) { 1493 Fmov(vd, static_cast<float>(imm)); 1494 return; 1495 } 1496 1497 VIXL_ASSERT(vd.Is1D() || vd.Is2D()); 1498 if (IsImmFP64(imm)) { 1499 fmov(vd, imm); 1500 } else { 1501 uint64_t rawbits = DoubleToRawbits(imm); 1502 if (vd.IsScalar()) { 1503 if (rawbits == 0) { 1504 fmov(vd, xzr); 1505 } else { 1506 ldr(vd, 1507 new Literal<double>(imm, 1508 &literal_pool_, 1509 RawLiteral::kDeletedOnPlacementByPool)); 1510 } 1511 } else { 1512 // TODO: consider NEON support for load literal. 1513 Movi(vd, rawbits); 1514 } 1515 } 1516 } 1517 1518 1519 void MacroAssembler::Fmov(VRegister vd, float imm) { 1520 VIXL_ASSERT(allow_macro_instructions_); 1521 // Floating point immediates are loaded through the literal pool. 1522 MacroEmissionCheckScope guard(this); 1523 1524 if (vd.Is1H() || vd.Is4H() || vd.Is8H()) { 1525 Fmov(vd, Float16(imm)); 1526 return; 1527 } 1528 1529 if (vd.Is1D() || vd.Is2D()) { 1530 Fmov(vd, static_cast<double>(imm)); 1531 return; 1532 } 1533 1534 VIXL_ASSERT(vd.Is1S() || vd.Is2S() || vd.Is4S()); 1535 if (IsImmFP32(imm)) { 1536 fmov(vd, imm); 1537 } else { 1538 uint32_t rawbits = FloatToRawbits(imm); 1539 if (vd.IsScalar()) { 1540 if (rawbits == 0) { 1541 fmov(vd, wzr); 1542 } else { 1543 ldr(vd, 1544 new Literal<float>(imm, 1545 &literal_pool_, 1546 RawLiteral::kDeletedOnPlacementByPool)); 1547 } 1548 } else { 1549 // TODO: consider NEON support for load literal. 1550 Movi(vd, rawbits); 1551 } 1552 } 1553 } 1554 1555 1556 void MacroAssembler::Fmov(VRegister vd, Float16 imm) { 1557 VIXL_ASSERT(allow_macro_instructions_); 1558 MacroEmissionCheckScope guard(this); 1559 1560 if (vd.Is1S() || vd.Is2S() || vd.Is4S()) { 1561 Fmov(vd, FPToFloat(imm, kIgnoreDefaultNaN)); 1562 return; 1563 } 1564 1565 if (vd.Is1D() || vd.Is2D()) { 1566 Fmov(vd, FPToDouble(imm, kIgnoreDefaultNaN)); 1567 return; 1568 } 1569 1570 VIXL_ASSERT(vd.Is1H() || vd.Is4H() || vd.Is8H()); 1571 uint16_t rawbits = Float16ToRawbits(imm); 1572 if (IsImmFP16(imm)) { 1573 fmov(vd, imm); 1574 } else { 1575 if (vd.IsScalar()) { 1576 if (rawbits == 0x0) { 1577 fmov(vd, wzr); 1578 } else { 1579 // We can use movz instead of the literal pool. 1580 UseScratchRegisterScope temps(this); 1581 Register temp = temps.AcquireW(); 1582 Mov(temp, rawbits); 1583 Fmov(vd, temp); 1584 } 1585 } else { 1586 // TODO: consider NEON support for load literal. 1587 Movi(vd, static_cast<uint64_t>(rawbits)); 1588 } 1589 } 1590 } 1591 1592 1593 void MacroAssembler::Neg(const Register& rd, const Operand& operand) { 1594 VIXL_ASSERT(allow_macro_instructions_); 1595 if (operand.IsImmediate()) { 1596 Mov(rd, -operand.GetImmediate()); 1597 } else { 1598 Sub(rd, AppropriateZeroRegFor(rd), operand); 1599 } 1600 } 1601 1602 1603 void MacroAssembler::Negs(const Register& rd, const Operand& operand) { 1604 VIXL_ASSERT(allow_macro_instructions_); 1605 Subs(rd, AppropriateZeroRegFor(rd), operand); 1606 } 1607 1608 1609 bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst, 1610 int64_t imm) { 1611 return OneInstrMoveImmediateHelper(this, dst, imm); 1612 } 1613 1614 1615 Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst, 1616 int64_t imm, 1617 PreShiftImmMode mode) { 1618 int reg_size = dst.GetSizeInBits(); 1619 1620 // Encode the immediate in a single move instruction, if possible. 1621 if (TryOneInstrMoveImmediate(dst, imm)) { 1622 // The move was successful; nothing to do here. 1623 } else { 1624 // Pre-shift the immediate to the least-significant bits of the register. 1625 int shift_low = CountTrailingZeros(imm, reg_size); 1626 if (mode == kLimitShiftForSP) { 1627 // When applied to the stack pointer, the subsequent arithmetic operation 1628 // can use the extend form to shift left by a maximum of four bits. Right 1629 // shifts are not allowed, so we filter them out later before the new 1630 // immediate is tested. 1631 shift_low = std::min(shift_low, 4); 1632 } 1633 int64_t imm_low = imm >> shift_low; 1634 1635 // Pre-shift the immediate to the most-significant bits of the register, 1636 // inserting set bits in the least-significant bits. 1637 int shift_high = CountLeadingZeros(imm, reg_size); 1638 int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1); 1639 1640 if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) { 1641 // The new immediate has been moved into the destination's low bits: 1642 // return a new leftward-shifting operand. 1643 return Operand(dst, LSL, shift_low); 1644 } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) { 1645 // The new immediate has been moved into the destination's high bits: 1646 // return a new rightward-shifting operand. 1647 return Operand(dst, LSR, shift_high); 1648 } else { 1649 Mov(dst, imm); 1650 } 1651 } 1652 return Operand(dst); 1653 } 1654 1655 1656 void MacroAssembler::Move(const GenericOperand& dst, 1657 const GenericOperand& src) { 1658 if (dst.Equals(src)) { 1659 return; 1660 } 1661 1662 VIXL_ASSERT(dst.IsValid() && src.IsValid()); 1663 1664 // The sizes of the operands must match exactly. 1665 VIXL_ASSERT(dst.GetSizeInBits() == src.GetSizeInBits()); 1666 VIXL_ASSERT(dst.GetSizeInBits() <= kXRegSize); 1667 int operand_size = static_cast<int>(dst.GetSizeInBits()); 1668 1669 if (dst.IsCPURegister() && src.IsCPURegister()) { 1670 CPURegister dst_reg = dst.GetCPURegister(); 1671 CPURegister src_reg = src.GetCPURegister(); 1672 if (dst_reg.IsRegister() && src_reg.IsRegister()) { 1673 Mov(Register(dst_reg), Register(src_reg)); 1674 } else if (dst_reg.IsVRegister() && src_reg.IsVRegister()) { 1675 Fmov(VRegister(dst_reg), VRegister(src_reg)); 1676 } else { 1677 if (dst_reg.IsRegister()) { 1678 Fmov(Register(dst_reg), VRegister(src_reg)); 1679 } else { 1680 Fmov(VRegister(dst_reg), Register(src_reg)); 1681 } 1682 } 1683 return; 1684 } 1685 1686 if (dst.IsMemOperand() && src.IsMemOperand()) { 1687 UseScratchRegisterScope temps(this); 1688 CPURegister temp = temps.AcquireCPURegisterOfSize(operand_size); 1689 Ldr(temp, src.GetMemOperand()); 1690 Str(temp, dst.GetMemOperand()); 1691 return; 1692 } 1693 1694 if (dst.IsCPURegister()) { 1695 Ldr(dst.GetCPURegister(), src.GetMemOperand()); 1696 } else { 1697 Str(src.GetCPURegister(), dst.GetMemOperand()); 1698 } 1699 } 1700 1701 1702 void MacroAssembler::ComputeAddress(const Register& dst, 1703 const MemOperand& mem_op) { 1704 // We cannot handle pre-indexing or post-indexing. 1705 VIXL_ASSERT(mem_op.GetAddrMode() == Offset); 1706 Register base = mem_op.GetBaseRegister(); 1707 if (mem_op.IsImmediateOffset()) { 1708 Add(dst, base, mem_op.GetOffset()); 1709 } else { 1710 VIXL_ASSERT(mem_op.IsRegisterOffset()); 1711 Register reg_offset = mem_op.GetRegisterOffset(); 1712 Shift shift = mem_op.GetShift(); 1713 Extend extend = mem_op.GetExtend(); 1714 if (shift == NO_SHIFT) { 1715 VIXL_ASSERT(extend != NO_EXTEND); 1716 Add(dst, base, Operand(reg_offset, extend, mem_op.GetShiftAmount())); 1717 } else { 1718 VIXL_ASSERT(extend == NO_EXTEND); 1719 Add(dst, base, Operand(reg_offset, shift, mem_op.GetShiftAmount())); 1720 } 1721 } 1722 } 1723 1724 1725 void MacroAssembler::AddSubMacro(const Register& rd, 1726 const Register& rn, 1727 const Operand& operand, 1728 FlagsUpdate S, 1729 AddSubOp op) { 1730 // Worst case is add/sub immediate: 1731 // * up to 4 instructions to materialise the constant 1732 // * 1 instruction for add/sub 1733 MacroEmissionCheckScope guard(this); 1734 1735 if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() && 1736 (S == LeaveFlags)) { 1737 // The instruction would be a nop. Avoid generating useless code. 1738 return; 1739 } 1740 1741 if ((operand.IsImmediate() && !IsImmAddSub(operand.GetImmediate())) || 1742 (rn.IsZero() && !operand.IsShiftedRegister()) || 1743 (operand.IsShiftedRegister() && (operand.GetShift() == ROR))) { 1744 UseScratchRegisterScope temps(this); 1745 Register temp = temps.AcquireSameSizeAs(rn); 1746 if (operand.IsImmediate()) { 1747 PreShiftImmMode mode = kAnyShift; 1748 1749 // If the destination or source register is the stack pointer, we can 1750 // only pre-shift the immediate right by values supported in the add/sub 1751 // extend encoding. 1752 if (rd.IsSP()) { 1753 // If the destination is SP and flags will be set, we can't pre-shift 1754 // the immediate at all. 1755 mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP; 1756 } else if (rn.IsSP()) { 1757 mode = kLimitShiftForSP; 1758 } 1759 1760 Operand imm_operand = 1761 MoveImmediateForShiftedOp(temp, operand.GetImmediate(), mode); 1762 AddSub(rd, rn, imm_operand, S, op); 1763 } else { 1764 Mov(temp, operand); 1765 AddSub(rd, rn, temp, S, op); 1766 } 1767 } else { 1768 AddSub(rd, rn, operand, S, op); 1769 } 1770 } 1771 1772 1773 void MacroAssembler::Adc(const Register& rd, 1774 const Register& rn, 1775 const Operand& operand) { 1776 VIXL_ASSERT(allow_macro_instructions_); 1777 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC); 1778 } 1779 1780 1781 void MacroAssembler::Adcs(const Register& rd, 1782 const Register& rn, 1783 const Operand& operand) { 1784 VIXL_ASSERT(allow_macro_instructions_); 1785 AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC); 1786 } 1787 1788 1789 void MacroAssembler::Sbc(const Register& rd, 1790 const Register& rn, 1791 const Operand& operand) { 1792 VIXL_ASSERT(allow_macro_instructions_); 1793 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC); 1794 } 1795 1796 1797 void MacroAssembler::Sbcs(const Register& rd, 1798 const Register& rn, 1799 const Operand& operand) { 1800 VIXL_ASSERT(allow_macro_instructions_); 1801 AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC); 1802 } 1803 1804 1805 void MacroAssembler::Ngc(const Register& rd, const Operand& operand) { 1806 VIXL_ASSERT(allow_macro_instructions_); 1807 Register zr = AppropriateZeroRegFor(rd); 1808 Sbc(rd, zr, operand); 1809 } 1810 1811 1812 void MacroAssembler::Ngcs(const Register& rd, const Operand& operand) { 1813 VIXL_ASSERT(allow_macro_instructions_); 1814 Register zr = AppropriateZeroRegFor(rd); 1815 Sbcs(rd, zr, operand); 1816 } 1817 1818 1819 void MacroAssembler::AddSubWithCarryMacro(const Register& rd, 1820 const Register& rn, 1821 const Operand& operand, 1822 FlagsUpdate S, 1823 AddSubWithCarryOp op) { 1824 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits()); 1825 // Worst case is addc/subc immediate: 1826 // * up to 4 instructions to materialise the constant 1827 // * 1 instruction for add/sub 1828 MacroEmissionCheckScope guard(this); 1829 UseScratchRegisterScope temps(this); 1830 1831 if (operand.IsImmediate() || 1832 (operand.IsShiftedRegister() && (operand.GetShift() == ROR))) { 1833 // Add/sub with carry (immediate or ROR shifted register.) 1834 Register temp = temps.AcquireSameSizeAs(rn); 1835 Mov(temp, operand); 1836 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1837 } else if (operand.IsShiftedRegister() && (operand.GetShiftAmount() != 0)) { 1838 // Add/sub with carry (shifted register). 1839 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits()); 1840 VIXL_ASSERT(operand.GetShift() != ROR); 1841 VIXL_ASSERT( 1842 IsUintN(rd.GetSizeInBits() == kXRegSize ? kXRegSizeLog2 : kWRegSizeLog2, 1843 operand.GetShiftAmount())); 1844 temps.Exclude(operand.GetRegister()); 1845 Register temp = temps.AcquireSameSizeAs(rn); 1846 EmitShift(temp, 1847 operand.GetRegister(), 1848 operand.GetShift(), 1849 operand.GetShiftAmount()); 1850 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1851 } else if (operand.IsExtendedRegister()) { 1852 // Add/sub with carry (extended register). 1853 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() <= rd.GetSizeInBits()); 1854 // Add/sub extended supports a shift <= 4. We want to support exactly the 1855 // same modes. 1856 VIXL_ASSERT(operand.GetShiftAmount() <= 4); 1857 VIXL_ASSERT( 1858 operand.GetRegister().Is64Bits() || 1859 ((operand.GetExtend() != UXTX) && (operand.GetExtend() != SXTX))); 1860 temps.Exclude(operand.GetRegister()); 1861 Register temp = temps.AcquireSameSizeAs(rn); 1862 EmitExtendShift(temp, 1863 operand.GetRegister(), 1864 operand.GetExtend(), 1865 operand.GetShiftAmount()); 1866 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1867 } else { 1868 // The addressing mode is directly supported by the instruction. 1869 AddSubWithCarry(rd, rn, operand, S, op); 1870 } 1871 } 1872 1873 1874 #define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \ 1875 void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \ 1876 VIXL_ASSERT(allow_macro_instructions_); \ 1877 LoadStoreMacro(REG, addr, OP); \ 1878 } 1879 LS_MACRO_LIST(DEFINE_FUNCTION) 1880 #undef DEFINE_FUNCTION 1881 1882 1883 void MacroAssembler::LoadStoreMacro(const CPURegister& rt, 1884 const MemOperand& addr, 1885 LoadStoreOp op) { 1886 // Worst case is ldr/str pre/post index: 1887 // * 1 instruction for ldr/str 1888 // * up to 4 instructions to materialise the constant 1889 // * 1 instruction to update the base 1890 MacroEmissionCheckScope guard(this); 1891 1892 int64_t offset = addr.GetOffset(); 1893 unsigned access_size = CalcLSDataSize(op); 1894 1895 // Check if an immediate offset fits in the immediate field of the 1896 // appropriate instruction. If not, emit two instructions to perform 1897 // the operation. 1898 if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, access_size) && 1899 !IsImmLSUnscaled(offset)) { 1900 // Immediate offset that can't be encoded using unsigned or unscaled 1901 // addressing modes. 1902 UseScratchRegisterScope temps(this); 1903 Register temp = temps.AcquireSameSizeAs(addr.GetBaseRegister()); 1904 Mov(temp, addr.GetOffset()); 1905 LoadStore(rt, MemOperand(addr.GetBaseRegister(), temp), op); 1906 } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) { 1907 // Post-index beyond unscaled addressing range. 1908 LoadStore(rt, MemOperand(addr.GetBaseRegister()), op); 1909 Add(addr.GetBaseRegister(), addr.GetBaseRegister(), Operand(offset)); 1910 } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) { 1911 // Pre-index beyond unscaled addressing range. 1912 Add(addr.GetBaseRegister(), addr.GetBaseRegister(), Operand(offset)); 1913 LoadStore(rt, MemOperand(addr.GetBaseRegister()), op); 1914 } else { 1915 // Encodable in one load/store instruction. 1916 LoadStore(rt, addr, op); 1917 } 1918 } 1919 1920 1921 #define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \ 1922 void MacroAssembler::FN(const REGTYPE REG, \ 1923 const REGTYPE REG2, \ 1924 const MemOperand& addr) { \ 1925 VIXL_ASSERT(allow_macro_instructions_); \ 1926 LoadStorePairMacro(REG, REG2, addr, OP); \ 1927 } 1928 LSPAIR_MACRO_LIST(DEFINE_FUNCTION) 1929 #undef DEFINE_FUNCTION 1930 1931 void MacroAssembler::LoadStorePairMacro(const CPURegister& rt, 1932 const CPURegister& rt2, 1933 const MemOperand& addr, 1934 LoadStorePairOp op) { 1935 // TODO(all): Should we support register offset for load-store-pair? 1936 VIXL_ASSERT(!addr.IsRegisterOffset()); 1937 // Worst case is ldp/stp immediate: 1938 // * 1 instruction for ldp/stp 1939 // * up to 4 instructions to materialise the constant 1940 // * 1 instruction to update the base 1941 MacroEmissionCheckScope guard(this); 1942 1943 int64_t offset = addr.GetOffset(); 1944 unsigned access_size = CalcLSPairDataSize(op); 1945 1946 // Check if the offset fits in the immediate field of the appropriate 1947 // instruction. If not, emit two instructions to perform the operation. 1948 if (IsImmLSPair(offset, access_size)) { 1949 // Encodable in one load/store pair instruction. 1950 LoadStorePair(rt, rt2, addr, op); 1951 } else { 1952 Register base = addr.GetBaseRegister(); 1953 if (addr.IsImmediateOffset()) { 1954 UseScratchRegisterScope temps(this); 1955 Register temp = temps.AcquireSameSizeAs(base); 1956 Add(temp, base, offset); 1957 LoadStorePair(rt, rt2, MemOperand(temp), op); 1958 } else if (addr.IsPostIndex()) { 1959 LoadStorePair(rt, rt2, MemOperand(base), op); 1960 Add(base, base, offset); 1961 } else { 1962 VIXL_ASSERT(addr.IsPreIndex()); 1963 Add(base, base, offset); 1964 LoadStorePair(rt, rt2, MemOperand(base), op); 1965 } 1966 } 1967 } 1968 1969 1970 void MacroAssembler::Prfm(PrefetchOperation op, const MemOperand& addr) { 1971 MacroEmissionCheckScope guard(this); 1972 1973 // There are no pre- or post-index modes for prfm. 1974 VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsRegisterOffset()); 1975 1976 // The access size is implicitly 8 bytes for all prefetch operations. 1977 unsigned size = kXRegSizeInBytesLog2; 1978 1979 // Check if an immediate offset fits in the immediate field of the 1980 // appropriate instruction. If not, emit two instructions to perform 1981 // the operation. 1982 if (addr.IsImmediateOffset() && !IsImmLSScaled(addr.GetOffset(), size) && 1983 !IsImmLSUnscaled(addr.GetOffset())) { 1984 // Immediate offset that can't be encoded using unsigned or unscaled 1985 // addressing modes. 1986 UseScratchRegisterScope temps(this); 1987 Register temp = temps.AcquireSameSizeAs(addr.GetBaseRegister()); 1988 Mov(temp, addr.GetOffset()); 1989 Prefetch(op, MemOperand(addr.GetBaseRegister(), temp)); 1990 } else { 1991 // Simple register-offsets are encodable in one instruction. 1992 Prefetch(op, addr); 1993 } 1994 } 1995 1996 1997 void MacroAssembler::Push(const CPURegister& src0, 1998 const CPURegister& src1, 1999 const CPURegister& src2, 2000 const CPURegister& src3) { 2001 VIXL_ASSERT(allow_macro_instructions_); 2002 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3)); 2003 VIXL_ASSERT(src0.IsValid()); 2004 2005 int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid(); 2006 int size = src0.GetSizeInBytes(); 2007 2008 PrepareForPush(count, size); 2009 PushHelper(count, size, src0, src1, src2, src3); 2010 } 2011 2012 2013 void MacroAssembler::Pop(const CPURegister& dst0, 2014 const CPURegister& dst1, 2015 const CPURegister& dst2, 2016 const CPURegister& dst3) { 2017 // It is not valid to pop into the same register more than once in one 2018 // instruction, not even into the zero register. 2019 VIXL_ASSERT(allow_macro_instructions_); 2020 VIXL_ASSERT(!AreAliased(dst0, dst1, dst2, dst3)); 2021 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3)); 2022 VIXL_ASSERT(dst0.IsValid()); 2023 2024 int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid(); 2025 int size = dst0.GetSizeInBytes(); 2026 2027 PrepareForPop(count, size); 2028 PopHelper(count, size, dst0, dst1, dst2, dst3); 2029 } 2030 2031 2032 void MacroAssembler::PushCPURegList(CPURegList registers) { 2033 VIXL_ASSERT(!registers.Overlaps(*GetScratchRegisterList())); 2034 VIXL_ASSERT(!registers.Overlaps(*GetScratchFPRegisterList())); 2035 VIXL_ASSERT(allow_macro_instructions_); 2036 2037 int reg_size = registers.GetRegisterSizeInBytes(); 2038 PrepareForPush(registers.GetCount(), reg_size); 2039 2040 // Bump the stack pointer and store two registers at the bottom. 2041 int size = registers.GetTotalSizeInBytes(); 2042 const CPURegister& bottom_0 = registers.PopLowestIndex(); 2043 const CPURegister& bottom_1 = registers.PopLowestIndex(); 2044 if (bottom_0.IsValid() && bottom_1.IsValid()) { 2045 Stp(bottom_0, bottom_1, MemOperand(StackPointer(), -size, PreIndex)); 2046 } else if (bottom_0.IsValid()) { 2047 Str(bottom_0, MemOperand(StackPointer(), -size, PreIndex)); 2048 } 2049 2050 int offset = 2 * reg_size; 2051 while (!registers.IsEmpty()) { 2052 const CPURegister& src0 = registers.PopLowestIndex(); 2053 const CPURegister& src1 = registers.PopLowestIndex(); 2054 if (src1.IsValid()) { 2055 Stp(src0, src1, MemOperand(StackPointer(), offset)); 2056 } else { 2057 Str(src0, MemOperand(StackPointer(), offset)); 2058 } 2059 offset += 2 * reg_size; 2060 } 2061 } 2062 2063 2064 void MacroAssembler::PopCPURegList(CPURegList registers) { 2065 VIXL_ASSERT(!registers.Overlaps(*GetScratchRegisterList())); 2066 VIXL_ASSERT(!registers.Overlaps(*GetScratchFPRegisterList())); 2067 VIXL_ASSERT(allow_macro_instructions_); 2068 2069 int reg_size = registers.GetRegisterSizeInBytes(); 2070 PrepareForPop(registers.GetCount(), reg_size); 2071 2072 2073 int size = registers.GetTotalSizeInBytes(); 2074 const CPURegister& bottom_0 = registers.PopLowestIndex(); 2075 const CPURegister& bottom_1 = registers.PopLowestIndex(); 2076 2077 int offset = 2 * reg_size; 2078 while (!registers.IsEmpty()) { 2079 const CPURegister& dst0 = registers.PopLowestIndex(); 2080 const CPURegister& dst1 = registers.PopLowestIndex(); 2081 if (dst1.IsValid()) { 2082 Ldp(dst0, dst1, MemOperand(StackPointer(), offset)); 2083 } else { 2084 Ldr(dst0, MemOperand(StackPointer(), offset)); 2085 } 2086 offset += 2 * reg_size; 2087 } 2088 2089 // Load the two registers at the bottom and drop the stack pointer. 2090 if (bottom_0.IsValid() && bottom_1.IsValid()) { 2091 Ldp(bottom_0, bottom_1, MemOperand(StackPointer(), size, PostIndex)); 2092 } else if (bottom_0.IsValid()) { 2093 Ldr(bottom_0, MemOperand(StackPointer(), size, PostIndex)); 2094 } 2095 } 2096 2097 2098 void MacroAssembler::PushMultipleTimes(int count, Register src) { 2099 VIXL_ASSERT(allow_macro_instructions_); 2100 int size = src.GetSizeInBytes(); 2101 2102 PrepareForPush(count, size); 2103 // Push up to four registers at a time if possible because if the current 2104 // stack pointer is sp and the register size is 32, registers must be pushed 2105 // in blocks of four in order to maintain the 16-byte alignment for sp. 2106 while (count >= 4) { 2107 PushHelper(4, size, src, src, src, src); 2108 count -= 4; 2109 } 2110 if (count >= 2) { 2111 PushHelper(2, size, src, src, NoReg, NoReg); 2112 count -= 2; 2113 } 2114 if (count == 1) { 2115 PushHelper(1, size, src, NoReg, NoReg, NoReg); 2116 count -= 1; 2117 } 2118 VIXL_ASSERT(count == 0); 2119 } 2120 2121 2122 void MacroAssembler::PushHelper(int count, 2123 int size, 2124 const CPURegister& src0, 2125 const CPURegister& src1, 2126 const CPURegister& src2, 2127 const CPURegister& src3) { 2128 // Ensure that we don't unintentionally modify scratch or debug registers. 2129 // Worst case for size is 2 stp. 2130 ExactAssemblyScope scope(this, 2131 2 * kInstructionSize, 2132 ExactAssemblyScope::kMaximumSize); 2133 2134 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3)); 2135 VIXL_ASSERT(size == src0.GetSizeInBytes()); 2136 2137 // When pushing multiple registers, the store order is chosen such that 2138 // Push(a, b) is equivalent to Push(a) followed by Push(b). 2139 switch (count) { 2140 case 1: 2141 VIXL_ASSERT(src1.IsNone() && src2.IsNone() && src3.IsNone()); 2142 str(src0, MemOperand(StackPointer(), -1 * size, PreIndex)); 2143 break; 2144 case 2: 2145 VIXL_ASSERT(src2.IsNone() && src3.IsNone()); 2146 stp(src1, src0, MemOperand(StackPointer(), -2 * size, PreIndex)); 2147 break; 2148 case 3: 2149 VIXL_ASSERT(src3.IsNone()); 2150 stp(src2, src1, MemOperand(StackPointer(), -3 * size, PreIndex)); 2151 str(src0, MemOperand(StackPointer(), 2 * size)); 2152 break; 2153 case 4: 2154 // Skip over 4 * size, then fill in the gap. This allows four W registers 2155 // to be pushed using sp, whilst maintaining 16-byte alignment for sp at 2156 // all times. 2157 stp(src3, src2, MemOperand(StackPointer(), -4 * size, PreIndex)); 2158 stp(src1, src0, MemOperand(StackPointer(), 2 * size)); 2159 break; 2160 default: 2161 VIXL_UNREACHABLE(); 2162 } 2163 } 2164 2165 2166 void MacroAssembler::PopHelper(int count, 2167 int size, 2168 const CPURegister& dst0, 2169 const CPURegister& dst1, 2170 const CPURegister& dst2, 2171 const CPURegister& dst3) { 2172 // Ensure that we don't unintentionally modify scratch or debug registers. 2173 // Worst case for size is 2 ldp. 2174 ExactAssemblyScope scope(this, 2175 2 * kInstructionSize, 2176 ExactAssemblyScope::kMaximumSize); 2177 2178 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3)); 2179 VIXL_ASSERT(size == dst0.GetSizeInBytes()); 2180 2181 // When popping multiple registers, the load order is chosen such that 2182 // Pop(a, b) is equivalent to Pop(a) followed by Pop(b). 2183 switch (count) { 2184 case 1: 2185 VIXL_ASSERT(dst1.IsNone() && dst2.IsNone() && dst3.IsNone()); 2186 ldr(dst0, MemOperand(StackPointer(), 1 * size, PostIndex)); 2187 break; 2188 case 2: 2189 VIXL_ASSERT(dst2.IsNone() && dst3.IsNone()); 2190 ldp(dst0, dst1, MemOperand(StackPointer(), 2 * size, PostIndex)); 2191 break; 2192 case 3: 2193 VIXL_ASSERT(dst3.IsNone()); 2194 ldr(dst2, MemOperand(StackPointer(), 2 * size)); 2195 ldp(dst0, dst1, MemOperand(StackPointer(), 3 * size, PostIndex)); 2196 break; 2197 case 4: 2198 // Load the higher addresses first, then load the lower addresses and skip 2199 // the whole block in the second instruction. This allows four W registers 2200 // to be popped using sp, whilst maintaining 16-byte alignment for sp at 2201 // all times. 2202 ldp(dst2, dst3, MemOperand(StackPointer(), 2 * size)); 2203 ldp(dst0, dst1, MemOperand(StackPointer(), 4 * size, PostIndex)); 2204 break; 2205 default: 2206 VIXL_UNREACHABLE(); 2207 } 2208 } 2209 2210 2211 void MacroAssembler::PrepareForPush(int count, int size) { 2212 if (sp.Is(StackPointer())) { 2213 // If the current stack pointer is sp, then it must be aligned to 16 bytes 2214 // on entry and the total size of the specified registers must also be a 2215 // multiple of 16 bytes. 2216 VIXL_ASSERT((count * size) % 16 == 0); 2217 } else { 2218 // Even if the current stack pointer is not the system stack pointer (sp), 2219 // the system stack pointer will still be modified in order to comply with 2220 // ABI rules about accessing memory below the system stack pointer. 2221 BumpSystemStackPointer(count * size); 2222 } 2223 } 2224 2225 2226 void MacroAssembler::PrepareForPop(int count, int size) { 2227 USE(count, size); 2228 if (sp.Is(StackPointer())) { 2229 // If the current stack pointer is sp, then it must be aligned to 16 bytes 2230 // on entry and the total size of the specified registers must also be a 2231 // multiple of 16 bytes. 2232 VIXL_ASSERT((count * size) % 16 == 0); 2233 } 2234 } 2235 2236 void MacroAssembler::Poke(const Register& src, const Operand& offset) { 2237 VIXL_ASSERT(allow_macro_instructions_); 2238 if (offset.IsImmediate()) { 2239 VIXL_ASSERT(offset.GetImmediate() >= 0); 2240 } 2241 2242 Str(src, MemOperand(StackPointer(), offset)); 2243 } 2244 2245 2246 void MacroAssembler::Peek(const Register& dst, const Operand& offset) { 2247 VIXL_ASSERT(allow_macro_instructions_); 2248 if (offset.IsImmediate()) { 2249 VIXL_ASSERT(offset.GetImmediate() >= 0); 2250 } 2251 2252 Ldr(dst, MemOperand(StackPointer(), offset)); 2253 } 2254 2255 2256 void MacroAssembler::Claim(const Operand& size) { 2257 VIXL_ASSERT(allow_macro_instructions_); 2258 2259 if (size.IsZero()) { 2260 return; 2261 } 2262 2263 if (size.IsImmediate()) { 2264 VIXL_ASSERT(size.GetImmediate() > 0); 2265 if (sp.Is(StackPointer())) { 2266 VIXL_ASSERT((size.GetImmediate() % 16) == 0); 2267 } 2268 } 2269 2270 if (!sp.Is(StackPointer())) { 2271 BumpSystemStackPointer(size); 2272 } 2273 2274 Sub(StackPointer(), StackPointer(), size); 2275 } 2276 2277 2278 void MacroAssembler::Drop(const Operand& size) { 2279 VIXL_ASSERT(allow_macro_instructions_); 2280 2281 if (size.IsZero()) { 2282 return; 2283 } 2284 2285 if (size.IsImmediate()) { 2286 VIXL_ASSERT(size.GetImmediate() > 0); 2287 if (sp.Is(StackPointer())) { 2288 VIXL_ASSERT((size.GetImmediate() % 16) == 0); 2289 } 2290 } 2291 2292 Add(StackPointer(), StackPointer(), size); 2293 } 2294 2295 2296 void MacroAssembler::PushCalleeSavedRegisters() { 2297 // Ensure that the macro-assembler doesn't use any scratch registers. 2298 // 10 stp will be emitted. 2299 // TODO(all): Should we use GetCalleeSaved and SavedFP. 2300 ExactAssemblyScope scope(this, 10 * kInstructionSize); 2301 2302 // This method must not be called unless the current stack pointer is sp. 2303 VIXL_ASSERT(sp.Is(StackPointer())); 2304 2305 MemOperand tos(sp, -2 * static_cast<int>(kXRegSizeInBytes), PreIndex); 2306 2307 stp(x29, x30, tos); 2308 stp(x27, x28, tos); 2309 stp(x25, x26, tos); 2310 stp(x23, x24, tos); 2311 stp(x21, x22, tos); 2312 stp(x19, x20, tos); 2313 2314 stp(d14, d15, tos); 2315 stp(d12, d13, tos); 2316 stp(d10, d11, tos); 2317 stp(d8, d9, tos); 2318 } 2319 2320 2321 void MacroAssembler::PopCalleeSavedRegisters() { 2322 // Ensure that the macro-assembler doesn't use any scratch registers. 2323 // 10 ldp will be emitted. 2324 // TODO(all): Should we use GetCalleeSaved and SavedFP. 2325 ExactAssemblyScope scope(this, 10 * kInstructionSize); 2326 2327 // This method must not be called unless the current stack pointer is sp. 2328 VIXL_ASSERT(sp.Is(StackPointer())); 2329 2330 MemOperand tos(sp, 2 * kXRegSizeInBytes, PostIndex); 2331 2332 ldp(d8, d9, tos); 2333 ldp(d10, d11, tos); 2334 ldp(d12, d13, tos); 2335 ldp(d14, d15, tos); 2336 2337 ldp(x19, x20, tos); 2338 ldp(x21, x22, tos); 2339 ldp(x23, x24, tos); 2340 ldp(x25, x26, tos); 2341 ldp(x27, x28, tos); 2342 ldp(x29, x30, tos); 2343 } 2344 2345 void MacroAssembler::LoadCPURegList(CPURegList registers, 2346 const MemOperand& src) { 2347 LoadStoreCPURegListHelper(kLoad, registers, src); 2348 } 2349 2350 void MacroAssembler::StoreCPURegList(CPURegList registers, 2351 const MemOperand& dst) { 2352 LoadStoreCPURegListHelper(kStore, registers, dst); 2353 } 2354 2355 2356 void MacroAssembler::LoadStoreCPURegListHelper(LoadStoreCPURegListAction op, 2357 CPURegList registers, 2358 const MemOperand& mem) { 2359 // We do not handle pre-indexing or post-indexing. 2360 VIXL_ASSERT(!(mem.IsPreIndex() || mem.IsPostIndex())); 2361 VIXL_ASSERT(!registers.Overlaps(tmp_list_)); 2362 VIXL_ASSERT(!registers.Overlaps(fptmp_list_)); 2363 VIXL_ASSERT(!registers.IncludesAliasOf(sp)); 2364 2365 UseScratchRegisterScope temps(this); 2366 2367 MemOperand loc = BaseMemOperandForLoadStoreCPURegList(registers, mem, &temps); 2368 const int reg_size = registers.GetRegisterSizeInBytes(); 2369 2370 VIXL_ASSERT(IsPowerOf2(reg_size)); 2371 2372 // Since we are operating on register pairs, we would like to align on double 2373 // the standard size; on the other hand, we don't want to insert an extra 2374 // operation, which will happen if the number of registers is even. Note that 2375 // the alignment of the base pointer is unknown here, but we assume that it 2376 // is more likely to be aligned. 2377 if (((loc.GetOffset() & (2 * reg_size - 1)) != 0) && 2378 ((registers.GetCount() % 2) != 0)) { 2379 if (op == kStore) { 2380 Str(registers.PopLowestIndex(), loc); 2381 } else { 2382 VIXL_ASSERT(op == kLoad); 2383 Ldr(registers.PopLowestIndex(), loc); 2384 } 2385 loc.AddOffset(reg_size); 2386 } 2387 while (registers.GetCount() >= 2) { 2388 const CPURegister& dst0 = registers.PopLowestIndex(); 2389 const CPURegister& dst1 = registers.PopLowestIndex(); 2390 if (op == kStore) { 2391 Stp(dst0, dst1, loc); 2392 } else { 2393 VIXL_ASSERT(op == kLoad); 2394 Ldp(dst0, dst1, loc); 2395 } 2396 loc.AddOffset(2 * reg_size); 2397 } 2398 if (!registers.IsEmpty()) { 2399 if (op == kStore) { 2400 Str(registers.PopLowestIndex(), loc); 2401 } else { 2402 VIXL_ASSERT(op == kLoad); 2403 Ldr(registers.PopLowestIndex(), loc); 2404 } 2405 } 2406 } 2407 2408 MemOperand MacroAssembler::BaseMemOperandForLoadStoreCPURegList( 2409 const CPURegList& registers, 2410 const MemOperand& mem, 2411 UseScratchRegisterScope* scratch_scope) { 2412 // If necessary, pre-compute the base address for the accesses. 2413 if (mem.IsRegisterOffset()) { 2414 Register reg_base = scratch_scope->AcquireX(); 2415 ComputeAddress(reg_base, mem); 2416 return MemOperand(reg_base); 2417 2418 } else if (mem.IsImmediateOffset()) { 2419 int reg_size = registers.GetRegisterSizeInBytes(); 2420 int total_size = registers.GetTotalSizeInBytes(); 2421 int64_t min_offset = mem.GetOffset(); 2422 int64_t max_offset = 2423 mem.GetOffset() + std::max(0, total_size - 2 * reg_size); 2424 if ((registers.GetCount() >= 2) && 2425 (!Assembler::IsImmLSPair(min_offset, WhichPowerOf2(reg_size)) || 2426 !Assembler::IsImmLSPair(max_offset, WhichPowerOf2(reg_size)))) { 2427 Register reg_base = scratch_scope->AcquireX(); 2428 ComputeAddress(reg_base, mem); 2429 return MemOperand(reg_base); 2430 } 2431 } 2432 2433 return mem; 2434 } 2435 2436 void MacroAssembler::BumpSystemStackPointer(const Operand& space) { 2437 VIXL_ASSERT(!sp.Is(StackPointer())); 2438 // TODO: Several callers rely on this not using scratch registers, so we use 2439 // the assembler directly here. However, this means that large immediate 2440 // values of 'space' cannot be handled. 2441 ExactAssemblyScope scope(this, kInstructionSize); 2442 sub(sp, StackPointer(), space); 2443 } 2444 2445 2446 // TODO(all): Fix printf for NEON registers, and resolve whether we should be 2447 // using FPRegister or VRegister here. 2448 2449 // This is the main Printf implementation. All callee-saved registers are 2450 // preserved, but NZCV and the caller-saved registers may be clobbered. 2451 void MacroAssembler::PrintfNoPreserve(const char* format, 2452 const CPURegister& arg0, 2453 const CPURegister& arg1, 2454 const CPURegister& arg2, 2455 const CPURegister& arg3) { 2456 // We cannot handle a caller-saved stack pointer. It doesn't make much sense 2457 // in most cases anyway, so this restriction shouldn't be too serious. 2458 VIXL_ASSERT(!kCallerSaved.IncludesAliasOf(StackPointer())); 2459 2460 // The provided arguments, and their proper PCS registers. 2461 CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3}; 2462 CPURegister pcs[kPrintfMaxArgCount]; 2463 2464 int arg_count = kPrintfMaxArgCount; 2465 2466 // The PCS varargs registers for printf. Note that x0 is used for the printf 2467 // format string. 2468 static const CPURegList kPCSVarargs = 2469 CPURegList(CPURegister::kRegister, kXRegSize, 1, arg_count); 2470 static const CPURegList kPCSVarargsFP = 2471 CPURegList(CPURegister::kVRegister, kDRegSize, 0, arg_count - 1); 2472 2473 // We can use caller-saved registers as scratch values, except for the 2474 // arguments and the PCS registers where they might need to go. 2475 UseScratchRegisterScope temps(this); 2476 temps.Include(kCallerSaved); 2477 temps.Include(kCallerSavedV); 2478 temps.Exclude(kPCSVarargs); 2479 temps.Exclude(kPCSVarargsFP); 2480 temps.Exclude(arg0, arg1, arg2, arg3); 2481 2482 // Copies of the arg lists that we can iterate through. 2483 CPURegList pcs_varargs = kPCSVarargs; 2484 CPURegList pcs_varargs_fp = kPCSVarargsFP; 2485 2486 // Place the arguments. There are lots of clever tricks and optimizations we 2487 // could use here, but Printf is a debug tool so instead we just try to keep 2488 // it simple: Move each input that isn't already in the right place to a 2489 // scratch register, then move everything back. 2490 for (unsigned i = 0; i < kPrintfMaxArgCount; i++) { 2491 // Work out the proper PCS register for this argument. 2492 if (args[i].IsRegister()) { 2493 pcs[i] = pcs_varargs.PopLowestIndex().X(); 2494 // We might only need a W register here. We need to know the size of the 2495 // argument so we can properly encode it for the simulator call. 2496 if (args[i].Is32Bits()) pcs[i] = pcs[i].W(); 2497 } else if (args[i].IsVRegister()) { 2498 // In C, floats are always cast to doubles for varargs calls. 2499 pcs[i] = pcs_varargs_fp.PopLowestIndex().D(); 2500 } else { 2501 VIXL_ASSERT(args[i].IsNone()); 2502 arg_count = i; 2503 break; 2504 } 2505 2506 // If the argument is already in the right place, leave it where it is. 2507 if (args[i].Aliases(pcs[i])) continue; 2508 2509 // Otherwise, if the argument is in a PCS argument register, allocate an 2510 // appropriate scratch register and then move it out of the way. 2511 if (kPCSVarargs.IncludesAliasOf(args[i]) || 2512 kPCSVarargsFP.IncludesAliasOf(args[i])) { 2513 if (args[i].IsRegister()) { 2514 Register old_arg = Register(args[i]); 2515 Register new_arg = temps.AcquireSameSizeAs(old_arg); 2516 Mov(new_arg, old_arg); 2517 args[i] = new_arg; 2518 } else { 2519 FPRegister old_arg = FPRegister(args[i]); 2520 FPRegister new_arg = temps.AcquireSameSizeAs(old_arg); 2521 Fmov(new_arg, old_arg); 2522 args[i] = new_arg; 2523 } 2524 } 2525 } 2526 2527 // Do a second pass to move values into their final positions and perform any 2528 // conversions that may be required. 2529 for (int i = 0; i < arg_count; i++) { 2530 VIXL_ASSERT(pcs[i].GetType() == args[i].GetType()); 2531 if (pcs[i].IsRegister()) { 2532 Mov(Register(pcs[i]), Register(args[i]), kDiscardForSameWReg); 2533 } else { 2534 VIXL_ASSERT(pcs[i].IsVRegister()); 2535 if (pcs[i].GetSizeInBits() == args[i].GetSizeInBits()) { 2536 Fmov(FPRegister(pcs[i]), FPRegister(args[i])); 2537 } else { 2538 Fcvt(FPRegister(pcs[i]), FPRegister(args[i])); 2539 } 2540 } 2541 } 2542 2543 // Load the format string into x0, as per the procedure-call standard. 2544 // 2545 // To make the code as portable as possible, the format string is encoded 2546 // directly in the instruction stream. It might be cleaner to encode it in a 2547 // literal pool, but since Printf is usually used for debugging, it is 2548 // beneficial for it to be minimally dependent on other features. 2549 temps.Exclude(x0); 2550 Label format_address; 2551 Adr(x0, &format_address); 2552 2553 // Emit the format string directly in the instruction stream. 2554 { 2555 BlockPoolsScope scope(this); 2556 // Data emitted: 2557 // branch 2558 // strlen(format) + 1 (includes null termination) 2559 // padding to next instruction 2560 // unreachable 2561 EmissionCheckScope guard(this, 2562 AlignUp(strlen(format) + 1, kInstructionSize) + 2563 2 * kInstructionSize); 2564 Label after_data; 2565 B(&after_data); 2566 Bind(&format_address); 2567 EmitString(format); 2568 Unreachable(); 2569 Bind(&after_data); 2570 } 2571 2572 // We don't pass any arguments on the stack, but we still need to align the C 2573 // stack pointer to a 16-byte boundary for PCS compliance. 2574 if (!sp.Is(StackPointer())) { 2575 Bic(sp, StackPointer(), 0xf); 2576 } 2577 2578 // Actually call printf. This part needs special handling for the simulator, 2579 // since the system printf function will use a different instruction set and 2580 // the procedure-call standard will not be compatible. 2581 if (generate_simulator_code_) { 2582 ExactAssemblyScope scope(this, kPrintfLength); 2583 hlt(kPrintfOpcode); 2584 dc32(arg_count); // kPrintfArgCountOffset 2585 2586 // Determine the argument pattern. 2587 uint32_t arg_pattern_list = 0; 2588 for (int i = 0; i < arg_count; i++) { 2589 uint32_t arg_pattern; 2590 if (pcs[i].IsRegister()) { 2591 arg_pattern = pcs[i].Is32Bits() ? kPrintfArgW : kPrintfArgX; 2592 } else { 2593 VIXL_ASSERT(pcs[i].Is64Bits()); 2594 arg_pattern = kPrintfArgD; 2595 } 2596 VIXL_ASSERT(arg_pattern < (1 << kPrintfArgPatternBits)); 2597 arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i)); 2598 } 2599 dc32(arg_pattern_list); // kPrintfArgPatternListOffset 2600 } else { 2601 Register tmp = temps.AcquireX(); 2602 Mov(tmp, reinterpret_cast<uintptr_t>(printf)); 2603 Blr(tmp); 2604 } 2605 } 2606 2607 2608 void MacroAssembler::Printf(const char* format, 2609 CPURegister arg0, 2610 CPURegister arg1, 2611 CPURegister arg2, 2612 CPURegister arg3) { 2613 // We can only print sp if it is the current stack pointer. 2614 if (!sp.Is(StackPointer())) { 2615 VIXL_ASSERT(!sp.Aliases(arg0)); 2616 VIXL_ASSERT(!sp.Aliases(arg1)); 2617 VIXL_ASSERT(!sp.Aliases(arg2)); 2618 VIXL_ASSERT(!sp.Aliases(arg3)); 2619 } 2620 2621 // Make sure that the macro assembler doesn't try to use any of our arguments 2622 // as scratch registers. 2623 UseScratchRegisterScope exclude_all(this); 2624 exclude_all.ExcludeAll(); 2625 2626 // Preserve all caller-saved registers as well as NZCV. 2627 // If sp is the stack pointer, PushCPURegList asserts that the size of each 2628 // list is a multiple of 16 bytes. 2629 PushCPURegList(kCallerSaved); 2630 PushCPURegList(kCallerSavedV); 2631 2632 { 2633 UseScratchRegisterScope temps(this); 2634 // We can use caller-saved registers as scratch values (except for argN). 2635 temps.Include(kCallerSaved); 2636 temps.Include(kCallerSavedV); 2637 temps.Exclude(arg0, arg1, arg2, arg3); 2638 2639 // If any of the arguments are the current stack pointer, allocate a new 2640 // register for them, and adjust the value to compensate for pushing the 2641 // caller-saved registers. 2642 bool arg0_sp = StackPointer().Aliases(arg0); 2643 bool arg1_sp = StackPointer().Aliases(arg1); 2644 bool arg2_sp = StackPointer().Aliases(arg2); 2645 bool arg3_sp = StackPointer().Aliases(arg3); 2646 if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) { 2647 // Allocate a register to hold the original stack pointer value, to pass 2648 // to PrintfNoPreserve as an argument. 2649 Register arg_sp = temps.AcquireX(); 2650 Add(arg_sp, 2651 StackPointer(), 2652 kCallerSaved.GetTotalSizeInBytes() + 2653 kCallerSavedV.GetTotalSizeInBytes()); 2654 if (arg0_sp) arg0 = Register(arg_sp.GetCode(), arg0.GetSizeInBits()); 2655 if (arg1_sp) arg1 = Register(arg_sp.GetCode(), arg1.GetSizeInBits()); 2656 if (arg2_sp) arg2 = Register(arg_sp.GetCode(), arg2.GetSizeInBits()); 2657 if (arg3_sp) arg3 = Register(arg_sp.GetCode(), arg3.GetSizeInBits()); 2658 } 2659 2660 // Preserve NZCV. 2661 Register tmp = temps.AcquireX(); 2662 Mrs(tmp, NZCV); 2663 Push(tmp, xzr); 2664 temps.Release(tmp); 2665 2666 PrintfNoPreserve(format, arg0, arg1, arg2, arg3); 2667 2668 // Restore NZCV. 2669 tmp = temps.AcquireX(); 2670 Pop(xzr, tmp); 2671 Msr(NZCV, tmp); 2672 temps.Release(tmp); 2673 } 2674 2675 PopCPURegList(kCallerSavedV); 2676 PopCPURegList(kCallerSaved); 2677 } 2678 2679 void MacroAssembler::Trace(TraceParameters parameters, TraceCommand command) { 2680 VIXL_ASSERT(allow_macro_instructions_); 2681 2682 if (generate_simulator_code_) { 2683 // The arguments to the trace pseudo instruction need to be contiguous in 2684 // memory, so make sure we don't try to emit a literal pool. 2685 ExactAssemblyScope scope(this, kTraceLength); 2686 2687 Label start; 2688 bind(&start); 2689 2690 // Refer to simulator-aarch64.h for a description of the marker and its 2691 // arguments. 2692 hlt(kTraceOpcode); 2693 2694 VIXL_ASSERT(GetSizeOfCodeGeneratedSince(&start) == kTraceParamsOffset); 2695 dc32(parameters); 2696 2697 VIXL_ASSERT(GetSizeOfCodeGeneratedSince(&start) == kTraceCommandOffset); 2698 dc32(command); 2699 } else { 2700 // Emit nothing on real hardware. 2701 USE(parameters, command); 2702 } 2703 } 2704 2705 2706 void MacroAssembler::Log(TraceParameters parameters) { 2707 VIXL_ASSERT(allow_macro_instructions_); 2708 2709 if (generate_simulator_code_) { 2710 // The arguments to the log pseudo instruction need to be contiguous in 2711 // memory, so make sure we don't try to emit a literal pool. 2712 ExactAssemblyScope scope(this, kLogLength); 2713 2714 Label start; 2715 bind(&start); 2716 2717 // Refer to simulator-aarch64.h for a description of the marker and its 2718 // arguments. 2719 hlt(kLogOpcode); 2720 2721 VIXL_ASSERT(GetSizeOfCodeGeneratedSince(&start) == kLogParamsOffset); 2722 dc32(parameters); 2723 } else { 2724 // Emit nothing on real hardware. 2725 USE(parameters); 2726 } 2727 } 2728 2729 2730 void MacroAssembler::EnableInstrumentation() { 2731 VIXL_ASSERT(!isprint(InstrumentStateEnable)); 2732 ExactAssemblyScope scope(this, kInstructionSize); 2733 movn(xzr, InstrumentStateEnable); 2734 } 2735 2736 2737 void MacroAssembler::DisableInstrumentation() { 2738 VIXL_ASSERT(!isprint(InstrumentStateDisable)); 2739 ExactAssemblyScope scope(this, kInstructionSize); 2740 movn(xzr, InstrumentStateDisable); 2741 } 2742 2743 2744 void MacroAssembler::AnnotateInstrumentation(const char* marker_name) { 2745 VIXL_ASSERT(strlen(marker_name) == 2); 2746 2747 // We allow only printable characters in the marker names. Unprintable 2748 // characters are reserved for controlling features of the instrumentation. 2749 VIXL_ASSERT(isprint(marker_name[0]) && isprint(marker_name[1])); 2750 2751 ExactAssemblyScope scope(this, kInstructionSize); 2752 movn(xzr, (marker_name[1] << 8) | marker_name[0]); 2753 } 2754 2755 2756 void MacroAssembler::SetSimulatorCPUFeatures(const CPUFeatures& features) { 2757 ConfigureSimulatorCPUFeaturesHelper(features, kSetCPUFeaturesOpcode); 2758 } 2759 2760 2761 void MacroAssembler::EnableSimulatorCPUFeatures(const CPUFeatures& features) { 2762 ConfigureSimulatorCPUFeaturesHelper(features, kEnableCPUFeaturesOpcode); 2763 } 2764 2765 2766 void MacroAssembler::DisableSimulatorCPUFeatures(const CPUFeatures& features) { 2767 ConfigureSimulatorCPUFeaturesHelper(features, kDisableCPUFeaturesOpcode); 2768 } 2769 2770 2771 void MacroAssembler::ConfigureSimulatorCPUFeaturesHelper( 2772 const CPUFeatures& features, DebugHltOpcode action) { 2773 VIXL_ASSERT(allow_macro_instructions_); 2774 VIXL_ASSERT(generate_simulator_code_); 2775 2776 typedef ConfigureCPUFeaturesElementType ElementType; 2777 VIXL_ASSERT(CPUFeatures::kNumberOfFeatures <= 2778 std::numeric_limits<ElementType>::max()); 2779 2780 size_t count = features.Count(); 2781 2782 size_t preamble_length = kConfigureCPUFeaturesListOffset; 2783 size_t list_length = (count + 1) * sizeof(ElementType); 2784 size_t padding_length = AlignUp(list_length, kInstructionSize) - list_length; 2785 2786 size_t total_length = preamble_length + list_length + padding_length; 2787 2788 // Check the overall code size as well as the size of each component. 2789 ExactAssemblyScope guard_total(this, total_length); 2790 2791 { // Preamble: the opcode itself. 2792 ExactAssemblyScope guard_preamble(this, preamble_length); 2793 hlt(action); 2794 } 2795 { // A kNone-terminated list of features. 2796 ExactAssemblyScope guard_list(this, list_length); 2797 for (CPUFeatures::const_iterator it = features.begin(); 2798 it != features.end(); 2799 ++it) { 2800 dc(static_cast<ElementType>(*it)); 2801 } 2802 dc(static_cast<ElementType>(CPUFeatures::kNone)); 2803 } 2804 { // Padding for instruction alignment. 2805 ExactAssemblyScope guard_padding(this, padding_length); 2806 for (size_t size = 0; size < padding_length; size += sizeof(ElementType)) { 2807 // The exact value is arbitrary. 2808 dc(static_cast<ElementType>(CPUFeatures::kNone)); 2809 } 2810 } 2811 } 2812 2813 void MacroAssembler::SaveSimulatorCPUFeatures() { 2814 VIXL_ASSERT(allow_macro_instructions_); 2815 VIXL_ASSERT(generate_simulator_code_); 2816 SingleEmissionCheckScope guard(this); 2817 hlt(kSaveCPUFeaturesOpcode); 2818 } 2819 2820 2821 void MacroAssembler::RestoreSimulatorCPUFeatures() { 2822 VIXL_ASSERT(allow_macro_instructions_); 2823 VIXL_ASSERT(generate_simulator_code_); 2824 SingleEmissionCheckScope guard(this); 2825 hlt(kRestoreCPUFeaturesOpcode); 2826 } 2827 2828 2829 void UseScratchRegisterScope::Open(MacroAssembler* masm) { 2830 VIXL_ASSERT(masm_ == NULL); 2831 VIXL_ASSERT(masm != NULL); 2832 masm_ = masm; 2833 2834 CPURegList* available = masm->GetScratchRegisterList(); 2835 CPURegList* available_fp = masm->GetScratchFPRegisterList(); 2836 old_available_ = available->GetList(); 2837 old_availablefp_ = available_fp->GetList(); 2838 VIXL_ASSERT(available->GetType() == CPURegister::kRegister); 2839 VIXL_ASSERT(available_fp->GetType() == CPURegister::kVRegister); 2840 2841 parent_ = masm->GetCurrentScratchRegisterScope(); 2842 masm->SetCurrentScratchRegisterScope(this); 2843 } 2844 2845 2846 void UseScratchRegisterScope::Close() { 2847 if (masm_ != NULL) { 2848 // Ensure that scopes nest perfectly, and do not outlive their parents. 2849 // This is a run-time check because the order of destruction of objects in 2850 // the _same_ scope is implementation-defined, and is likely to change in 2851 // optimised builds. 2852 VIXL_CHECK(masm_->GetCurrentScratchRegisterScope() == this); 2853 masm_->SetCurrentScratchRegisterScope(parent_); 2854 2855 masm_->GetScratchRegisterList()->SetList(old_available_); 2856 masm_->GetScratchFPRegisterList()->SetList(old_availablefp_); 2857 2858 masm_ = NULL; 2859 } 2860 } 2861 2862 2863 bool UseScratchRegisterScope::IsAvailable(const CPURegister& reg) const { 2864 return masm_->GetScratchRegisterList()->IncludesAliasOf(reg) || 2865 masm_->GetScratchFPRegisterList()->IncludesAliasOf(reg); 2866 } 2867 2868 2869 Register UseScratchRegisterScope::AcquireRegisterOfSize(int size_in_bits) { 2870 int code = AcquireNextAvailable(masm_->GetScratchRegisterList()).GetCode(); 2871 return Register(code, size_in_bits); 2872 } 2873 2874 2875 FPRegister UseScratchRegisterScope::AcquireVRegisterOfSize(int size_in_bits) { 2876 int code = AcquireNextAvailable(masm_->GetScratchFPRegisterList()).GetCode(); 2877 return FPRegister(code, size_in_bits); 2878 } 2879 2880 2881 void UseScratchRegisterScope::Release(const CPURegister& reg) { 2882 VIXL_ASSERT(masm_ != NULL); 2883 if (reg.IsRegister()) { 2884 ReleaseByCode(masm_->GetScratchRegisterList(), reg.GetCode()); 2885 } else if (reg.IsVRegister()) { 2886 ReleaseByCode(masm_->GetScratchFPRegisterList(), reg.GetCode()); 2887 } else { 2888 VIXL_ASSERT(reg.IsNone()); 2889 } 2890 } 2891 2892 2893 void UseScratchRegisterScope::Include(const CPURegList& list) { 2894 VIXL_ASSERT(masm_ != NULL); 2895 if (list.GetType() == CPURegister::kRegister) { 2896 // Make sure that neither sp nor xzr are included the list. 2897 IncludeByRegList(masm_->GetScratchRegisterList(), 2898 list.GetList() & ~(xzr.GetBit() | sp.GetBit())); 2899 } else { 2900 VIXL_ASSERT(list.GetType() == CPURegister::kVRegister); 2901 IncludeByRegList(masm_->GetScratchFPRegisterList(), list.GetList()); 2902 } 2903 } 2904 2905 2906 void UseScratchRegisterScope::Include(const Register& reg1, 2907 const Register& reg2, 2908 const Register& reg3, 2909 const Register& reg4) { 2910 VIXL_ASSERT(masm_ != NULL); 2911 RegList include = 2912 reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit(); 2913 // Make sure that neither sp nor xzr are included the list. 2914 include &= ~(xzr.GetBit() | sp.GetBit()); 2915 2916 IncludeByRegList(masm_->GetScratchRegisterList(), include); 2917 } 2918 2919 2920 void UseScratchRegisterScope::Include(const FPRegister& reg1, 2921 const FPRegister& reg2, 2922 const FPRegister& reg3, 2923 const FPRegister& reg4) { 2924 RegList include = 2925 reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit(); 2926 IncludeByRegList(masm_->GetScratchFPRegisterList(), include); 2927 } 2928 2929 2930 void UseScratchRegisterScope::Exclude(const CPURegList& list) { 2931 if (list.GetType() == CPURegister::kRegister) { 2932 ExcludeByRegList(masm_->GetScratchRegisterList(), list.GetList()); 2933 } else { 2934 VIXL_ASSERT(list.GetType() == CPURegister::kVRegister); 2935 ExcludeByRegList(masm_->GetScratchFPRegisterList(), list.GetList()); 2936 } 2937 } 2938 2939 2940 void UseScratchRegisterScope::Exclude(const Register& reg1, 2941 const Register& reg2, 2942 const Register& reg3, 2943 const Register& reg4) { 2944 RegList exclude = 2945 reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit(); 2946 ExcludeByRegList(masm_->GetScratchRegisterList(), exclude); 2947 } 2948 2949 2950 void UseScratchRegisterScope::Exclude(const FPRegister& reg1, 2951 const FPRegister& reg2, 2952 const FPRegister& reg3, 2953 const FPRegister& reg4) { 2954 RegList excludefp = 2955 reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit(); 2956 ExcludeByRegList(masm_->GetScratchFPRegisterList(), excludefp); 2957 } 2958 2959 2960 void UseScratchRegisterScope::Exclude(const CPURegister& reg1, 2961 const CPURegister& reg2, 2962 const CPURegister& reg3, 2963 const CPURegister& reg4) { 2964 RegList exclude = 0; 2965 RegList excludefp = 0; 2966 2967 const CPURegister regs[] = {reg1, reg2, reg3, reg4}; 2968 2969 for (size_t i = 0; i < ArrayLength(regs); i++) { 2970 if (regs[i].IsRegister()) { 2971 exclude |= regs[i].GetBit(); 2972 } else if (regs[i].IsFPRegister()) { 2973 excludefp |= regs[i].GetBit(); 2974 } else { 2975 VIXL_ASSERT(regs[i].IsNone()); 2976 } 2977 } 2978 2979 ExcludeByRegList(masm_->GetScratchRegisterList(), exclude); 2980 ExcludeByRegList(masm_->GetScratchFPRegisterList(), excludefp); 2981 } 2982 2983 2984 void UseScratchRegisterScope::ExcludeAll() { 2985 ExcludeByRegList(masm_->GetScratchRegisterList(), 2986 masm_->GetScratchRegisterList()->GetList()); 2987 ExcludeByRegList(masm_->GetScratchFPRegisterList(), 2988 masm_->GetScratchFPRegisterList()->GetList()); 2989 } 2990 2991 2992 CPURegister UseScratchRegisterScope::AcquireNextAvailable( 2993 CPURegList* available) { 2994 VIXL_CHECK(!available->IsEmpty()); 2995 CPURegister result = available->PopLowestIndex(); 2996 VIXL_ASSERT(!AreAliased(result, xzr, sp)); 2997 return result; 2998 } 2999 3000 3001 void UseScratchRegisterScope::ReleaseByCode(CPURegList* available, int code) { 3002 ReleaseByRegList(available, static_cast<RegList>(1) << code); 3003 } 3004 3005 3006 void UseScratchRegisterScope::ReleaseByRegList(CPURegList* available, 3007 RegList regs) { 3008 available->SetList(available->GetList() | regs); 3009 } 3010 3011 3012 void UseScratchRegisterScope::IncludeByRegList(CPURegList* available, 3013 RegList regs) { 3014 available->SetList(available->GetList() | regs); 3015 } 3016 3017 3018 void UseScratchRegisterScope::ExcludeByRegList(CPURegList* available, 3019 RegList exclude) { 3020 available->SetList(available->GetList() & ~exclude); 3021 } 3022 3023 } // namespace aarch64 3024 } // namespace vixl 3025