1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <iostream> 18 #include <type_traits> 19 20 #include "assembler_arm_vixl.h" 21 #include "base/bit_utils.h" 22 #include "base/bit_utils_iterator.h" 23 #include "entrypoints/quick/quick_entrypoints.h" 24 #include "heap_poisoning.h" 25 #include "thread.h" 26 27 using namespace vixl::aarch32; // NOLINT(build/namespaces) 28 29 using vixl::ExactAssemblyScope; 30 using vixl::CodeBufferCheckScope; 31 32 namespace art { 33 namespace arm { 34 35 #ifdef ___ 36 #error "ARM Assembler macro already defined." 37 #else 38 #define ___ vixl_masm_. 39 #endif 40 41 // Thread register definition. 42 extern const vixl32::Register tr(TR); 43 // Marking register definition. 44 extern const vixl32::Register mr(MR); 45 46 void ArmVIXLAssembler::FinalizeCode() { 47 vixl_masm_.FinalizeCode(); 48 } 49 50 size_t ArmVIXLAssembler::CodeSize() const { 51 return vixl_masm_.GetSizeOfCodeGenerated(); 52 } 53 54 const uint8_t* ArmVIXLAssembler::CodeBufferBaseAddress() const { 55 return vixl_masm_.GetBuffer().GetStartAddress<const uint8_t*>(); 56 } 57 58 void ArmVIXLAssembler::FinalizeInstructions(const MemoryRegion& region) { 59 // Copy the instructions from the buffer. 60 MemoryRegion from(vixl_masm_.GetBuffer()->GetStartAddress<void*>(), CodeSize()); 61 region.CopyFrom(0, from); 62 } 63 64 void ArmVIXLAssembler::PoisonHeapReference(vixl::aarch32::Register reg) { 65 // reg = -reg. 66 ___ Rsb(reg, reg, 0); 67 } 68 69 void ArmVIXLAssembler::UnpoisonHeapReference(vixl::aarch32::Register reg) { 70 // reg = -reg. 71 ___ Rsb(reg, reg, 0); 72 } 73 74 void ArmVIXLAssembler::MaybePoisonHeapReference(vixl32::Register reg) { 75 if (kPoisonHeapReferences) { 76 PoisonHeapReference(reg); 77 } 78 } 79 80 void ArmVIXLAssembler::MaybeUnpoisonHeapReference(vixl32::Register reg) { 81 if (kPoisonHeapReferences) { 82 UnpoisonHeapReference(reg); 83 } 84 } 85 86 void ArmVIXLAssembler::GenerateMarkingRegisterCheck(vixl32::Register temp, int code) { 87 // The Marking Register is only used in the Baker read barrier configuration. 88 DCHECK(kEmitCompilerReadBarrier); 89 DCHECK(kUseBakerReadBarrier); 90 91 vixl32::Label mr_is_ok; 92 93 // temp = self.tls32_.is.gc_marking 94 ___ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value())); 95 // Check that mr == self.tls32_.is.gc_marking. 96 ___ Cmp(mr, temp); 97 ___ B(eq, &mr_is_ok, /* far_target */ false); 98 ___ Bkpt(code); 99 ___ Bind(&mr_is_ok); 100 } 101 102 void ArmVIXLAssembler::LoadImmediate(vixl32::Register rd, int32_t value) { 103 // TODO(VIXL): Implement this optimization in VIXL. 104 if (!ShifterOperandCanAlwaysHold(value) && ShifterOperandCanAlwaysHold(~value)) { 105 ___ Mvn(rd, ~value); 106 } else { 107 ___ Mov(rd, value); 108 } 109 } 110 111 bool ArmVIXLAssembler::ShifterOperandCanAlwaysHold(uint32_t immediate) { 112 return vixl_masm_.IsModifiedImmediate(immediate); 113 } 114 115 bool ArmVIXLAssembler::ShifterOperandCanHold(Opcode opcode, 116 uint32_t immediate, 117 vixl::aarch32::FlagsUpdate update_flags) { 118 switch (opcode) { 119 case ADD: 120 case SUB: 121 // Less than (or equal to) 12 bits can be done if we don't need to set condition codes. 122 if (IsUint<12>(immediate) && update_flags != vixl::aarch32::SetFlags) { 123 return true; 124 } 125 return ShifterOperandCanAlwaysHold(immediate); 126 127 case MOV: 128 // TODO: Support less than or equal to 12bits. 129 return ShifterOperandCanAlwaysHold(immediate); 130 131 case MVN: 132 default: 133 return ShifterOperandCanAlwaysHold(immediate); 134 } 135 } 136 137 bool ArmVIXLAssembler::CanSplitLoadStoreOffset(int32_t allowed_offset_bits, 138 int32_t offset, 139 /*out*/ int32_t* add_to_base, 140 /*out*/ int32_t* offset_for_load_store) { 141 int32_t other_bits = offset & ~allowed_offset_bits; 142 if (ShifterOperandCanAlwaysHold(other_bits) || ShifterOperandCanAlwaysHold(-other_bits)) { 143 *add_to_base = offset & ~allowed_offset_bits; 144 *offset_for_load_store = offset & allowed_offset_bits; 145 return true; 146 } 147 return false; 148 } 149 150 int32_t ArmVIXLAssembler::AdjustLoadStoreOffset(int32_t allowed_offset_bits, 151 vixl32::Register temp, 152 vixl32::Register base, 153 int32_t offset) { 154 DCHECK_NE(offset & ~allowed_offset_bits, 0); 155 int32_t add_to_base, offset_for_load; 156 if (CanSplitLoadStoreOffset(allowed_offset_bits, offset, &add_to_base, &offset_for_load)) { 157 ___ Add(temp, base, add_to_base); 158 return offset_for_load; 159 } else { 160 ___ Mov(temp, offset); 161 ___ Add(temp, temp, base); 162 return 0; 163 } 164 } 165 166 // TODO(VIXL): Implement this in VIXL. 167 int32_t ArmVIXLAssembler::GetAllowedLoadOffsetBits(LoadOperandType type) { 168 switch (type) { 169 case kLoadSignedByte: 170 case kLoadSignedHalfword: 171 case kLoadUnsignedHalfword: 172 case kLoadUnsignedByte: 173 case kLoadWord: 174 // We can encode imm12 offset. 175 return 0xfff; 176 case kLoadSWord: 177 case kLoadDWord: 178 case kLoadWordPair: 179 // We can encode imm8:'00' offset. 180 return 0xff << 2; 181 default: 182 LOG(FATAL) << "UNREACHABLE"; 183 UNREACHABLE(); 184 } 185 } 186 187 // TODO(VIXL): Implement this in VIXL. 188 int32_t ArmVIXLAssembler::GetAllowedStoreOffsetBits(StoreOperandType type) { 189 switch (type) { 190 case kStoreHalfword: 191 case kStoreByte: 192 case kStoreWord: 193 // We can encode imm12 offset. 194 return 0xfff; 195 case kStoreSWord: 196 case kStoreDWord: 197 case kStoreWordPair: 198 // We can encode imm8:'00' offset. 199 return 0xff << 2; 200 default: 201 LOG(FATAL) << "UNREACHABLE"; 202 UNREACHABLE(); 203 } 204 } 205 206 // TODO(VIXL): Implement this in VIXL. 207 static bool CanHoldLoadOffsetThumb(LoadOperandType type, int offset) { 208 switch (type) { 209 case kLoadSignedByte: 210 case kLoadSignedHalfword: 211 case kLoadUnsignedHalfword: 212 case kLoadUnsignedByte: 213 case kLoadWord: 214 return IsAbsoluteUint<12>(offset); 215 case kLoadSWord: 216 case kLoadDWord: 217 return IsAbsoluteUint<10>(offset) && IsAligned<4>(offset); // VFP addressing mode. 218 case kLoadWordPair: 219 return IsAbsoluteUint<10>(offset) && IsAligned<4>(offset); 220 default: 221 LOG(FATAL) << "UNREACHABLE"; 222 UNREACHABLE(); 223 } 224 } 225 226 // TODO(VIXL): Implement this in VIXL. 227 static bool CanHoldStoreOffsetThumb(StoreOperandType type, int offset) { 228 switch (type) { 229 case kStoreHalfword: 230 case kStoreByte: 231 case kStoreWord: 232 return IsAbsoluteUint<12>(offset); 233 case kStoreSWord: 234 case kStoreDWord: 235 return IsAbsoluteUint<10>(offset) && IsAligned<4>(offset); // VFP addressing mode. 236 case kStoreWordPair: 237 return IsAbsoluteUint<10>(offset) && IsAligned<4>(offset); 238 default: 239 LOG(FATAL) << "UNREACHABLE"; 240 UNREACHABLE(); 241 } 242 } 243 244 // Implementation note: this method must emit at most one instruction when 245 // Address::CanHoldStoreOffsetThumb. 246 // TODO(VIXL): Implement AdjustLoadStoreOffset logic in VIXL. 247 void ArmVIXLAssembler::StoreToOffset(StoreOperandType type, 248 vixl32::Register reg, 249 vixl32::Register base, 250 int32_t offset) { 251 vixl32::Register tmp_reg; 252 UseScratchRegisterScope temps(&vixl_masm_); 253 254 if (!CanHoldStoreOffsetThumb(type, offset)) { 255 CHECK_NE(base.GetCode(), kIpCode); 256 if ((reg.GetCode() != kIpCode) && 257 (!vixl_masm_.GetScratchRegisterList()->IsEmpty()) && 258 ((type != kStoreWordPair) || (reg.GetCode() + 1 != kIpCode))) { 259 tmp_reg = temps.Acquire(); 260 } else { 261 // Be careful not to use ip twice (for `reg` (or `reg` + 1 in 262 // the case of a word-pair store) and `base`) to build the 263 // Address object used by the store instruction(s) below. 264 // Instead, save R5 on the stack (or R6 if R5 is already used by 265 // `base`), use it as secondary temporary register, and restore 266 // it after the store instruction has been emitted. 267 tmp_reg = (base.GetCode() != 5) ? r5 : r6; 268 ___ Push(tmp_reg); 269 if (base.GetCode() == kSpCode) { 270 offset += kRegisterSize; 271 } 272 } 273 // TODO: Implement indexed store (not available for STRD), inline AdjustLoadStoreOffset() 274 // and in the "unsplittable" path get rid of the "add" by using the store indexed instead. 275 offset = AdjustLoadStoreOffset(GetAllowedStoreOffsetBits(type), tmp_reg, base, offset); 276 base = tmp_reg; 277 } 278 DCHECK(CanHoldStoreOffsetThumb(type, offset)); 279 switch (type) { 280 case kStoreByte: 281 ___ Strb(reg, MemOperand(base, offset)); 282 break; 283 case kStoreHalfword: 284 ___ Strh(reg, MemOperand(base, offset)); 285 break; 286 case kStoreWord: 287 ___ Str(reg, MemOperand(base, offset)); 288 break; 289 case kStoreWordPair: 290 ___ Strd(reg, vixl32::Register(reg.GetCode() + 1), MemOperand(base, offset)); 291 break; 292 default: 293 LOG(FATAL) << "UNREACHABLE"; 294 UNREACHABLE(); 295 } 296 if ((tmp_reg.IsValid()) && (tmp_reg.GetCode() != kIpCode)) { 297 CHECK(tmp_reg.Is(r5) || tmp_reg.Is(r6)) << tmp_reg; 298 ___ Pop(tmp_reg); 299 } 300 } 301 302 // Implementation note: this method must emit at most one instruction when 303 // Address::CanHoldLoadOffsetThumb. 304 // TODO(VIXL): Implement AdjustLoadStoreOffset logic in VIXL. 305 void ArmVIXLAssembler::LoadFromOffset(LoadOperandType type, 306 vixl32::Register dest, 307 vixl32::Register base, 308 int32_t offset) { 309 if (!CanHoldLoadOffsetThumb(type, offset)) { 310 CHECK(!base.Is(ip)); 311 // Inlined AdjustLoadStoreOffset() allows us to pull a few more tricks. 312 int32_t allowed_offset_bits = GetAllowedLoadOffsetBits(type); 313 DCHECK_NE(offset & ~allowed_offset_bits, 0); 314 int32_t add_to_base, offset_for_load; 315 if (CanSplitLoadStoreOffset(allowed_offset_bits, offset, &add_to_base, &offset_for_load)) { 316 // Use reg for the adjusted base. If it's low reg, we may end up using 16-bit load. 317 AddConstant(dest, base, add_to_base); 318 base = dest; 319 offset = offset_for_load; 320 } else { 321 UseScratchRegisterScope temps(&vixl_masm_); 322 vixl32::Register temp = (dest.Is(base)) ? temps.Acquire() : dest; 323 LoadImmediate(temp, offset); 324 // TODO: Implement indexed load (not available for LDRD) and use it here to avoid the ADD. 325 // Use reg for the adjusted base. If it's low reg, we may end up using 16-bit load. 326 ___ Add(dest, dest, (dest.Is(base)) ? temp : base); 327 base = dest; 328 offset = 0; 329 } 330 } 331 332 DCHECK(CanHoldLoadOffsetThumb(type, offset)); 333 switch (type) { 334 case kLoadSignedByte: 335 ___ Ldrsb(dest, MemOperand(base, offset)); 336 break; 337 case kLoadUnsignedByte: 338 ___ Ldrb(dest, MemOperand(base, offset)); 339 break; 340 case kLoadSignedHalfword: 341 ___ Ldrsh(dest, MemOperand(base, offset)); 342 break; 343 case kLoadUnsignedHalfword: 344 ___ Ldrh(dest, MemOperand(base, offset)); 345 break; 346 case kLoadWord: 347 CHECK(!dest.IsSP()); 348 ___ Ldr(dest, MemOperand(base, offset)); 349 break; 350 case kLoadWordPair: 351 ___ Ldrd(dest, vixl32::Register(dest.GetCode() + 1), MemOperand(base, offset)); 352 break; 353 default: 354 LOG(FATAL) << "UNREACHABLE"; 355 UNREACHABLE(); 356 } 357 } 358 359 void ArmVIXLAssembler::StoreSToOffset(vixl32::SRegister source, 360 vixl32::Register base, 361 int32_t offset) { 362 ___ Vstr(source, MemOperand(base, offset)); 363 } 364 365 void ArmVIXLAssembler::StoreDToOffset(vixl32::DRegister source, 366 vixl32::Register base, 367 int32_t offset) { 368 ___ Vstr(source, MemOperand(base, offset)); 369 } 370 371 void ArmVIXLAssembler::LoadSFromOffset(vixl32::SRegister reg, 372 vixl32::Register base, 373 int32_t offset) { 374 ___ Vldr(reg, MemOperand(base, offset)); 375 } 376 377 void ArmVIXLAssembler::LoadDFromOffset(vixl32::DRegister reg, 378 vixl32::Register base, 379 int32_t offset) { 380 ___ Vldr(reg, MemOperand(base, offset)); 381 } 382 383 // Prefer Str to Add/Stm in ArmVIXLAssembler::StoreRegisterList and 384 // ArmVIXLAssembler::LoadRegisterList where this generates less code (size). 385 static constexpr int kRegListThreshold = 4; 386 387 void ArmVIXLAssembler::StoreRegisterList(RegList regs, size_t stack_offset) { 388 int number_of_regs = POPCOUNT(static_cast<uint32_t>(regs)); 389 if (number_of_regs != 0) { 390 if (number_of_regs > kRegListThreshold) { 391 UseScratchRegisterScope temps(GetVIXLAssembler()); 392 vixl32::Register base = sp; 393 if (stack_offset != 0) { 394 base = temps.Acquire(); 395 DCHECK_EQ(regs & (1u << base.GetCode()), 0u); 396 ___ Add(base, sp, Operand::From(stack_offset)); 397 } 398 ___ Stm(base, NO_WRITE_BACK, RegisterList(regs)); 399 } else { 400 for (uint32_t i : LowToHighBits(static_cast<uint32_t>(regs))) { 401 ___ Str(vixl32::Register(i), MemOperand(sp, stack_offset)); 402 stack_offset += kRegSizeInBytes; 403 } 404 } 405 } 406 } 407 408 void ArmVIXLAssembler::LoadRegisterList(RegList regs, size_t stack_offset) { 409 int number_of_regs = POPCOUNT(static_cast<uint32_t>(regs)); 410 if (number_of_regs != 0) { 411 if (number_of_regs > kRegListThreshold) { 412 UseScratchRegisterScope temps(GetVIXLAssembler()); 413 vixl32::Register base = sp; 414 if (stack_offset != 0) { 415 base = temps.Acquire(); 416 ___ Add(base, sp, Operand::From(stack_offset)); 417 } 418 ___ Ldm(base, NO_WRITE_BACK, RegisterList(regs)); 419 } else { 420 for (uint32_t i : LowToHighBits(static_cast<uint32_t>(regs))) { 421 ___ Ldr(vixl32::Register(i), MemOperand(sp, stack_offset)); 422 stack_offset += kRegSizeInBytes; 423 } 424 } 425 } 426 } 427 428 void ArmVIXLAssembler::AddConstant(vixl32::Register rd, int32_t value) { 429 AddConstant(rd, rd, value); 430 } 431 432 // TODO(VIXL): think about using adds which updates flags where possible. 433 void ArmVIXLAssembler::AddConstant(vixl32::Register rd, 434 vixl32::Register rn, 435 int32_t value) { 436 DCHECK(vixl_masm_.OutsideITBlock()); 437 // TODO(VIXL): implement this optimization in VIXL. 438 if (value == 0) { 439 if (!rd.Is(rn)) { 440 ___ Mov(rd, rn); 441 } 442 return; 443 } 444 ___ Add(rd, rn, value); 445 } 446 447 // Inside IT block we must use assembler, macroassembler instructions are not permitted. 448 void ArmVIXLAssembler::AddConstantInIt(vixl32::Register rd, 449 vixl32::Register rn, 450 int32_t value, 451 vixl32::Condition cond) { 452 DCHECK(vixl_masm_.InITBlock()); 453 if (value == 0) { 454 ___ mov(cond, rd, rn); 455 } else { 456 ___ add(cond, rd, rn, value); 457 } 458 } 459 460 void ArmVIXLMacroAssembler::CompareAndBranchIfZero(vixl32::Register rn, 461 vixl32::Label* label, 462 bool is_far_target) { 463 if (!is_far_target && rn.IsLow() && !label->IsBound()) { 464 // In T32, Cbz/Cbnz instructions have following limitations: 465 // - There are only 7 bits (i:imm5:0) to encode branch target address (cannot be far target). 466 // - Only low registers (i.e R0 .. R7) can be encoded. 467 // - Only forward branches (unbound labels) are supported. 468 Cbz(rn, label); 469 return; 470 } 471 Cmp(rn, 0); 472 B(eq, label, is_far_target); 473 } 474 475 void ArmVIXLMacroAssembler::CompareAndBranchIfNonZero(vixl32::Register rn, 476 vixl32::Label* label, 477 bool is_far_target) { 478 if (!is_far_target && rn.IsLow() && !label->IsBound()) { 479 Cbnz(rn, label); 480 return; 481 } 482 Cmp(rn, 0); 483 B(ne, label, is_far_target); 484 } 485 486 void ArmVIXLMacroAssembler::B(vixl32::Label* label) { 487 if (!label->IsBound()) { 488 // Try to use a 16-bit encoding of the B instruction. 489 DCHECK(OutsideITBlock()); 490 BPreferNear(label); 491 return; 492 } 493 MacroAssembler::B(label); 494 } 495 496 void ArmVIXLMacroAssembler::B(vixl32::Condition cond, vixl32::Label* label, bool is_far_target) { 497 if (!label->IsBound() && !is_far_target) { 498 // Try to use a 16-bit encoding of the B instruction. 499 DCHECK(OutsideITBlock()); 500 BPreferNear(cond, label); 501 return; 502 } 503 MacroAssembler::B(cond, label); 504 } 505 506 } // namespace arm 507 } // namespace art 508