1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "linker/arm64/relative_patcher_arm64.h" 18 19 #include "arch/arm64/asm_support_arm64.h" 20 #include "arch/arm64/instruction_set_features_arm64.h" 21 #include "art_method.h" 22 #include "base/bit_utils.h" 23 #include "compiled_method.h" 24 #include "driver/compiler_driver.h" 25 #include "entrypoints/quick/quick_entrypoints_enum.h" 26 #include "linker/output_stream.h" 27 #include "lock_word.h" 28 #include "mirror/object.h" 29 #include "mirror/array-inl.h" 30 #include "oat.h" 31 #include "oat_quick_method_header.h" 32 #include "utils/arm64/assembler_arm64.h" 33 34 namespace art { 35 namespace linker { 36 37 namespace { 38 39 // Maximum positive and negative displacement for method call measured from the patch location. 40 // (Signed 28 bit displacement with the last two bits 0 has range [-2^27, 2^27-4] measured from 41 // the ARM64 PC pointing to the BL.) 42 constexpr uint32_t kMaxMethodCallPositiveDisplacement = (1u << 27) - 4u; 43 constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 27); 44 45 // Maximum positive and negative displacement for a conditional branch measured from the patch 46 // location. (Signed 21 bit displacement with the last two bits 0 has range [-2^20, 2^20-4] 47 // measured from the ARM64 PC pointing to the B.cond.) 48 constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 4u; 49 constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20); 50 51 // The ADRP thunk for erratum 843419 is 2 instructions, i.e. 8 bytes. 52 constexpr uint32_t kAdrpThunkSize = 8u; 53 54 inline bool IsAdrpPatch(const LinkerPatch& patch) { 55 switch (patch.GetType()) { 56 case LinkerPatch::Type::kMethod: 57 case LinkerPatch::Type::kCall: 58 case LinkerPatch::Type::kCallRelative: 59 case LinkerPatch::Type::kType: 60 case LinkerPatch::Type::kString: 61 case LinkerPatch::Type::kBakerReadBarrierBranch: 62 return false; 63 case LinkerPatch::Type::kTypeRelative: 64 case LinkerPatch::Type::kTypeBssEntry: 65 case LinkerPatch::Type::kStringRelative: 66 case LinkerPatch::Type::kStringBssEntry: 67 case LinkerPatch::Type::kDexCacheArray: 68 return patch.LiteralOffset() == patch.PcInsnOffset(); 69 } 70 } 71 72 inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) { 73 if (num_adrp == 0u) { 74 return 0u; 75 } 76 uint32_t alignment_bytes = CompiledMethod::AlignCode(code_size, kArm64) - code_size; 77 return kAdrpThunkSize * num_adrp + alignment_bytes; 78 } 79 80 } // anonymous namespace 81 82 Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, 83 const Arm64InstructionSetFeatures* features) 84 : ArmBaseRelativePatcher(provider, kArm64), 85 fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), 86 reserved_adrp_thunks_(0u), 87 processed_adrp_thunks_(0u) { 88 if (fix_cortex_a53_843419_) { 89 adrp_thunk_locations_.reserve(16u); 90 current_method_thunks_.reserve(16u * kAdrpThunkSize); 91 } 92 } 93 94 uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, 95 const CompiledMethod* compiled_method, 96 MethodReference method_ref) { 97 if (!fix_cortex_a53_843419_) { 98 DCHECK(adrp_thunk_locations_.empty()); 99 return ReserveSpaceInternal(offset, compiled_method, method_ref, 0u); 100 } 101 102 // Add thunks for previous method if any. 103 if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { 104 size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; 105 offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; 106 reserved_adrp_thunks_ = adrp_thunk_locations_.size(); 107 } 108 109 // Count the number of ADRP insns as the upper bound on the number of thunks needed 110 // and use it to reserve space for other linker patches. 111 size_t num_adrp = 0u; 112 DCHECK(compiled_method != nullptr); 113 for (const LinkerPatch& patch : compiled_method->GetPatches()) { 114 if (IsAdrpPatch(patch)) { 115 ++num_adrp; 116 } 117 } 118 ArrayRef<const uint8_t> code = compiled_method->GetQuickCode(); 119 uint32_t max_extra_space = MaxExtraSpace(num_adrp, code.size()); 120 offset = ReserveSpaceInternal(offset, compiled_method, method_ref, max_extra_space); 121 if (num_adrp == 0u) { 122 return offset; 123 } 124 125 // Now that we have the actual offset where the code will be placed, locate the ADRP insns 126 // that actually require the thunk. 127 uint32_t quick_code_offset = compiled_method->AlignCode(offset + sizeof(OatQuickMethodHeader)); 128 uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); 129 DCHECK(compiled_method != nullptr); 130 for (const LinkerPatch& patch : compiled_method->GetPatches()) { 131 if (IsAdrpPatch(patch)) { 132 uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); 133 if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { 134 adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); 135 thunk_offset += kAdrpThunkSize; 136 } 137 } 138 } 139 return offset; 140 } 141 142 uint32_t Arm64RelativePatcher::ReserveSpaceEnd(uint32_t offset) { 143 if (!fix_cortex_a53_843419_) { 144 DCHECK(adrp_thunk_locations_.empty()); 145 } else { 146 // Add thunks for the last method if any. 147 if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { 148 size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; 149 offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; 150 reserved_adrp_thunks_ = adrp_thunk_locations_.size(); 151 } 152 } 153 return ArmBaseRelativePatcher::ReserveSpaceEnd(offset); 154 } 155 156 uint32_t Arm64RelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { 157 if (fix_cortex_a53_843419_) { 158 if (!current_method_thunks_.empty()) { 159 uint32_t aligned_offset = CompiledMethod::AlignCode(offset, kArm64); 160 if (kIsDebugBuild) { 161 CHECK_ALIGNED(current_method_thunks_.size(), kAdrpThunkSize); 162 size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; 163 CHECK_LE(num_thunks, processed_adrp_thunks_); 164 for (size_t i = 0u; i != num_thunks; ++i) { 165 const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; 166 CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); 167 } 168 } 169 uint32_t aligned_code_delta = aligned_offset - offset; 170 if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { 171 return 0u; 172 } 173 if (!WriteMiscThunk(out, ArrayRef<const uint8_t>(current_method_thunks_))) { 174 return 0u; 175 } 176 offset = aligned_offset + current_method_thunks_.size(); 177 current_method_thunks_.clear(); 178 } 179 } 180 return ArmBaseRelativePatcher::WriteThunks(out, offset); 181 } 182 183 void Arm64RelativePatcher::PatchCall(std::vector<uint8_t>* code, 184 uint32_t literal_offset, 185 uint32_t patch_offset, uint32_t 186 target_offset) { 187 DCHECK_LE(literal_offset + 4u, code->size()); 188 DCHECK_EQ(literal_offset & 3u, 0u); 189 DCHECK_EQ(patch_offset & 3u, 0u); 190 DCHECK_EQ(target_offset & 3u, 0u); 191 uint32_t displacement = CalculateMethodCallDisplacement(patch_offset, target_offset & ~1u); 192 DCHECK_EQ(displacement & 3u, 0u); 193 DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. 194 uint32_t insn = (displacement & 0x0fffffffu) >> 2; 195 insn |= 0x94000000; // BL 196 197 // Check that we're just overwriting an existing BL. 198 DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); 199 // Write the new BL. 200 SetInsn(code, literal_offset, insn); 201 } 202 203 void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, 204 const LinkerPatch& patch, 205 uint32_t patch_offset, 206 uint32_t target_offset) { 207 DCHECK_EQ(patch_offset & 3u, 0u); 208 DCHECK_EQ(target_offset & 3u, 0u); 209 uint32_t literal_offset = patch.LiteralOffset(); 210 uint32_t insn = GetInsn(code, literal_offset); 211 uint32_t pc_insn_offset = patch.PcInsnOffset(); 212 uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); 213 bool wide = (insn & 0x40000000) != 0; 214 uint32_t shift = wide ? 3u : 2u; 215 if (literal_offset == pc_insn_offset) { 216 // Check it's an ADRP with imm == 0 (unset). 217 DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) 218 << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; 219 if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && 220 adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { 221 DCHECK(NeedsErratum843419Thunk(ArrayRef<const uint8_t>(*code), 222 literal_offset, patch_offset)); 223 uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; 224 uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); 225 uint32_t adrp = PatchAdrp(insn, adrp_disp); 226 227 uint32_t out_disp = thunk_offset - patch_offset; 228 DCHECK_EQ(out_disp & 3u, 0u); 229 DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. 230 insn = (out_disp & 0x0fffffffu) >> shift; 231 insn |= 0x14000000; // B <thunk> 232 233 uint32_t back_disp = -out_disp; 234 DCHECK_EQ(back_disp & 3u, 0u); 235 DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. 236 uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; 237 b_back |= 0x14000000; // B <back> 238 size_t thunks_code_offset = current_method_thunks_.size(); 239 current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); 240 SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); 241 SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); 242 static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); 243 244 processed_adrp_thunks_ += 1u; 245 } else { 246 insn = PatchAdrp(insn, disp); 247 } 248 // Write the new ADRP (or B to the erratum 843419 thunk). 249 SetInsn(code, literal_offset, insn); 250 } else { 251 if ((insn & 0xfffffc00) == 0x91000000) { 252 // ADD immediate, 64-bit with imm12 == 0 (unset). 253 if (!kEmitCompilerReadBarrier) { 254 DCHECK(patch.GetType() == LinkerPatch::Type::kStringRelative || 255 patch.GetType() == LinkerPatch::Type::kTypeRelative) << patch.GetType(); 256 } else { 257 // With the read barrier (non-Baker) enabled, it could be kStringBssEntry or kTypeBssEntry. 258 DCHECK(patch.GetType() == LinkerPatch::Type::kStringRelative || 259 patch.GetType() == LinkerPatch::Type::kTypeRelative || 260 patch.GetType() == LinkerPatch::Type::kStringBssEntry || 261 patch.GetType() == LinkerPatch::Type::kTypeBssEntry) << patch.GetType(); 262 } 263 shift = 0u; // No shift for ADD. 264 } else { 265 // LDR/STR 32-bit or 64-bit with imm12 == 0 (unset). 266 DCHECK(patch.GetType() == LinkerPatch::Type::kDexCacheArray || 267 patch.GetType() == LinkerPatch::Type::kTypeBssEntry || 268 patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); 269 DCHECK_EQ(insn & 0xbfbffc00, 0xb9000000) << std::hex << insn; 270 } 271 if (kIsDebugBuild) { 272 uint32_t adrp = GetInsn(code, pc_insn_offset); 273 if ((adrp & 0x9f000000u) != 0x90000000u) { 274 CHECK(fix_cortex_a53_843419_); 275 CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B <thunk> 276 CHECK_ALIGNED(current_method_thunks_.size(), kAdrpThunkSize); 277 size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; 278 CHECK_LE(num_thunks, processed_adrp_thunks_); 279 uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; 280 for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { 281 CHECK_NE(i, processed_adrp_thunks_); 282 if (adrp_thunk_locations_[i].first == b_offset) { 283 size_t idx = num_thunks - (processed_adrp_thunks_ - i); 284 adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); 285 break; 286 } 287 } 288 } 289 CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points 290 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. 291 } 292 uint32_t imm12 = (disp & 0xfffu) >> shift; 293 insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); 294 SetInsn(code, literal_offset, insn); 295 } 296 } 297 298 void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* code, 299 const LinkerPatch& patch, 300 uint32_t patch_offset) { 301 DCHECK_ALIGNED(patch_offset, 4u); 302 uint32_t literal_offset = patch.LiteralOffset(); 303 DCHECK_ALIGNED(literal_offset, 4u); 304 DCHECK_LT(literal_offset, code->size()); 305 uint32_t insn = GetInsn(code, literal_offset); 306 DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched) 307 ThunkKey key = GetBakerReadBarrierKey(patch); 308 if (kIsDebugBuild) { 309 // Check that the next instruction matches the expected LDR. 310 switch (key.GetType()) { 311 case ThunkType::kBakerReadBarrierField: { 312 DCHECK_GE(code->size() - literal_offset, 8u); 313 uint32_t next_insn = GetInsn(code, literal_offset + 4u); 314 // LDR (immediate) with correct base_reg. 315 CheckValidReg(next_insn & 0x1fu); // Check destination register. 316 CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (key.GetOffsetParams().base_reg << 5)); 317 break; 318 } 319 case ThunkType::kBakerReadBarrierRoot: { 320 DCHECK_GE(literal_offset, 4u); 321 uint32_t prev_insn = GetInsn(code, literal_offset - 4u); 322 // LDR (immediate) with correct root_reg. 323 CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | key.GetRootParams().root_reg); 324 break; 325 } 326 default: 327 LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType()); 328 UNREACHABLE(); 329 } 330 } 331 uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); 332 DCHECK_ALIGNED(target_offset, 4u); 333 uint32_t disp = target_offset - patch_offset; 334 DCHECK((disp >> 20) == 0u || (disp >> 20) == 4095u); // 21-bit signed. 335 insn |= (disp << (5 - 2)) & 0x00ffffe0u; // Shift bits 2-20 to 5-23. 336 SetInsn(code, literal_offset, insn); 337 } 338 339 ArmBaseRelativePatcher::ThunkKey Arm64RelativePatcher::GetBakerReadBarrierKey( 340 const LinkerPatch& patch) { 341 DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kBakerReadBarrierBranch); 342 uint32_t value = patch.GetBakerCustomValue1(); 343 BakerReadBarrierKind type = BakerReadBarrierKindField::Decode(value); 344 ThunkParams params; 345 switch (type) { 346 case BakerReadBarrierKind::kField: 347 params.offset_params.base_reg = BakerReadBarrierFirstRegField::Decode(value); 348 CheckValidReg(params.offset_params.base_reg); 349 params.offset_params.holder_reg = BakerReadBarrierSecondRegField::Decode(value); 350 CheckValidReg(params.offset_params.holder_reg); 351 break; 352 case BakerReadBarrierKind::kGcRoot: 353 params.root_params.root_reg = BakerReadBarrierFirstRegField::Decode(value); 354 CheckValidReg(params.root_params.root_reg); 355 params.root_params.dummy = 0u; 356 DCHECK_EQ(BakerReadBarrierSecondRegField::Decode(value), kInvalidEncodedReg); 357 break; 358 default: 359 LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(type); 360 UNREACHABLE(); 361 } 362 constexpr uint8_t kTypeTranslationOffset = 1u; 363 static_assert(static_cast<uint32_t>(BakerReadBarrierKind::kField) + kTypeTranslationOffset == 364 static_cast<uint32_t>(ThunkType::kBakerReadBarrierField), 365 "Thunk type translation check."); 366 static_assert(static_cast<uint32_t>(BakerReadBarrierKind::kGcRoot) + kTypeTranslationOffset == 367 static_cast<uint32_t>(ThunkType::kBakerReadBarrierRoot), 368 "Thunk type translation check."); 369 return ThunkKey(static_cast<ThunkType>(static_cast<uint32_t>(type) + kTypeTranslationOffset), 370 params); 371 } 372 373 #define __ assembler.GetVIXLAssembler()-> 374 375 static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, 376 vixl::aarch64::Register base_reg, 377 vixl::aarch64::MemOperand& lock_word, 378 vixl::aarch64::Label* slow_path) { 379 using namespace vixl::aarch64; // NOLINT(build/namespaces) 380 // Load the lock word containing the rb_state. 381 __ Ldr(ip0.W(), lock_word); 382 // Given the numeric representation, it's enough to check the low bit of the rb_state. 383 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 384 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 385 __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path); 386 static_assert( 387 BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET, 388 "Field and array LDR offsets must be the same to reuse the same code."); 389 // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning). 390 static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), 391 "Field LDR must be 1 instruction (4B) before the return address label; " 392 " 2 instructions (8B) for heap poisoning."); 393 __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); 394 // Introduce a dependency on the lock_word including rb_state, 395 // to prevent load-load reordering, and without using 396 // a memory barrier (which would be more expensive). 397 __ Add(base_reg, base_reg, Operand(vixl::aarch64::ip0, LSR, 32)); 398 __ Br(lr); // And return back to the function. 399 // Note: The fake dependency is unnecessary for the slow path. 400 } 401 402 std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) { 403 using namespace vixl::aarch64; // NOLINT(build/namespaces) 404 ArenaPool pool; 405 ArenaAllocator arena(&pool); 406 arm64::Arm64Assembler assembler(&arena); 407 408 switch (key.GetType()) { 409 case ThunkType::kMethodCall: { 410 // The thunk just uses the entry point in the ArtMethod. This works even for calls 411 // to the generic JNI and interpreter trampolines. 412 Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( 413 kArm64PointerSize).Int32Value()); 414 assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); 415 break; 416 } 417 case ThunkType::kBakerReadBarrierField: { 418 // Check if the holder is gray and, if not, add fake dependency to the base register 419 // and return to the LDR instruction to load the reference. Otherwise, use introspection 420 // to load the reference and call the entrypoint (in IP1) that performs further checks 421 // on the reference and marks it if needed. 422 auto holder_reg = Register::GetXRegFromCode(key.GetOffsetParams().holder_reg); 423 auto base_reg = Register::GetXRegFromCode(key.GetOffsetParams().base_reg); 424 UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); 425 temps.Exclude(ip0, ip1); 426 // If base_reg differs from holder_reg, the offset was too large and we must have 427 // emitted an explicit null check before the load. Otherwise, we need to null-check 428 // the holder as we do not necessarily do that check before going to the thunk. 429 vixl::aarch64::Label throw_npe; 430 if (holder_reg.Is(base_reg)) { 431 __ Cbz(holder_reg.W(), &throw_npe); 432 } 433 vixl::aarch64::Label slow_path; 434 MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); 435 EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); 436 __ Bind(&slow_path); 437 MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); 438 __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. 439 __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. 440 __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. 441 // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. 442 __ Br(ip1); // Jump to the entrypoint. 443 if (holder_reg.Is(base_reg)) { 444 // Add null check slow path. The stack map is at the address pointed to by LR. 445 __ Bind(&throw_npe); 446 int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value(); 447 __ Ldr(ip0, MemOperand(vixl::aarch64::x19, offset)); 448 __ Br(ip0); 449 } 450 break; 451 } 452 case ThunkType::kBakerReadBarrierRoot: { 453 // Check if the reference needs to be marked and if so (i.e. not null, not marked yet 454 // and it does not have a forwarding address), call the correct introspection entrypoint; 455 // otherwise return the reference (or the extracted forwarding address). 456 // There is no gray bit check for GC roots. 457 auto root_reg = Register::GetWRegFromCode(key.GetRootParams().root_reg); 458 UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); 459 temps.Exclude(ip0, ip1); 460 vixl::aarch64::Label return_label, not_marked, forwarding_address; 461 __ Cbz(root_reg, &return_label); 462 MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value()); 463 __ Ldr(ip0.W(), lock_word); 464 __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked); 465 __ Bind(&return_label); 466 __ Br(lr); 467 __ Bind(¬_marked); 468 __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); 469 __ B(&forwarding_address, mi); 470 // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to 471 // art_quick_read_barrier_mark_introspection_gc_roots. 472 __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); 473 __ Mov(ip0.W(), root_reg); 474 __ Br(ip1); 475 __ Bind(&forwarding_address); 476 __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift); 477 __ Br(lr); 478 break; 479 } 480 } 481 482 // Ensure we emit the literal pool. 483 assembler.FinalizeCode(); 484 std::vector<uint8_t> thunk_code(assembler.CodeSize()); 485 MemoryRegion code(thunk_code.data(), thunk_code.size()); 486 assembler.FinalizeInstructions(code); 487 return thunk_code; 488 } 489 490 #undef __ 491 492 uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(ThunkType type) { 493 switch (type) { 494 case ThunkType::kMethodCall: 495 return kMaxMethodCallPositiveDisplacement; 496 case ThunkType::kBakerReadBarrierField: 497 case ThunkType::kBakerReadBarrierRoot: 498 return kMaxBcondPositiveDisplacement; 499 } 500 } 501 502 uint32_t Arm64RelativePatcher::MaxNegativeDisplacement(ThunkType type) { 503 switch (type) { 504 case ThunkType::kMethodCall: 505 return kMaxMethodCallNegativeDisplacement; 506 case ThunkType::kBakerReadBarrierField: 507 case ThunkType::kBakerReadBarrierRoot: 508 return kMaxBcondNegativeDisplacement; 509 } 510 } 511 512 uint32_t Arm64RelativePatcher::PatchAdrp(uint32_t adrp, uint32_t disp) { 513 return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. 514 // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. 515 ((disp & 0x00003000u) << (29 - 12)) | 516 // The next 16 bits are encoded in bits 5-22. 517 ((disp & 0xffffc000u) >> (12 + 2 - 5)) | 518 // Since the target_offset is based on the beginning of the oat file and the 519 // image space precedes the oat file, the target_offset into image space will 520 // be negative yet passed as uint32_t. Therefore we limit the displacement 521 // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from 522 // the highest bit of the displacement. This is encoded in bit 23. 523 ((disp & 0x80000000u) >> (31 - 23)); 524 } 525 526 bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, 527 uint32_t literal_offset, 528 uint32_t patch_offset) { 529 DCHECK_EQ(patch_offset & 0x3u, 0u); 530 if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc 531 uint32_t adrp = GetInsn(code, literal_offset); 532 DCHECK_EQ(adrp & 0x9f000000, 0x90000000); 533 uint32_t next_offset = patch_offset + 4u; 534 uint32_t next_insn = GetInsn(code, literal_offset + 4u); 535 536 // Below we avoid patching sequences where the adrp is followed by a load which can easily 537 // be proved to be aligned. 538 539 // First check if the next insn is the LDR using the result of the ADRP. 540 // LDR <Wt>, [<Xn>, #pimm], where <Xn> == ADRP destination reg. 541 if ((next_insn & 0xffc00000) == 0xb9400000 && 542 (((next_insn >> 5) ^ adrp) & 0x1f) == 0) { 543 return false; 544 } 545 546 // And since LinkerPatch::Type::kStringRelative is using the result of the ADRP 547 // for an ADD immediate, check for that as well. We generalize a bit to include 548 // ADD/ADDS/SUB/SUBS immediate that either uses the ADRP destination or stores 549 // the result to a different register. 550 if ((next_insn & 0x1f000000) == 0x11000000 && 551 ((((next_insn >> 5) ^ adrp) & 0x1f) == 0 || ((next_insn ^ adrp) & 0x1f) != 0)) { 552 return false; 553 } 554 555 // LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing. 556 if ((next_insn & 0xff000000) == 0x18000000) { 557 return false; 558 } 559 560 // LDR <Xt>, <label> is aligned iff the pc + displacement is a multiple of 8. 561 if ((next_insn & 0xff000000) == 0x58000000) { 562 bool is_aligned_load = (((next_offset >> 2) ^ (next_insn >> 5)) & 1) == 0; 563 return !is_aligned_load; 564 } 565 566 // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned loads, as SP is 567 // guaranteed to be 128-bits aligned and <pimm> is multiple of the load size. 568 if ((next_insn & 0xbfc003e0) == 0xb94003e0) { 569 return false; 570 } 571 return true; 572 } 573 return false; 574 } 575 576 void Arm64RelativePatcher::SetInsn(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { 577 DCHECK_LE(offset + 4u, code->size()); 578 DCHECK_EQ(offset & 3u, 0u); 579 uint8_t* addr = &(*code)[offset]; 580 addr[0] = (value >> 0) & 0xff; 581 addr[1] = (value >> 8) & 0xff; 582 addr[2] = (value >> 16) & 0xff; 583 addr[3] = (value >> 24) & 0xff; 584 } 585 586 uint32_t Arm64RelativePatcher::GetInsn(ArrayRef<const uint8_t> code, uint32_t offset) { 587 DCHECK_LE(offset + 4u, code.size()); 588 DCHECK_EQ(offset & 3u, 0u); 589 const uint8_t* addr = &code[offset]; 590 return 591 (static_cast<uint32_t>(addr[0]) << 0) + 592 (static_cast<uint32_t>(addr[1]) << 8) + 593 (static_cast<uint32_t>(addr[2]) << 16)+ 594 (static_cast<uint32_t>(addr[3]) << 24); 595 } 596 597 template <typename Alloc> 598 uint32_t Arm64RelativePatcher::GetInsn(std::vector<uint8_t, Alloc>* code, uint32_t offset) { 599 return GetInsn(ArrayRef<const uint8_t>(*code), offset); 600 } 601 602 } // namespace linker 603 } // namespace art 604
