1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "linker/arm64/relative_patcher_arm64.h" 18 19 #include "arch/arm64/instruction_set_features_arm64.h" 20 #include "art_method.h" 21 #include "compiled_method.h" 22 #include "driver/compiler_driver.h" 23 #include "utils/arm64/assembler_arm64.h" 24 #include "oat.h" 25 #include "output_stream.h" 26 27 namespace art { 28 namespace linker { 29 30 Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, 31 const Arm64InstructionSetFeatures* features) 32 : ArmBaseRelativePatcher(provider, kArm64, CompileThunkCode(), 33 kMaxPositiveDisplacement, kMaxNegativeDisplacement), 34 fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), 35 reserved_adrp_thunks_(0u), 36 processed_adrp_thunks_(0u) { 37 if (fix_cortex_a53_843419_) { 38 adrp_thunk_locations_.reserve(16u); 39 current_method_thunks_.reserve(16u * kAdrpThunkSize); 40 } 41 } 42 43 uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, 44 const CompiledMethod* compiled_method, 45 MethodReference method_ref) { 46 if (!fix_cortex_a53_843419_) { 47 DCHECK(adrp_thunk_locations_.empty()); 48 return ReserveSpaceInternal(offset, compiled_method, method_ref, 0u); 49 } 50 51 // Add thunks for previous method if any. 52 if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { 53 size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; 54 offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; 55 reserved_adrp_thunks_ = adrp_thunk_locations_.size(); 56 } 57 58 // Count the number of ADRP insns as the upper bound on the number of thunks needed 59 // and use it to reserve space for other linker patches. 60 size_t num_adrp = 0u; 61 DCHECK(compiled_method != nullptr); 62 for (const LinkerPatch& patch : compiled_method->GetPatches()) { 63 if (patch.Type() == kLinkerPatchDexCacheArray && 64 patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch 65 ++num_adrp; 66 } 67 } 68 offset = ReserveSpaceInternal(offset, compiled_method, method_ref, kAdrpThunkSize * num_adrp); 69 if (num_adrp == 0u) { 70 return offset; 71 } 72 73 // Now that we have the actual offset where the code will be placed, locate the ADRP insns 74 // that actually require the thunk. 75 uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); 76 ArrayRef<const uint8_t> code(*compiled_method->GetQuickCode()); 77 uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); 78 DCHECK(compiled_method != nullptr); 79 for (const LinkerPatch& patch : compiled_method->GetPatches()) { 80 if (patch.Type() == kLinkerPatchDexCacheArray && 81 patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch 82 uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); 83 if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { 84 adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); 85 thunk_offset += kAdrpThunkSize; 86 } 87 } 88 } 89 return offset; 90 } 91 92 uint32_t Arm64RelativePatcher::ReserveSpaceEnd(uint32_t offset) { 93 if (!fix_cortex_a53_843419_) { 94 DCHECK(adrp_thunk_locations_.empty()); 95 } else { 96 // Add thunks for the last method if any. 97 if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { 98 size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; 99 offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; 100 reserved_adrp_thunks_ = adrp_thunk_locations_.size(); 101 } 102 } 103 return ArmBaseRelativePatcher::ReserveSpaceEnd(offset); 104 } 105 106 uint32_t Arm64RelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { 107 if (fix_cortex_a53_843419_) { 108 if (!current_method_thunks_.empty()) { 109 uint32_t aligned_offset = CompiledMethod::AlignCode(offset, kArm64); 110 if (kIsDebugBuild) { 111 CHECK(IsAligned<kAdrpThunkSize>(current_method_thunks_.size())); 112 size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; 113 CHECK_LE(num_thunks, processed_adrp_thunks_); 114 for (size_t i = 0u; i != num_thunks; ++i) { 115 const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; 116 CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); 117 } 118 } 119 uint32_t aligned_code_delta = aligned_offset - offset; 120 if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { 121 return 0u; 122 } 123 if (!WriteMiscThunk(out, ArrayRef<const uint8_t>(current_method_thunks_))) { 124 return 0u; 125 } 126 offset = aligned_offset + current_method_thunks_.size(); 127 current_method_thunks_.clear(); 128 } 129 } 130 return ArmBaseRelativePatcher::WriteThunks(out, offset); 131 } 132 133 void Arm64RelativePatcher::PatchCall(std::vector<uint8_t>* code, uint32_t literal_offset, 134 uint32_t patch_offset, uint32_t target_offset) { 135 DCHECK_LE(literal_offset + 4u, code->size()); 136 DCHECK_EQ(literal_offset & 3u, 0u); 137 DCHECK_EQ(patch_offset & 3u, 0u); 138 DCHECK_EQ(target_offset & 3u, 0u); 139 uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); 140 DCHECK_EQ(displacement & 3u, 0u); 141 DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. 142 uint32_t insn = (displacement & 0x0fffffffu) >> 2; 143 insn |= 0x94000000; // BL 144 145 // Check that we're just overwriting an existing BL. 146 DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); 147 // Write the new BL. 148 SetInsn(code, literal_offset, insn); 149 } 150 151 void Arm64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, 152 const LinkerPatch& patch, 153 uint32_t patch_offset, 154 uint32_t target_offset) { 155 DCHECK_EQ(patch_offset & 3u, 0u); 156 DCHECK_EQ(target_offset & 3u, 0u); 157 uint32_t literal_offset = patch.LiteralOffset(); 158 uint32_t insn = GetInsn(code, literal_offset); 159 uint32_t pc_insn_offset = patch.PcInsnOffset(); 160 uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); 161 bool wide = (insn & 0x40000000) != 0; 162 uint32_t shift = wide ? 3u : 2u; 163 if (literal_offset == pc_insn_offset) { 164 // Check it's an ADRP with imm == 0 (unset). 165 DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) 166 << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; 167 if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && 168 adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { 169 DCHECK(NeedsErratum843419Thunk(ArrayRef<const uint8_t>(*code), 170 literal_offset, patch_offset)); 171 uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; 172 uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); 173 uint32_t adrp = PatchAdrp(insn, adrp_disp); 174 175 uint32_t out_disp = thunk_offset - patch_offset; 176 DCHECK_EQ(out_disp & 3u, 0u); 177 DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. 178 insn = (out_disp & 0x0fffffffu) >> shift; 179 insn |= 0x14000000; // B <thunk> 180 181 uint32_t back_disp = -out_disp; 182 DCHECK_EQ(back_disp & 3u, 0u); 183 DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. 184 uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; 185 b_back |= 0x14000000; // B <back> 186 size_t thunks_code_offset = current_method_thunks_.size(); 187 current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); 188 SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); 189 SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); 190 static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); 191 192 processed_adrp_thunks_ += 1u; 193 } else { 194 insn = PatchAdrp(insn, disp); 195 } 196 // Write the new ADRP (or B to the erratum 843419 thunk). 197 SetInsn(code, literal_offset, insn); 198 } else { 199 // LDR 32-bit or 64-bit with imm12 == 0 (unset). 200 DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << insn; 201 if (kIsDebugBuild) { 202 uint32_t adrp = GetInsn(code, pc_insn_offset); 203 if ((adrp & 0x9f000000u) != 0x90000000u) { 204 CHECK(fix_cortex_a53_843419_); 205 CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B <thunk> 206 CHECK(IsAligned<kAdrpThunkSize>(current_method_thunks_.size())); 207 size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; 208 CHECK_LE(num_thunks, processed_adrp_thunks_); 209 uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; 210 for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { 211 CHECK_NE(i, processed_adrp_thunks_); 212 if (adrp_thunk_locations_[i].first == b_offset) { 213 size_t idx = num_thunks - (processed_adrp_thunks_ - i); 214 adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); 215 break; 216 } 217 } 218 } 219 CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points 220 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. 221 } 222 uint32_t imm12 = (disp & 0xfffu) >> shift; 223 insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); 224 SetInsn(code, literal_offset, insn); 225 } 226 } 227 228 std::vector<uint8_t> Arm64RelativePatcher::CompileThunkCode() { 229 // The thunk just uses the entry point in the ArtMethod. This works even for calls 230 // to the generic JNI and interpreter trampolines. 231 arm64::Arm64Assembler assembler; 232 Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( 233 kArm64PointerSize).Int32Value()); 234 assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); 235 // Ensure we emit the literal pool. 236 assembler.EmitSlowPaths(); 237 std::vector<uint8_t> thunk_code(assembler.CodeSize()); 238 MemoryRegion code(thunk_code.data(), thunk_code.size()); 239 assembler.FinalizeInstructions(code); 240 return thunk_code; 241 } 242 243 uint32_t Arm64RelativePatcher::PatchAdrp(uint32_t adrp, uint32_t disp) { 244 return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. 245 // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. 246 ((disp & 0x00003000u) << (29 - 12)) | 247 // The next 16 bits are encoded in bits 5-22. 248 ((disp & 0xffffc000u) >> (12 + 2 - 5)) | 249 // Since the target_offset is based on the beginning of the oat file and the 250 // image space precedes the oat file, the target_offset into image space will 251 // be negative yet passed as uint32_t. Therefore we limit the displacement 252 // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from 253 // the highest bit of the displacement. This is encoded in bit 23. 254 ((disp & 0x80000000u) >> (31 - 23)); 255 } 256 257 bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, 258 uint32_t literal_offset, 259 uint32_t patch_offset) { 260 DCHECK_EQ(patch_offset & 0x3u, 0u); 261 if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc 262 uint32_t adrp = GetInsn(code, literal_offset); 263 DCHECK_EQ(adrp & 0xff000000, 0x90000000); 264 uint32_t next_offset = patch_offset + 4u; 265 uint32_t next_insn = GetInsn(code, literal_offset + 4u); 266 267 // Below we avoid patching sequences where the adrp is followed by a load which can easily 268 // be proved to be aligned. 269 270 // First check if the next insn is the LDR using the result of the ADRP. 271 // LDR <Wt>, [<Xn>, #pimm], where <Xn> == ADRP destination reg. 272 if ((next_insn & 0xffc00000) == 0xb9400000 && 273 (((next_insn >> 5) ^ adrp) & 0x1f) == 0) { 274 return false; 275 } 276 277 // LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing. 278 if ((next_insn & 0xff000000) == 0x18000000) { 279 return false; 280 } 281 282 // LDR <Xt>, <label> is aligned iff the pc + displacement is a multiple of 8. 283 if ((next_insn & 0xff000000) == 0x58000000) { 284 bool is_aligned_load = (((next_offset >> 2) ^ (next_insn >> 5)) & 1) == 0; 285 return !is_aligned_load; 286 } 287 288 // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned loads, as SP is 289 // guaranteed to be 128-bits aligned and <pimm> is multiple of the load size. 290 if ((next_insn & 0xbfc003e0) == 0xb94003e0) { 291 return false; 292 } 293 return true; 294 } 295 return false; 296 } 297 298 void Arm64RelativePatcher::SetInsn(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { 299 DCHECK_LE(offset + 4u, code->size()); 300 DCHECK_EQ(offset & 3u, 0u); 301 uint8_t* addr = &(*code)[offset]; 302 addr[0] = (value >> 0) & 0xff; 303 addr[1] = (value >> 8) & 0xff; 304 addr[2] = (value >> 16) & 0xff; 305 addr[3] = (value >> 24) & 0xff; 306 } 307 308 uint32_t Arm64RelativePatcher::GetInsn(ArrayRef<const uint8_t> code, uint32_t offset) { 309 DCHECK_LE(offset + 4u, code.size()); 310 DCHECK_EQ(offset & 3u, 0u); 311 const uint8_t* addr = &code[offset]; 312 return 313 (static_cast<uint32_t>(addr[0]) << 0) + 314 (static_cast<uint32_t>(addr[1]) << 8) + 315 (static_cast<uint32_t>(addr[2]) << 16)+ 316 (static_cast<uint32_t>(addr[3]) << 24); 317 } 318 319 template <typename Alloc> 320 uint32_t Arm64RelativePatcher::GetInsn(std::vector<uint8_t, Alloc>* code, uint32_t offset) { 321 return GetInsn(ArrayRef<const uint8_t>(*code), offset); 322 } 323 324 } // namespace linker 325 } // namespace art 326