1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "assembler_arm64.h" 18 #include "base/logging.h" 19 #include "entrypoints/quick/quick_entrypoints.h" 20 #include "offsets.h" 21 #include "thread.h" 22 #include "utils.h" 23 24 namespace art { 25 namespace arm64 { 26 27 #ifdef ___ 28 #error "ARM64 Assembler macro already defined." 29 #else 30 #define ___ vixl_masm_-> 31 #endif 32 33 void Arm64Assembler::EmitSlowPaths() { 34 if (!exception_blocks_.empty()) { 35 for (size_t i = 0; i < exception_blocks_.size(); i++) { 36 EmitExceptionPoll(exception_blocks_.at(i)); 37 } 38 } 39 ___ FinalizeCode(); 40 } 41 42 size_t Arm64Assembler::CodeSize() const { 43 return ___ SizeOfCodeGenerated(); 44 } 45 46 void Arm64Assembler::FinalizeInstructions(const MemoryRegion& region) { 47 // Copy the instructions from the buffer. 48 MemoryRegion from(reinterpret_cast<void*>(vixl_buf_), CodeSize()); 49 region.CopyFrom(0, from); 50 } 51 52 void Arm64Assembler::GetCurrentThread(ManagedRegister tr) { 53 ___ Mov(reg_x(tr.AsArm64().AsCoreRegister()), reg_x(ETR)); 54 } 55 56 void Arm64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister /* scratch */) { 57 StoreToOffset(ETR, SP, offset.Int32Value()); 58 } 59 60 // See Arm64 PCS Section 5.2.2.1. 61 void Arm64Assembler::IncreaseFrameSize(size_t adjust) { 62 CHECK_ALIGNED(adjust, kStackAlignment); 63 AddConstant(SP, -adjust); 64 } 65 66 // See Arm64 PCS Section 5.2.2.1. 67 void Arm64Assembler::DecreaseFrameSize(size_t adjust) { 68 CHECK_ALIGNED(adjust, kStackAlignment); 69 AddConstant(SP, adjust); 70 } 71 72 void Arm64Assembler::AddConstant(Register rd, int32_t value, Condition cond) { 73 AddConstant(rd, rd, value, cond); 74 } 75 76 void Arm64Assembler::AddConstant(Register rd, Register rn, int32_t value, 77 Condition cond) { 78 if ((cond == AL) || (cond == NV)) { 79 // VIXL macro-assembler handles all variants. 80 ___ Add(reg_x(rd), reg_x(rn), value); 81 } else { 82 // temp = rd + value 83 // rd = cond ? temp : rn 84 vixl::UseScratchRegisterScope temps(vixl_masm_); 85 temps.Exclude(reg_x(rd), reg_x(rn)); 86 vixl::Register temp = temps.AcquireX(); 87 ___ Add(temp, reg_x(rn), value); 88 ___ Csel(reg_x(rd), temp, reg_x(rd), COND_OP(cond)); 89 } 90 } 91 92 void Arm64Assembler::StoreWToOffset(StoreOperandType type, WRegister source, 93 Register base, int32_t offset) { 94 switch (type) { 95 case kStoreByte: 96 ___ Strb(reg_w(source), MEM_OP(reg_x(base), offset)); 97 break; 98 case kStoreHalfword: 99 ___ Strh(reg_w(source), MEM_OP(reg_x(base), offset)); 100 break; 101 case kStoreWord: 102 ___ Str(reg_w(source), MEM_OP(reg_x(base), offset)); 103 break; 104 default: 105 LOG(FATAL) << "UNREACHABLE"; 106 } 107 } 108 109 void Arm64Assembler::StoreToOffset(Register source, Register base, int32_t offset) { 110 CHECK_NE(source, SP); 111 ___ Str(reg_x(source), MEM_OP(reg_x(base), offset)); 112 } 113 114 void Arm64Assembler::StoreSToOffset(SRegister source, Register base, int32_t offset) { 115 ___ Str(reg_s(source), MEM_OP(reg_x(base), offset)); 116 } 117 118 void Arm64Assembler::StoreDToOffset(DRegister source, Register base, int32_t offset) { 119 ___ Str(reg_d(source), MEM_OP(reg_x(base), offset)); 120 } 121 122 void Arm64Assembler::Store(FrameOffset offs, ManagedRegister m_src, size_t size) { 123 Arm64ManagedRegister src = m_src.AsArm64(); 124 if (src.IsNoRegister()) { 125 CHECK_EQ(0u, size); 126 } else if (src.IsWRegister()) { 127 CHECK_EQ(4u, size); 128 StoreWToOffset(kStoreWord, src.AsWRegister(), SP, offs.Int32Value()); 129 } else if (src.IsCoreRegister()) { 130 CHECK_EQ(8u, size); 131 StoreToOffset(src.AsCoreRegister(), SP, offs.Int32Value()); 132 } else if (src.IsSRegister()) { 133 StoreSToOffset(src.AsSRegister(), SP, offs.Int32Value()); 134 } else { 135 CHECK(src.IsDRegister()) << src; 136 StoreDToOffset(src.AsDRegister(), SP, offs.Int32Value()); 137 } 138 } 139 140 void Arm64Assembler::StoreRef(FrameOffset offs, ManagedRegister m_src) { 141 Arm64ManagedRegister src = m_src.AsArm64(); 142 CHECK(src.IsCoreRegister()) << src; 143 StoreWToOffset(kStoreWord, src.AsOverlappingCoreRegisterLow(), SP, 144 offs.Int32Value()); 145 } 146 147 void Arm64Assembler::StoreRawPtr(FrameOffset offs, ManagedRegister m_src) { 148 Arm64ManagedRegister src = m_src.AsArm64(); 149 CHECK(src.IsCoreRegister()) << src; 150 StoreToOffset(src.AsCoreRegister(), SP, offs.Int32Value()); 151 } 152 153 void Arm64Assembler::StoreImmediateToFrame(FrameOffset offs, uint32_t imm, 154 ManagedRegister m_scratch) { 155 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 156 CHECK(scratch.IsCoreRegister()) << scratch; 157 LoadImmediate(scratch.AsCoreRegister(), imm); 158 StoreWToOffset(kStoreWord, scratch.AsOverlappingCoreRegisterLow(), SP, 159 offs.Int32Value()); 160 } 161 162 void Arm64Assembler::StoreImmediateToThread64(ThreadOffset<8> offs, uint32_t imm, 163 ManagedRegister m_scratch) { 164 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 165 CHECK(scratch.IsCoreRegister()) << scratch; 166 LoadImmediate(scratch.AsCoreRegister(), imm); 167 StoreToOffset(scratch.AsCoreRegister(), ETR, offs.Int32Value()); 168 } 169 170 void Arm64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> tr_offs, 171 FrameOffset fr_offs, 172 ManagedRegister m_scratch) { 173 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 174 CHECK(scratch.IsCoreRegister()) << scratch; 175 AddConstant(scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); 176 StoreToOffset(scratch.AsCoreRegister(), ETR, tr_offs.Int32Value()); 177 } 178 179 void Arm64Assembler::StoreStackPointerToThread64(ThreadOffset<8> tr_offs) { 180 vixl::UseScratchRegisterScope temps(vixl_masm_); 181 vixl::Register temp = temps.AcquireX(); 182 ___ Mov(temp, reg_x(SP)); 183 ___ Str(temp, MEM_OP(reg_x(ETR), tr_offs.Int32Value())); 184 } 185 186 void Arm64Assembler::StoreSpanning(FrameOffset dest_off, ManagedRegister m_source, 187 FrameOffset in_off, ManagedRegister m_scratch) { 188 Arm64ManagedRegister source = m_source.AsArm64(); 189 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 190 StoreToOffset(source.AsCoreRegister(), SP, dest_off.Int32Value()); 191 LoadFromOffset(scratch.AsCoreRegister(), SP, in_off.Int32Value()); 192 StoreToOffset(scratch.AsCoreRegister(), SP, dest_off.Int32Value() + 8); 193 } 194 195 // Load routines. 196 void Arm64Assembler::LoadImmediate(Register dest, int32_t value, 197 Condition cond) { 198 if ((cond == AL) || (cond == NV)) { 199 ___ Mov(reg_x(dest), value); 200 } else { 201 // temp = value 202 // rd = cond ? temp : rd 203 if (value != 0) { 204 vixl::UseScratchRegisterScope temps(vixl_masm_); 205 temps.Exclude(reg_x(dest)); 206 vixl::Register temp = temps.AcquireX(); 207 ___ Mov(temp, value); 208 ___ Csel(reg_x(dest), temp, reg_x(dest), COND_OP(cond)); 209 } else { 210 ___ Csel(reg_x(dest), reg_x(XZR), reg_x(dest), COND_OP(cond)); 211 } 212 } 213 } 214 215 void Arm64Assembler::LoadWFromOffset(LoadOperandType type, WRegister dest, 216 Register base, int32_t offset) { 217 switch (type) { 218 case kLoadSignedByte: 219 ___ Ldrsb(reg_w(dest), MEM_OP(reg_x(base), offset)); 220 break; 221 case kLoadSignedHalfword: 222 ___ Ldrsh(reg_w(dest), MEM_OP(reg_x(base), offset)); 223 break; 224 case kLoadUnsignedByte: 225 ___ Ldrb(reg_w(dest), MEM_OP(reg_x(base), offset)); 226 break; 227 case kLoadUnsignedHalfword: 228 ___ Ldrh(reg_w(dest), MEM_OP(reg_x(base), offset)); 229 break; 230 case kLoadWord: 231 ___ Ldr(reg_w(dest), MEM_OP(reg_x(base), offset)); 232 break; 233 default: 234 LOG(FATAL) << "UNREACHABLE"; 235 } 236 } 237 238 // Note: We can extend this member by adding load type info - see 239 // sign extended A64 load variants. 240 void Arm64Assembler::LoadFromOffset(Register dest, Register base, 241 int32_t offset) { 242 CHECK_NE(dest, SP); 243 ___ Ldr(reg_x(dest), MEM_OP(reg_x(base), offset)); 244 } 245 246 void Arm64Assembler::LoadSFromOffset(SRegister dest, Register base, 247 int32_t offset) { 248 ___ Ldr(reg_s(dest), MEM_OP(reg_x(base), offset)); 249 } 250 251 void Arm64Assembler::LoadDFromOffset(DRegister dest, Register base, 252 int32_t offset) { 253 ___ Ldr(reg_d(dest), MEM_OP(reg_x(base), offset)); 254 } 255 256 void Arm64Assembler::Load(Arm64ManagedRegister dest, Register base, 257 int32_t offset, size_t size) { 258 if (dest.IsNoRegister()) { 259 CHECK_EQ(0u, size) << dest; 260 } else if (dest.IsWRegister()) { 261 CHECK_EQ(4u, size) << dest; 262 ___ Ldr(reg_w(dest.AsWRegister()), MEM_OP(reg_x(base), offset)); 263 } else if (dest.IsCoreRegister()) { 264 CHECK_NE(dest.AsCoreRegister(), SP) << dest; 265 if (size == 4u) { 266 ___ Ldr(reg_w(dest.AsOverlappingCoreRegisterLow()), MEM_OP(reg_x(base), offset)); 267 } else { 268 CHECK_EQ(8u, size) << dest; 269 ___ Ldr(reg_x(dest.AsCoreRegister()), MEM_OP(reg_x(base), offset)); 270 } 271 } else if (dest.IsSRegister()) { 272 ___ Ldr(reg_s(dest.AsSRegister()), MEM_OP(reg_x(base), offset)); 273 } else { 274 CHECK(dest.IsDRegister()) << dest; 275 ___ Ldr(reg_d(dest.AsDRegister()), MEM_OP(reg_x(base), offset)); 276 } 277 } 278 279 void Arm64Assembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) { 280 return Load(m_dst.AsArm64(), SP, src.Int32Value(), size); 281 } 282 283 void Arm64Assembler::LoadFromThread64(ManagedRegister m_dst, ThreadOffset<8> src, size_t size) { 284 return Load(m_dst.AsArm64(), ETR, src.Int32Value(), size); 285 } 286 287 void Arm64Assembler::LoadRef(ManagedRegister m_dst, FrameOffset offs) { 288 Arm64ManagedRegister dst = m_dst.AsArm64(); 289 CHECK(dst.IsCoreRegister()) << dst; 290 LoadWFromOffset(kLoadWord, dst.AsOverlappingCoreRegisterLow(), SP, offs.Int32Value()); 291 } 292 293 void Arm64Assembler::LoadRef(ManagedRegister m_dst, ManagedRegister m_base, 294 MemberOffset offs) { 295 Arm64ManagedRegister dst = m_dst.AsArm64(); 296 Arm64ManagedRegister base = m_base.AsArm64(); 297 CHECK(dst.IsCoreRegister() && base.IsCoreRegister()); 298 LoadWFromOffset(kLoadWord, dst.AsOverlappingCoreRegisterLow(), base.AsCoreRegister(), 299 offs.Int32Value()); 300 } 301 302 void Arm64Assembler::LoadRawPtr(ManagedRegister m_dst, ManagedRegister m_base, Offset offs) { 303 Arm64ManagedRegister dst = m_dst.AsArm64(); 304 Arm64ManagedRegister base = m_base.AsArm64(); 305 CHECK(dst.IsCoreRegister() && base.IsCoreRegister()); 306 // Remove dst and base form the temp list - higher level API uses IP1, IP0. 307 vixl::UseScratchRegisterScope temps(vixl_masm_); 308 temps.Exclude(reg_x(dst.AsCoreRegister()), reg_x(base.AsCoreRegister())); 309 ___ Ldr(reg_x(dst.AsCoreRegister()), MEM_OP(reg_x(base.AsCoreRegister()), offs.Int32Value())); 310 } 311 312 void Arm64Assembler::LoadRawPtrFromThread64(ManagedRegister m_dst, ThreadOffset<8> offs) { 313 Arm64ManagedRegister dst = m_dst.AsArm64(); 314 CHECK(dst.IsCoreRegister()) << dst; 315 LoadFromOffset(dst.AsCoreRegister(), ETR, offs.Int32Value()); 316 } 317 318 // Copying routines. 319 void Arm64Assembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t size) { 320 Arm64ManagedRegister dst = m_dst.AsArm64(); 321 Arm64ManagedRegister src = m_src.AsArm64(); 322 if (!dst.Equals(src)) { 323 if (dst.IsCoreRegister()) { 324 if (size == 4) { 325 CHECK(src.IsWRegister()); 326 ___ Mov(reg_x(dst.AsCoreRegister()), reg_w(src.AsWRegister())); 327 } else { 328 if (src.IsCoreRegister()) { 329 ___ Mov(reg_x(dst.AsCoreRegister()), reg_x(src.AsCoreRegister())); 330 } else { 331 ___ Mov(reg_x(dst.AsCoreRegister()), reg_w(src.AsWRegister())); 332 } 333 } 334 } else if (dst.IsWRegister()) { 335 CHECK(src.IsWRegister()) << src; 336 ___ Mov(reg_w(dst.AsWRegister()), reg_w(src.AsWRegister())); 337 } else if (dst.IsSRegister()) { 338 CHECK(src.IsSRegister()) << src; 339 ___ Fmov(reg_s(dst.AsSRegister()), reg_s(src.AsSRegister())); 340 } else { 341 CHECK(dst.IsDRegister()) << dst; 342 CHECK(src.IsDRegister()) << src; 343 ___ Fmov(reg_d(dst.AsDRegister()), reg_d(src.AsDRegister())); 344 } 345 } 346 } 347 348 void Arm64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 349 ThreadOffset<8> tr_offs, 350 ManagedRegister m_scratch) { 351 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 352 CHECK(scratch.IsCoreRegister()) << scratch; 353 LoadFromOffset(scratch.AsCoreRegister(), ETR, tr_offs.Int32Value()); 354 StoreToOffset(scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); 355 } 356 357 void Arm64Assembler::CopyRawPtrToThread64(ThreadOffset<8> tr_offs, 358 FrameOffset fr_offs, 359 ManagedRegister m_scratch) { 360 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 361 CHECK(scratch.IsCoreRegister()) << scratch; 362 LoadFromOffset(scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); 363 StoreToOffset(scratch.AsCoreRegister(), ETR, tr_offs.Int32Value()); 364 } 365 366 void Arm64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 367 ManagedRegister m_scratch) { 368 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 369 CHECK(scratch.IsCoreRegister()) << scratch; 370 LoadWFromOffset(kLoadWord, scratch.AsOverlappingCoreRegisterLow(), 371 SP, src.Int32Value()); 372 StoreWToOffset(kStoreWord, scratch.AsOverlappingCoreRegisterLow(), 373 SP, dest.Int32Value()); 374 } 375 376 void Arm64Assembler::Copy(FrameOffset dest, FrameOffset src, 377 ManagedRegister m_scratch, size_t size) { 378 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 379 CHECK(scratch.IsCoreRegister()) << scratch; 380 CHECK(size == 4 || size == 8) << size; 381 if (size == 4) { 382 LoadWFromOffset(kLoadWord, scratch.AsOverlappingCoreRegisterLow(), SP, src.Int32Value()); 383 StoreWToOffset(kStoreWord, scratch.AsOverlappingCoreRegisterLow(), SP, dest.Int32Value()); 384 } else if (size == 8) { 385 LoadFromOffset(scratch.AsCoreRegister(), SP, src.Int32Value()); 386 StoreToOffset(scratch.AsCoreRegister(), SP, dest.Int32Value()); 387 } else { 388 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8"; 389 } 390 } 391 392 void Arm64Assembler::Copy(FrameOffset dest, ManagedRegister src_base, Offset src_offset, 393 ManagedRegister m_scratch, size_t size) { 394 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 395 Arm64ManagedRegister base = src_base.AsArm64(); 396 CHECK(base.IsCoreRegister()) << base; 397 CHECK(scratch.IsCoreRegister() || scratch.IsWRegister()) << scratch; 398 CHECK(size == 4 || size == 8) << size; 399 if (size == 4) { 400 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), base.AsCoreRegister(), 401 src_offset.Int32Value()); 402 StoreWToOffset(kStoreWord, scratch.AsWRegister(), SP, dest.Int32Value()); 403 } else if (size == 8) { 404 LoadFromOffset(scratch.AsCoreRegister(), base.AsCoreRegister(), src_offset.Int32Value()); 405 StoreToOffset(scratch.AsCoreRegister(), SP, dest.Int32Value()); 406 } else { 407 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8"; 408 } 409 } 410 411 void Arm64Assembler::Copy(ManagedRegister m_dest_base, Offset dest_offs, FrameOffset src, 412 ManagedRegister m_scratch, size_t size) { 413 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 414 Arm64ManagedRegister base = m_dest_base.AsArm64(); 415 CHECK(base.IsCoreRegister()) << base; 416 CHECK(scratch.IsCoreRegister() || scratch.IsWRegister()) << scratch; 417 CHECK(size == 4 || size == 8) << size; 418 if (size == 4) { 419 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), SP, src.Int32Value()); 420 StoreWToOffset(kStoreWord, scratch.AsWRegister(), base.AsCoreRegister(), 421 dest_offs.Int32Value()); 422 } else if (size == 8) { 423 LoadFromOffset(scratch.AsCoreRegister(), SP, src.Int32Value()); 424 StoreToOffset(scratch.AsCoreRegister(), base.AsCoreRegister(), dest_offs.Int32Value()); 425 } else { 426 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8"; 427 } 428 } 429 430 void Arm64Assembler::Copy(FrameOffset /*dst*/, FrameOffset /*src_base*/, Offset /*src_offset*/, 431 ManagedRegister /*mscratch*/, size_t /*size*/) { 432 UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant"; 433 } 434 435 void Arm64Assembler::Copy(ManagedRegister m_dest, Offset dest_offset, 436 ManagedRegister m_src, Offset src_offset, 437 ManagedRegister m_scratch, size_t size) { 438 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 439 Arm64ManagedRegister src = m_src.AsArm64(); 440 Arm64ManagedRegister dest = m_dest.AsArm64(); 441 CHECK(dest.IsCoreRegister()) << dest; 442 CHECK(src.IsCoreRegister()) << src; 443 CHECK(scratch.IsCoreRegister() || scratch.IsWRegister()) << scratch; 444 CHECK(size == 4 || size == 8) << size; 445 if (size == 4) { 446 if (scratch.IsWRegister()) { 447 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), src.AsCoreRegister(), 448 src_offset.Int32Value()); 449 StoreWToOffset(kStoreWord, scratch.AsWRegister(), dest.AsCoreRegister(), 450 dest_offset.Int32Value()); 451 } else { 452 LoadWFromOffset(kLoadWord, scratch.AsOverlappingCoreRegisterLow(), src.AsCoreRegister(), 453 src_offset.Int32Value()); 454 StoreWToOffset(kStoreWord, scratch.AsOverlappingCoreRegisterLow(), dest.AsCoreRegister(), 455 dest_offset.Int32Value()); 456 } 457 } else if (size == 8) { 458 LoadFromOffset(scratch.AsCoreRegister(), src.AsCoreRegister(), src_offset.Int32Value()); 459 StoreToOffset(scratch.AsCoreRegister(), dest.AsCoreRegister(), dest_offset.Int32Value()); 460 } else { 461 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8"; 462 } 463 } 464 465 void Arm64Assembler::Copy(FrameOffset /*dst*/, Offset /*dest_offset*/, 466 FrameOffset /*src*/, Offset /*src_offset*/, 467 ManagedRegister /*scratch*/, size_t /*size*/) { 468 UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant"; 469 } 470 471 void Arm64Assembler::MemoryBarrier(ManagedRegister m_scratch) { 472 // TODO: Should we check that m_scratch is IP? - see arm. 473 #if ANDROID_SMP != 0 474 ___ Dmb(vixl::InnerShareable, vixl::BarrierAll); 475 #endif 476 } 477 478 void Arm64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 479 Arm64ManagedRegister reg = mreg.AsArm64(); 480 CHECK(size == 1 || size == 2) << size; 481 CHECK(reg.IsWRegister()) << reg; 482 if (size == 1) { 483 ___ sxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister())); 484 } else { 485 ___ sxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister())); 486 } 487 } 488 489 void Arm64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 490 Arm64ManagedRegister reg = mreg.AsArm64(); 491 CHECK(size == 1 || size == 2) << size; 492 CHECK(reg.IsWRegister()) << reg; 493 if (size == 1) { 494 ___ uxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister())); 495 } else { 496 ___ uxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister())); 497 } 498 } 499 500 void Arm64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 501 // TODO: not validating references. 502 } 503 504 void Arm64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 505 // TODO: not validating references. 506 } 507 508 void Arm64Assembler::Call(ManagedRegister m_base, Offset offs, ManagedRegister m_scratch) { 509 Arm64ManagedRegister base = m_base.AsArm64(); 510 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 511 CHECK(base.IsCoreRegister()) << base; 512 CHECK(scratch.IsCoreRegister()) << scratch; 513 LoadFromOffset(scratch.AsCoreRegister(), base.AsCoreRegister(), offs.Int32Value()); 514 ___ Blr(reg_x(scratch.AsCoreRegister())); 515 } 516 517 void Arm64Assembler::JumpTo(ManagedRegister m_base, Offset offs, ManagedRegister m_scratch) { 518 Arm64ManagedRegister base = m_base.AsArm64(); 519 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 520 CHECK(base.IsCoreRegister()) << base; 521 CHECK(scratch.IsCoreRegister()) << scratch; 522 // Remove base and scratch form the temp list - higher level API uses IP1, IP0. 523 vixl::UseScratchRegisterScope temps(vixl_masm_); 524 temps.Exclude(reg_x(base.AsCoreRegister()), reg_x(scratch.AsCoreRegister())); 525 ___ Ldr(reg_x(scratch.AsCoreRegister()), MEM_OP(reg_x(base.AsCoreRegister()), offs.Int32Value())); 526 ___ Br(reg_x(scratch.AsCoreRegister())); 527 } 528 529 void Arm64Assembler::Call(FrameOffset base, Offset offs, ManagedRegister m_scratch) { 530 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 531 CHECK(scratch.IsCoreRegister()) << scratch; 532 // Call *(*(SP + base) + offset) 533 LoadWFromOffset(kLoadWord, scratch.AsOverlappingCoreRegisterLow(), SP, base.Int32Value()); 534 LoadFromOffset(scratch.AsCoreRegister(), scratch.AsCoreRegister(), offs.Int32Value()); 535 ___ Blr(reg_x(scratch.AsCoreRegister())); 536 } 537 538 void Arm64Assembler::CallFromThread64(ThreadOffset<8> /*offset*/, ManagedRegister /*scratch*/) { 539 UNIMPLEMENTED(FATAL) << "Unimplemented Call() variant"; 540 } 541 542 void Arm64Assembler::CreateHandleScopeEntry(ManagedRegister m_out_reg, FrameOffset handle_scope_offs, 543 ManagedRegister m_in_reg, bool null_allowed) { 544 Arm64ManagedRegister out_reg = m_out_reg.AsArm64(); 545 Arm64ManagedRegister in_reg = m_in_reg.AsArm64(); 546 // For now we only hold stale handle scope entries in x registers. 547 CHECK(in_reg.IsNoRegister() || in_reg.IsCoreRegister()) << in_reg; 548 CHECK(out_reg.IsCoreRegister()) << out_reg; 549 if (null_allowed) { 550 // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is 551 // the address in the handle scope holding the reference. 552 // e.g. out_reg = (handle == 0) ? 0 : (SP+handle_offset) 553 if (in_reg.IsNoRegister()) { 554 LoadWFromOffset(kLoadWord, out_reg.AsOverlappingCoreRegisterLow(), SP, 555 handle_scope_offs.Int32Value()); 556 in_reg = out_reg; 557 } 558 ___ Cmp(reg_w(in_reg.AsOverlappingCoreRegisterLow()), 0); 559 if (!out_reg.Equals(in_reg)) { 560 LoadImmediate(out_reg.AsCoreRegister(), 0, EQ); 561 } 562 AddConstant(out_reg.AsCoreRegister(), SP, handle_scope_offs.Int32Value(), NE); 563 } else { 564 AddConstant(out_reg.AsCoreRegister(), SP, handle_scope_offs.Int32Value(), AL); 565 } 566 } 567 568 void Arm64Assembler::CreateHandleScopeEntry(FrameOffset out_off, FrameOffset handle_scope_offset, 569 ManagedRegister m_scratch, bool null_allowed) { 570 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 571 CHECK(scratch.IsCoreRegister()) << scratch; 572 if (null_allowed) { 573 LoadWFromOffset(kLoadWord, scratch.AsOverlappingCoreRegisterLow(), SP, 574 handle_scope_offset.Int32Value()); 575 // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is 576 // the address in the handle scope holding the reference. 577 // e.g. scratch = (scratch == 0) ? 0 : (SP+handle_scope_offset) 578 ___ Cmp(reg_w(scratch.AsOverlappingCoreRegisterLow()), 0); 579 // Move this logic in add constants with flags. 580 AddConstant(scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), NE); 581 } else { 582 AddConstant(scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), AL); 583 } 584 StoreToOffset(scratch.AsCoreRegister(), SP, out_off.Int32Value()); 585 } 586 587 void Arm64Assembler::LoadReferenceFromHandleScope(ManagedRegister m_out_reg, 588 ManagedRegister m_in_reg) { 589 Arm64ManagedRegister out_reg = m_out_reg.AsArm64(); 590 Arm64ManagedRegister in_reg = m_in_reg.AsArm64(); 591 CHECK(out_reg.IsCoreRegister()) << out_reg; 592 CHECK(in_reg.IsCoreRegister()) << in_reg; 593 vixl::Label exit; 594 if (!out_reg.Equals(in_reg)) { 595 // FIXME: Who sets the flags here? 596 LoadImmediate(out_reg.AsCoreRegister(), 0, EQ); 597 } 598 ___ Cbz(reg_x(in_reg.AsCoreRegister()), &exit); 599 LoadFromOffset(out_reg.AsCoreRegister(), in_reg.AsCoreRegister(), 0); 600 ___ Bind(&exit); 601 } 602 603 void Arm64Assembler::ExceptionPoll(ManagedRegister m_scratch, size_t stack_adjust) { 604 CHECK_ALIGNED(stack_adjust, kStackAlignment); 605 Arm64ManagedRegister scratch = m_scratch.AsArm64(); 606 Arm64Exception *current_exception = new Arm64Exception(scratch, stack_adjust); 607 exception_blocks_.push_back(current_exception); 608 LoadFromOffset(scratch.AsCoreRegister(), ETR, Thread::ExceptionOffset<8>().Int32Value()); 609 ___ Cbnz(reg_x(scratch.AsCoreRegister()), current_exception->Entry()); 610 } 611 612 void Arm64Assembler::EmitExceptionPoll(Arm64Exception *exception) { 613 vixl::UseScratchRegisterScope temps(vixl_masm_); 614 temps.Exclude(reg_x(exception->scratch_.AsCoreRegister())); 615 vixl::Register temp = temps.AcquireX(); 616 617 // Bind exception poll entry. 618 ___ Bind(exception->Entry()); 619 if (exception->stack_adjust_ != 0) { // Fix up the frame. 620 DecreaseFrameSize(exception->stack_adjust_); 621 } 622 // Pass exception object as argument. 623 // Don't care about preserving X0 as this won't return. 624 ___ Mov(reg_x(X0), reg_x(exception->scratch_.AsCoreRegister())); 625 ___ Ldr(temp, MEM_OP(reg_x(ETR), QUICK_ENTRYPOINT_OFFSET(8, pDeliverException).Int32Value())); 626 627 // Move ETR(Callee saved) back to TR(Caller saved) reg. We use ETR on calls 628 // to external functions that might trash TR. We do not need the original 629 // ETR(X21) saved in BuildFrame(). 630 ___ Mov(reg_x(TR), reg_x(ETR)); 631 632 ___ Blr(temp); 633 // Call should never return. 634 ___ Brk(); 635 } 636 637 constexpr size_t kFramePointerSize = 8; 638 639 void Arm64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 640 const std::vector<ManagedRegister>& callee_save_regs, 641 const ManagedRegisterEntrySpills& entry_spills) { 642 CHECK_ALIGNED(frame_size, kStackAlignment); 643 CHECK(X0 == method_reg.AsArm64().AsCoreRegister()); 644 645 // TODO: *create APCS FP - end of FP chain; 646 // *add support for saving a different set of callee regs. 647 // For now we check that the size of callee regs vector is 11. 648 CHECK_EQ(callee_save_regs.size(), kJniRefSpillRegsSize); 649 // Increase frame to required size - must be at least space to push StackReference<Method>. 650 CHECK_GT(frame_size, kJniRefSpillRegsSize * kFramePointerSize); 651 IncreaseFrameSize(frame_size); 652 653 // TODO: Ugly hard code... 654 // Should generate these according to the spill mask automatically. 655 // TUNING: Use stp. 656 // Note: Must match Arm64JniCallingConvention::CoreSpillMask(). 657 size_t reg_offset = frame_size; 658 reg_offset -= 8; 659 StoreToOffset(LR, SP, reg_offset); 660 reg_offset -= 8; 661 StoreToOffset(X29, SP, reg_offset); 662 reg_offset -= 8; 663 StoreToOffset(X28, SP, reg_offset); 664 reg_offset -= 8; 665 StoreToOffset(X27, SP, reg_offset); 666 reg_offset -= 8; 667 StoreToOffset(X26, SP, reg_offset); 668 reg_offset -= 8; 669 StoreToOffset(X25, SP, reg_offset); 670 reg_offset -= 8; 671 StoreToOffset(X24, SP, reg_offset); 672 reg_offset -= 8; 673 StoreToOffset(X23, SP, reg_offset); 674 reg_offset -= 8; 675 StoreToOffset(X22, SP, reg_offset); 676 reg_offset -= 8; 677 StoreToOffset(X21, SP, reg_offset); 678 reg_offset -= 8; 679 StoreToOffset(X20, SP, reg_offset); 680 681 // Move TR(Caller saved) to ETR(Callee saved). The original (ETR)X21 has been saved on stack. 682 // This way we make sure that TR is not trashed by native code. 683 ___ Mov(reg_x(ETR), reg_x(TR)); 684 685 // Write StackReference<Method>. 686 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 687 StoreWToOffset(StoreOperandType::kStoreWord, W0, SP, 0); 688 689 // Write out entry spills 690 int32_t offset = frame_size + sizeof(StackReference<mirror::ArtMethod>); 691 for (size_t i = 0; i < entry_spills.size(); ++i) { 692 Arm64ManagedRegister reg = entry_spills.at(i).AsArm64(); 693 if (reg.IsNoRegister()) { 694 // only increment stack offset. 695 ManagedRegisterSpill spill = entry_spills.at(i); 696 offset += spill.getSize(); 697 } else if (reg.IsCoreRegister()) { 698 StoreToOffset(reg.AsCoreRegister(), SP, offset); 699 offset += 8; 700 } else if (reg.IsWRegister()) { 701 StoreWToOffset(kStoreWord, reg.AsWRegister(), SP, offset); 702 offset += 4; 703 } else if (reg.IsDRegister()) { 704 StoreDToOffset(reg.AsDRegister(), SP, offset); 705 offset += 8; 706 } else if (reg.IsSRegister()) { 707 StoreSToOffset(reg.AsSRegister(), SP, offset); 708 offset += 4; 709 } 710 } 711 } 712 713 void Arm64Assembler::RemoveFrame(size_t frame_size, const std::vector<ManagedRegister>& callee_save_regs) { 714 CHECK_ALIGNED(frame_size, kStackAlignment); 715 716 // For now we only check that the size of the frame is greater than the spill size. 717 CHECK_EQ(callee_save_regs.size(), kJniRefSpillRegsSize); 718 CHECK_GT(frame_size, kJniRefSpillRegsSize * kFramePointerSize); 719 720 // We move ETR(aapcs64 callee saved) back to TR(aapcs64 caller saved) which might have 721 // been trashed in the native call. The original ETR(X21) is restored from stack. 722 ___ Mov(reg_x(TR), reg_x(ETR)); 723 724 // TODO: Ugly hard code... 725 // Should generate these according to the spill mask automatically. 726 // TUNING: Use ldp. 727 // Note: Must match Arm64JniCallingConvention::CoreSpillMask(). 728 size_t reg_offset = frame_size; 729 reg_offset -= 8; 730 LoadFromOffset(LR, SP, reg_offset); 731 reg_offset -= 8; 732 LoadFromOffset(X29, SP, reg_offset); 733 reg_offset -= 8; 734 LoadFromOffset(X28, SP, reg_offset); 735 reg_offset -= 8; 736 LoadFromOffset(X27, SP, reg_offset); 737 reg_offset -= 8; 738 LoadFromOffset(X26, SP, reg_offset); 739 reg_offset -= 8; 740 LoadFromOffset(X25, SP, reg_offset); 741 reg_offset -= 8; 742 LoadFromOffset(X24, SP, reg_offset); 743 reg_offset -= 8; 744 LoadFromOffset(X23, SP, reg_offset); 745 reg_offset -= 8; 746 LoadFromOffset(X22, SP, reg_offset); 747 reg_offset -= 8; 748 LoadFromOffset(X21, SP, reg_offset); 749 reg_offset -= 8; 750 LoadFromOffset(X20, SP, reg_offset); 751 752 // Decrease frame size to start of callee saved regs. 753 DecreaseFrameSize(frame_size); 754 755 // Pop callee saved and return to LR. 756 ___ Ret(); 757 } 758 759 } // namespace arm64 760 } // namespace art 761
