1 //===- subzero/src/IceTargetLoweringX8664.cpp - x86-64 lowering -----------===// 2 // 3 // The Subzero Code Generator 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// 10 /// \file 11 /// \brief Implements the TargetLoweringX8664 class, which consists almost 12 /// entirely of the lowering sequence for each high-level instruction. 13 /// 14 //===----------------------------------------------------------------------===// 15 #include "IceTargetLoweringX8664.h" 16 17 #include "IceDefs.h" 18 #include "IceTargetLoweringX8664Traits.h" 19 20 namespace X8664 { 21 std::unique_ptr<::Ice::TargetLowering> createTargetLowering(::Ice::Cfg *Func) { 22 return ::Ice::X8664::TargetX8664::create(Func); 23 } 24 25 std::unique_ptr<::Ice::TargetDataLowering> 26 createTargetDataLowering(::Ice::GlobalContext *Ctx) { 27 return ::Ice::X8664::TargetDataX86<::Ice::X8664::TargetX8664Traits>::create( 28 Ctx); 29 } 30 31 std::unique_ptr<::Ice::TargetHeaderLowering> 32 createTargetHeaderLowering(::Ice::GlobalContext *Ctx) { 33 return ::Ice::X8664::TargetHeaderX86::create(Ctx); 34 } 35 36 void staticInit(::Ice::GlobalContext *Ctx) { 37 ::Ice::X8664::TargetX8664::staticInit(Ctx); 38 } 39 40 bool shouldBePooled(const class ::Ice::Constant *C) { 41 return ::Ice::X8664::TargetX8664::shouldBePooled(C); 42 } 43 44 ::Ice::Type getPointerType() { 45 return ::Ice::X8664::TargetX8664::getPointerType(); 46 } 47 48 } // end of namespace X8664 49 50 namespace Ice { 51 namespace X8664 { 52 53 //------------------------------------------------------------------------------ 54 // ______ ______ ______ __ ______ ______ 55 // /\__ _\ /\ == \ /\ __ \ /\ \ /\__ _\ /\ ___\ 56 // \/_/\ \/ \ \ __< \ \ __ \ \ \ \ \/_/\ \/ \ \___ \ 57 // \ \_\ \ \_\ \_\ \ \_\ \_\ \ \_\ \ \_\ \/\_____\ 58 // \/_/ \/_/ /_/ \/_/\/_/ \/_/ \/_/ \/_____/ 59 // 60 //------------------------------------------------------------------------------ 61 const TargetX8664Traits::TableFcmpType TargetX8664Traits::TableFcmp[] = { 62 #define X(val, dflt, swapS, C1, C2, swapV, pred) \ 63 { \ 64 dflt, swapS, X8664::Traits::Cond::C1, X8664::Traits::Cond::C2, swapV, \ 65 X8664::Traits::Cond::pred \ 66 } \ 67 , 68 FCMPX8664_TABLE 69 #undef X 70 }; 71 72 const size_t TargetX8664Traits::TableFcmpSize = llvm::array_lengthof(TableFcmp); 73 74 const TargetX8664Traits::TableIcmp32Type TargetX8664Traits::TableIcmp32[] = { 75 #define X(val, C_32, C1_64, C2_64, C3_64) \ 76 { X8664::Traits::Cond::C_32 } \ 77 , 78 ICMPX8664_TABLE 79 #undef X 80 }; 81 82 const size_t TargetX8664Traits::TableIcmp32Size = 83 llvm::array_lengthof(TableIcmp32); 84 85 const TargetX8664Traits::TableIcmp64Type TargetX8664Traits::TableIcmp64[] = { 86 #define X(val, C_32, C1_64, C2_64, C3_64) \ 87 { \ 88 X8664::Traits::Cond::C1_64, X8664::Traits::Cond::C2_64, \ 89 X8664::Traits::Cond::C3_64 \ 90 } \ 91 , 92 ICMPX8664_TABLE 93 #undef X 94 }; 95 96 const size_t TargetX8664Traits::TableIcmp64Size = 97 llvm::array_lengthof(TableIcmp64); 98 99 const TargetX8664Traits::TableTypeX8664AttributesType 100 TargetX8664Traits::TableTypeX8664Attributes[] = { 101 #define X(tag, elty, cvt, sdss, pdps, spsd, int_, unpack, pack, width, fld) \ 102 { IceType_##elty } \ 103 , 104 ICETYPEX8664_TABLE 105 #undef X 106 }; 107 108 const size_t TargetX8664Traits::TableTypeX8664AttributesSize = 109 llvm::array_lengthof(TableTypeX8664Attributes); 110 111 const uint32_t TargetX8664Traits::X86_STACK_ALIGNMENT_BYTES = 16; 112 const char *TargetX8664Traits::TargetName = "X8664"; 113 114 template <> 115 std::array<SmallBitVector, RCX86_NUM> 116 TargetX86Base<X8664::Traits>::TypeToRegisterSet = {{}}; 117 118 template <> 119 std::array<SmallBitVector, RCX86_NUM> 120 TargetX86Base<X8664::Traits>::TypeToRegisterSetUnfiltered = {{}}; 121 122 template <> 123 std::array<SmallBitVector, 124 TargetX86Base<X8664::Traits>::Traits::RegisterSet::Reg_NUM> 125 TargetX86Base<X8664::Traits>::RegisterAliases = {{}}; 126 127 template <> 128 FixupKind TargetX86Base<X8664::Traits>::PcRelFixup = 129 TargetX86Base<X8664::Traits>::Traits::FK_PcRel; 130 131 template <> 132 FixupKind TargetX86Base<X8664::Traits>::AbsFixup = 133 TargetX86Base<X8664::Traits>::Traits::FK_Abs; 134 135 //------------------------------------------------------------------------------ 136 // __ ______ __ __ ______ ______ __ __ __ ______ 137 // /\ \ /\ __ \/\ \ _ \ \/\ ___\/\ == \/\ \/\ "-.\ \/\ ___\ 138 // \ \ \___\ \ \/\ \ \ \/ ".\ \ \ __\\ \ __<\ \ \ \ \-. \ \ \__ \ 139 // \ \_____\ \_____\ \__/".~\_\ \_____\ \_\ \_\ \_\ \_\\"\_\ \_____\ 140 // \/_____/\/_____/\/_/ \/_/\/_____/\/_/ /_/\/_/\/_/ \/_/\/_____/ 141 // 142 //------------------------------------------------------------------------------ 143 void TargetX8664::_add_sp(Operand *Adjustment) { 144 Variable *rsp = 145 getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64); 146 if (!NeedSandboxing) { 147 _add(rsp, Adjustment); 148 return; 149 } 150 151 Variable *esp = 152 getPhysicalRegister(Traits::RegisterSet::Reg_esp, IceType_i32); 153 Variable *r15 = 154 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 155 156 // When incrementing rsp, NaCl sandboxing requires the following sequence 157 // 158 // .bundle_start 159 // add Adjustment, %esp 160 // add %r15, %rsp 161 // .bundle_end 162 // 163 // In Subzero, even though rsp and esp alias each other, defining one does not 164 // define the other. Therefore, we must emit 165 // 166 // .bundle_start 167 // %esp = fake-def %rsp 168 // add Adjustment, %esp 169 // %rsp = fake-def %esp 170 // add %r15, %rsp 171 // .bundle_end 172 // 173 // The fake-defs ensure that the 174 // 175 // add Adjustment, %esp 176 // 177 // instruction is not DCE'd. 178 AutoBundle _(this); 179 _redefined(Context.insert<InstFakeDef>(esp, rsp)); 180 _add(esp, Adjustment); 181 _redefined(Context.insert<InstFakeDef>(rsp, esp)); 182 _add(rsp, r15); 183 } 184 185 void TargetX8664::_mov_sp(Operand *NewValue) { 186 assert(NewValue->getType() == IceType_i32); 187 188 Variable *esp = getPhysicalRegister(Traits::RegisterSet::Reg_esp); 189 Variable *rsp = 190 getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64); 191 192 AutoBundle _(this); 193 194 _redefined(Context.insert<InstFakeDef>(esp, rsp)); 195 _redefined(_mov(esp, NewValue)); 196 _redefined(Context.insert<InstFakeDef>(rsp, esp)); 197 198 if (!NeedSandboxing) { 199 return; 200 } 201 202 Variable *r15 = 203 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 204 _add(rsp, r15); 205 } 206 207 void TargetX8664::_push_rbp() { 208 assert(NeedSandboxing); 209 210 Constant *_0 = Ctx->getConstantZero(IceType_i32); 211 Variable *ebp = 212 getPhysicalRegister(Traits::RegisterSet::Reg_ebp, IceType_i32); 213 Variable *rsp = 214 getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64); 215 auto *TopOfStack = llvm::cast<X86OperandMem>( 216 legalize(X86OperandMem::create(Func, IceType_i32, rsp, _0), 217 Legal_Reg | Legal_Mem)); 218 219 // Emits a sequence: 220 // 221 // .bundle_start 222 // push 0 223 // mov %ebp, %(rsp) 224 // .bundle_end 225 // 226 // to avoid leaking the upper 32-bits (i.e., the sandbox address.) 227 AutoBundle _(this); 228 _push(_0); 229 Context.insert<typename Traits::Insts::Store>(ebp, TopOfStack); 230 } 231 232 void TargetX8664::_link_bp() { 233 Variable *esp = 234 getPhysicalRegister(Traits::RegisterSet::Reg_esp, IceType_i32); 235 Variable *rsp = 236 getPhysicalRegister(Traits::RegisterSet::Reg_rsp, Traits::WordType); 237 Variable *ebp = 238 getPhysicalRegister(Traits::RegisterSet::Reg_ebp, IceType_i32); 239 Variable *rbp = 240 getPhysicalRegister(Traits::RegisterSet::Reg_rbp, Traits::WordType); 241 Variable *r15 = 242 getPhysicalRegister(Traits::RegisterSet::Reg_r15, Traits::WordType); 243 244 if (!NeedSandboxing) { 245 _push(rbp); 246 _mov(rbp, rsp); 247 } else { 248 _push_rbp(); 249 250 AutoBundle _(this); 251 _redefined(Context.insert<InstFakeDef>(ebp, rbp)); 252 _redefined(Context.insert<InstFakeDef>(esp, rsp)); 253 _mov(ebp, esp); 254 _redefined(Context.insert<InstFakeDef>(rsp, esp)); 255 _add(rbp, r15); 256 } 257 // Keep ebp live for late-stage liveness analysis (e.g. asm-verbose mode). 258 Context.insert<InstFakeUse>(rbp); 259 } 260 261 void TargetX8664::_unlink_bp() { 262 Variable *rsp = 263 getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64); 264 Variable *rbp = 265 getPhysicalRegister(Traits::RegisterSet::Reg_rbp, IceType_i64); 266 Variable *ebp = 267 getPhysicalRegister(Traits::RegisterSet::Reg_ebp, IceType_i32); 268 // For late-stage liveness analysis (e.g. asm-verbose mode), adding a fake 269 // use of rsp before the assignment of rsp=rbp keeps previous rsp 270 // adjustments from being dead-code eliminated. 271 Context.insert<InstFakeUse>(rsp); 272 if (!NeedSandboxing) { 273 _mov(rsp, rbp); 274 _pop(rbp); 275 } else { 276 _mov_sp(ebp); 277 278 Variable *r15 = 279 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 280 Variable *rcx = 281 getPhysicalRegister(Traits::RegisterSet::Reg_rcx, IceType_i64); 282 Variable *ecx = 283 getPhysicalRegister(Traits::RegisterSet::Reg_ecx, IceType_i32); 284 285 _pop(rcx); 286 Context.insert<InstFakeDef>(ecx, rcx); 287 AutoBundle _(this); 288 _mov(ebp, ecx); 289 290 _redefined(Context.insert<InstFakeDef>(rbp, ebp)); 291 _add(rbp, r15); 292 } 293 } 294 295 void TargetX8664::_push_reg(Variable *Reg) { 296 Variable *rbp = 297 getPhysicalRegister(Traits::RegisterSet::Reg_rbp, Traits::WordType); 298 if (Reg != rbp || !NeedSandboxing) { 299 _push(Reg); 300 } else { 301 _push_rbp(); 302 } 303 } 304 305 void TargetX8664::emitGetIP(CfgNode *Node) { 306 // No IP base register is needed on X86-64. 307 (void)Node; 308 } 309 310 namespace { 311 bool isAssignedToRspOrRbp(const Variable *Var) { 312 if (Var == nullptr) { 313 return false; 314 } 315 316 if (Var->isRematerializable()) { 317 return true; 318 } 319 320 if (!Var->hasReg()) { 321 return false; 322 } 323 324 const auto RegNum = Var->getRegNum(); 325 if ((RegNum == Traits::RegisterSet::Reg_rsp) || 326 (RegNum == Traits::RegisterSet::Reg_rbp)) { 327 return true; 328 } 329 330 return false; 331 } 332 } // end of anonymous namespace 333 334 Traits::X86OperandMem *TargetX8664::_sandbox_mem_reference(X86OperandMem *Mem) { 335 if (SandboxingType == ST_None) { 336 return Mem; 337 } 338 339 if (SandboxingType == ST_Nonsfi) { 340 llvm::report_fatal_error( 341 "_sandbox_mem_reference not implemented for nonsfi"); 342 } 343 344 // In x86_64-nacl, all memory references are relative to a base register 345 // (%r15, %rsp, %rbp, or %rip). 346 347 Variable *Base = Mem->getBase(); 348 Variable *Index = Mem->getIndex(); 349 uint16_t Shift = 0; 350 Variable *ZeroReg = RebasePtr; 351 Constant *Offset = Mem->getOffset(); 352 Variable *T = nullptr; 353 354 bool AbsoluteAddress = false; 355 if (Base == nullptr && Index == nullptr) { 356 if (llvm::isa<ConstantRelocatable>(Offset)) { 357 // Mem is RIP-relative. There's no need to rebase it. 358 return Mem; 359 } 360 // Offset is an absolute address, so we need to emit 361 // Offset(%r15) 362 AbsoluteAddress = true; 363 } 364 365 if (Mem->getIsRebased()) { 366 // If Mem.IsRebased, then we don't need to update Mem, as it's already been 367 // updated to contain a reference to one of %rsp, %rbp, or %r15. 368 // We don't return early because we still need to zero extend Index. 369 assert(ZeroReg == Base || AbsoluteAddress || isAssignedToRspOrRbp(Base)); 370 if (!AbsoluteAddress) { 371 // If Mem is an absolute address, no need to update ZeroReg (which is 372 // already set to %r15.) 373 ZeroReg = Base; 374 } 375 if (Index != nullptr) { 376 T = makeReg(IceType_i32); 377 _mov(T, Index); 378 Shift = Mem->getShift(); 379 } 380 } else { 381 if (Base != nullptr) { 382 // If Base is a valid base pointer we don't need to use the RebasePtr. By 383 // doing this we might save us the need to zero extend the memory operand. 384 if (isAssignedToRspOrRbp(Base)) { 385 ZeroReg = Base; 386 } else { 387 T = Base; 388 } 389 } 390 391 if (Index != nullptr) { 392 assert(!Index->isRematerializable()); 393 // If Index is not nullptr, it is mandatory that T is a nullptr. 394 // Otherwise, the lowering generated a memory operand with two registers. 395 // Note that Base might still be non-nullptr, but it must be a valid 396 // base register. 397 if (T != nullptr) { 398 llvm::report_fatal_error("memory reference contains base and index."); 399 } 400 // If the Index is not shifted, and it is a Valid Base, and the ZeroReg is 401 // still RebasePtr, then we do ZeroReg = Index, and hopefully prevent the 402 // need to zero-extend the memory operand (which may still happen -- see 403 // NeedLea below.) 404 if (Shift == 0 && isAssignedToRspOrRbp(Index) && ZeroReg == RebasePtr) { 405 ZeroReg = Index; 406 } else { 407 T = Index; 408 Shift = Mem->getShift(); 409 } 410 } 411 } 412 413 // NeedsLea is a flag indicating whether Mem needs to be materialized to a GPR 414 // prior to being used. A LEA is needed if Mem.Offset is a constant 415 // relocatable with a nonzero offset, or if Mem.Offset is a nonzero immediate; 416 // but only when the address mode contains a "user" register other than the 417 // rsp/rbp/r15 base. In both these cases, the LEA is needed to ensure the 418 // sandboxed memory operand will only use the lower 32-bits of T+Offset. 419 bool NeedsLea = false; 420 if (!Mem->getIsRebased()) { 421 bool IsOffsetZero = false; 422 if (Offset == nullptr) { 423 IsOffsetZero = true; 424 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Offset)) { 425 IsOffsetZero = (CR->getOffset() == 0); 426 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Offset)) { 427 IsOffsetZero = (Imm->getValue() == 0); 428 } else { 429 llvm::report_fatal_error("Unexpected Offset type."); 430 } 431 if (!IsOffsetZero) { 432 if (Base != nullptr && Base != ZeroReg) 433 NeedsLea = true; 434 if (Index != nullptr && Index != ZeroReg) 435 NeedsLea = true; 436 } 437 } 438 439 RegNumT RegNum, RegNum32; 440 if (T != nullptr) { 441 if (T->hasReg()) { 442 RegNum = Traits::getGprForType(IceType_i64, T->getRegNum()); 443 RegNum32 = Traits::getGprForType(IceType_i32, RegNum); 444 // At this point, if T was assigned to rsp/rbp, then we would have already 445 // made this the ZeroReg. 446 assert(RegNum != Traits::RegisterSet::Reg_rsp); 447 assert(RegNum != Traits::RegisterSet::Reg_rbp); 448 } 449 450 switch (T->getType()) { 451 default: 452 llvm::report_fatal_error("Mem pointer should be a 32-bit GPR."); 453 case IceType_i64: 454 // Even though "default:" would also catch T.Type == IceType_i64, an 455 // explicit 'case IceType_i64' shows that memory operands are always 456 // supposed to be 32-bits. 457 llvm::report_fatal_error("Mem pointer should not be a 64-bit GPR."); 458 case IceType_i32: { 459 Variable *T64 = makeReg(IceType_i64, RegNum); 460 auto *Movzx = _movzx(T64, T); 461 if (!NeedsLea) { 462 // This movzx is only needed when Mem does not need to be lea'd into a 463 // temporary. If an lea is going to be emitted, then eliding this movzx 464 // is safe because the emitted lea will write a 32-bit result -- 465 // implicitly zero-extended to 64-bit. 466 Movzx->setMustKeep(); 467 } 468 T = T64; 469 } break; 470 } 471 } 472 473 if (NeedsLea) { 474 Variable *NewT = makeReg(IceType_i32, RegNum32); 475 Variable *Base = T; 476 Variable *Index = T; 477 static constexpr bool NotRebased = false; 478 if (Shift == 0) { 479 Index = nullptr; 480 } else { 481 Base = nullptr; 482 } 483 _lea(NewT, Traits::X86OperandMem::create( 484 Func, Mem->getType(), Base, Offset, Index, Shift, 485 Traits::X86OperandMem::DefaultSegment, NotRebased)); 486 487 T = makeReg(IceType_i64, RegNum); 488 _movzx(T, NewT); 489 Shift = 0; 490 Offset = nullptr; 491 } 492 493 static constexpr bool IsRebased = true; 494 return Traits::X86OperandMem::create( 495 Func, Mem->getType(), ZeroReg, Offset, T, Shift, 496 Traits::X86OperandMem::DefaultSegment, IsRebased); 497 } 498 499 void TargetX8664::_sub_sp(Operand *Adjustment) { 500 Variable *rsp = 501 getPhysicalRegister(Traits::RegisterSet::Reg_rsp, Traits::WordType); 502 503 if (NeedSandboxing) { 504 Variable *esp = 505 getPhysicalRegister(Traits::RegisterSet::Reg_esp, IceType_i32); 506 Variable *r15 = 507 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 508 509 // .bundle_start 510 // sub Adjustment, %esp 511 // add %r15, %rsp 512 // .bundle_end 513 AutoBundle _(this); 514 _redefined(Context.insert<InstFakeDef>(esp, rsp)); 515 _sub(esp, Adjustment); 516 _redefined(Context.insert<InstFakeDef>(rsp, esp)); 517 _add(rsp, r15); 518 } else { 519 _sub(rsp, Adjustment); 520 } 521 522 // Add a fake use of the stack pointer, to prevent the stack pointer adustment 523 // from being dead-code eliminated in a function that doesn't return. 524 Context.insert<InstFakeUse>(rsp); 525 } 526 527 void TargetX8664::initRebasePtr() { 528 switch (SandboxingType) { 529 case ST_Nonsfi: 530 // Probably no implementation is needed, but error to be safe for now. 531 llvm::report_fatal_error( 532 "initRebasePtr() is not yet implemented on x32-nonsfi."); 533 case ST_NaCl: 534 RebasePtr = getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 535 break; 536 case ST_None: 537 // nothing. 538 break; 539 } 540 } 541 542 void TargetX8664::initSandbox() { 543 assert(SandboxingType == ST_NaCl); 544 Context.init(Func->getEntryNode()); 545 Context.setInsertPoint(Context.getCur()); 546 Variable *r15 = 547 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 548 Context.insert<InstFakeDef>(r15); 549 Context.insert<InstFakeUse>(r15); 550 } 551 552 namespace { 553 bool isRematerializable(const Variable *Var) { 554 return Var != nullptr && Var->isRematerializable(); 555 } 556 } // end of anonymous namespace 557 558 bool TargetX8664::legalizeOptAddrForSandbox(OptAddr *Addr) { 559 if (SandboxingType == ST_Nonsfi) { 560 llvm::report_fatal_error("Nonsfi not yet implemented for x8664."); 561 } 562 563 if (isRematerializable(Addr->Base)) { 564 if (Addr->Index == RebasePtr) { 565 Addr->Index = nullptr; 566 Addr->Shift = 0; 567 } 568 return true; 569 } 570 571 if (isRematerializable(Addr->Index)) { 572 if (Addr->Base == RebasePtr) { 573 Addr->Base = nullptr; 574 } 575 return true; 576 } 577 578 assert(Addr->Base != RebasePtr && Addr->Index != RebasePtr); 579 580 if (Addr->Base == nullptr) { 581 return true; 582 } 583 584 if (Addr->Index == nullptr) { 585 return true; 586 } 587 588 return false; 589 } 590 591 void TargetX8664::lowerIndirectJump(Variable *JumpTarget) { 592 std::unique_ptr<AutoBundle> Bundler; 593 594 if (!NeedSandboxing) { 595 if (JumpTarget->getType() != IceType_i64) { 596 Variable *T = makeReg(IceType_i64); 597 _movzx(T, JumpTarget); 598 JumpTarget = T; 599 } 600 } else { 601 Variable *T = makeReg(IceType_i32); 602 Variable *T64 = makeReg(IceType_i64); 603 Variable *r15 = 604 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 605 606 _mov(T, JumpTarget); 607 Bundler = makeUnique<AutoBundle>(this); 608 const SizeT BundleSize = 609 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); 610 _and(T, Ctx->getConstantInt32(~(BundleSize - 1))); 611 _movzx(T64, T); 612 _add(T64, r15); 613 JumpTarget = T64; 614 } 615 616 _jmp(JumpTarget); 617 } 618 619 Inst *TargetX8664::emitCallToTarget(Operand *CallTarget, Variable *ReturnReg) { 620 Inst *NewCall = nullptr; 621 auto *CallTargetR = llvm::dyn_cast<Variable>(CallTarget); 622 if (NeedSandboxing) { 623 // In NaCl sandbox, calls are replaced by a push/jmp pair: 624 // 625 // push .after_call 626 // jmp CallTarget 627 // .align bundle_size 628 // after_call: 629 // 630 // In order to emit this sequence, we need a temporary label ("after_call", 631 // in this example.) 632 // 633 // The operand to push is a ConstantRelocatable. The easy way to implement 634 // this sequence is to create a ConstantRelocatable(0, "after_call"), but 635 // this ends up creating more relocations for the linker to resolve. 636 // Therefore, we create a ConstantRelocatable from the name of the function 637 // being compiled (i.e., ConstantRelocatable(after_call - Func, Func). 638 // 639 // By default, ConstantRelocatables are emitted (in textual output) as 640 // 641 // ConstantName + Offset 642 // 643 // ReturnReloc has an offset that is only known during binary emission. 644 // Therefore, we set a custom emit string for ReturnReloc that will be 645 // used instead. In this particular case, the code will be emitted as 646 // 647 // push .after_call 648 InstX86Label *ReturnAddress = InstX86Label::create(Func, this); 649 auto *ReturnRelocOffset = RelocOffset::create(Func->getAssembler()); 650 ReturnAddress->setRelocOffset(ReturnRelocOffset); 651 constexpr RelocOffsetT NoFixedOffset = 0; 652 const std::string EmitString = 653 BuildDefs::dump() ? ReturnAddress->getLabelName().toString() : ""; 654 auto *ReturnReloc = ConstantRelocatable::create( 655 Func->getAssembler(), IceType_i32, 656 RelocatableTuple(NoFixedOffset, {ReturnRelocOffset}, 657 Func->getFunctionName(), EmitString)); 658 /* AutoBundle scoping */ { 659 std::unique_ptr<AutoBundle> Bundler; 660 if (CallTargetR == nullptr) { 661 Bundler = makeUnique<AutoBundle>(this, InstBundleLock::Opt_PadToEnd); 662 _push(ReturnReloc); 663 } else { 664 Variable *T = makeReg(IceType_i32); 665 Variable *T64 = makeReg(IceType_i64); 666 Variable *r15 = 667 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 668 669 _mov(T, CallTargetR); 670 Bundler = makeUnique<AutoBundle>(this, InstBundleLock::Opt_PadToEnd); 671 _push(ReturnReloc); 672 const SizeT BundleSize = 673 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); 674 _and(T, Ctx->getConstantInt32(~(BundleSize - 1))); 675 _movzx(T64, T); 676 _add(T64, r15); 677 CallTarget = T64; 678 } 679 680 NewCall = Context.insert<Traits::Insts::Jmp>(CallTarget); 681 } 682 if (ReturnReg != nullptr) { 683 Context.insert<InstFakeDef>(ReturnReg); 684 } 685 686 Context.insert(ReturnAddress); 687 } else { 688 if (CallTargetR != nullptr && CallTarget->getType() == IceType_i32) { 689 // x86-64 in PNaCl is ILP32. Therefore, CallTarget is i32, but the 690 // emitted call needs an i64 register (for textual asm.) 691 Variable *T = makeReg(IceType_i64); 692 _movzx(T, CallTargetR); 693 CallTarget = T; 694 } else if (llvm::isa<Constant>(CallTarget) && 695 CallTarget->getType() == IceType_i64) { 696 // x86-64 does not support 64-bit direct calls, so write the value 697 // to a register and make an indirect call. 698 Variable *T = makeReg(IceType_i64); 699 _mov(T, CallTarget); 700 CallTarget = T; 701 } 702 NewCall = Context.insert<Traits::Insts::Call>(ReturnReg, CallTarget); 703 } 704 return NewCall; 705 } 706 707 Variable *TargetX8664::moveReturnValueToRegister(Operand *Value, 708 Type ReturnType) { 709 if (isVectorType(ReturnType) || isScalarFloatingType(ReturnType)) { 710 return legalizeToReg(Value, Traits::RegisterSet::Reg_xmm0); 711 } else { 712 assert(ReturnType == IceType_i32 || ReturnType == IceType_i64); 713 Variable *Reg = nullptr; 714 _mov(Reg, Value, 715 Traits::getGprForType(ReturnType, Traits::RegisterSet::Reg_rax)); 716 return Reg; 717 } 718 } 719 720 void TargetX8664::emitSandboxedReturn() { 721 Variable *T_rcx = makeReg(IceType_i64, Traits::RegisterSet::Reg_rcx); 722 Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx); 723 _pop(T_rcx); 724 _mov(T_ecx, T_rcx); 725 // lowerIndirectJump(T_ecx); 726 Variable *r15 = 727 getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64); 728 729 /* AutoBundle scoping */ { 730 AutoBundle _(this); 731 const SizeT BundleSize = 732 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); 733 _and(T_ecx, Ctx->getConstantInt32(~(BundleSize - 1))); 734 Context.insert<InstFakeDef>(T_rcx, T_ecx); 735 _add(T_rcx, r15); 736 737 _jmp(T_rcx); 738 } 739 } 740 741 // In some cases, there are x-macros tables for both high-level and low-level 742 // instructions/operands that use the same enum key value. The tables are kept 743 // separate to maintain a proper separation between abstraction layers. There 744 // is a risk that the tables could get out of sync if enum values are reordered 745 // or if entries are added or deleted. The following dummy namespaces use 746 // static_asserts to ensure everything is kept in sync. 747 748 namespace { 749 // Validate the enum values in FCMPX8664_TABLE. 750 namespace dummy1 { 751 // Define a temporary set of enum values based on low-level table entries. 752 enum _tmp_enum { 753 #define X(val, dflt, swapS, C1, C2, swapV, pred) _tmp_##val, 754 FCMPX8664_TABLE 755 #undef X 756 _num 757 }; 758 // Define a set of constants based on high-level table entries. 759 #define X(tag, str) static const int _table1_##tag = InstFcmp::tag; 760 ICEINSTFCMP_TABLE 761 #undef X 762 // Define a set of constants based on low-level table entries, and ensure the 763 // table entry keys are consistent. 764 #define X(val, dflt, swapS, C1, C2, swapV, pred) \ 765 static const int _table2_##val = _tmp_##val; \ 766 static_assert( \ 767 _table1_##val == _table2_##val, \ 768 "Inconsistency between FCMPX8664_TABLE and ICEINSTFCMP_TABLE"); 769 FCMPX8664_TABLE 770 #undef X 771 // Repeat the static asserts with respect to the high-level table entries in 772 // case the high-level table has extra entries. 773 #define X(tag, str) \ 774 static_assert( \ 775 _table1_##tag == _table2_##tag, \ 776 "Inconsistency between FCMPX8664_TABLE and ICEINSTFCMP_TABLE"); 777 ICEINSTFCMP_TABLE 778 #undef X 779 } // end of namespace dummy1 780 781 // Validate the enum values in ICMPX8664_TABLE. 782 namespace dummy2 { 783 // Define a temporary set of enum values based on low-level table entries. 784 enum _tmp_enum { 785 #define X(val, C_32, C1_64, C2_64, C3_64) _tmp_##val, 786 ICMPX8664_TABLE 787 #undef X 788 _num 789 }; 790 // Define a set of constants based on high-level table entries. 791 #define X(tag, reverse, str) static const int _table1_##tag = InstIcmp::tag; 792 ICEINSTICMP_TABLE 793 #undef X 794 // Define a set of constants based on low-level table entries, and ensure the 795 // table entry keys are consistent. 796 #define X(val, C_32, C1_64, C2_64, C3_64) \ 797 static const int _table2_##val = _tmp_##val; \ 798 static_assert( \ 799 _table1_##val == _table2_##val, \ 800 "Inconsistency between ICMPX8664_TABLE and ICEINSTICMP_TABLE"); 801 ICMPX8664_TABLE 802 #undef X 803 // Repeat the static asserts with respect to the high-level table entries in 804 // case the high-level table has extra entries. 805 #define X(tag, reverse, str) \ 806 static_assert( \ 807 _table1_##tag == _table2_##tag, \ 808 "Inconsistency between ICMPX8664_TABLE and ICEINSTICMP_TABLE"); 809 ICEINSTICMP_TABLE 810 #undef X 811 } // end of namespace dummy2 812 813 // Validate the enum values in ICETYPEX8664_TABLE. 814 namespace dummy3 { 815 // Define a temporary set of enum values based on low-level table entries. 816 enum _tmp_enum { 817 #define X(tag, elty, cvt, sdss, pdps, spsd, int_, unpack, pack, width, fld) \ 818 _tmp_##tag, 819 ICETYPEX8664_TABLE 820 #undef X 821 _num 822 }; 823 // Define a set of constants based on high-level table entries. 824 #define X(tag, sizeLog2, align, elts, elty, str, rcstr) \ 825 static const int _table1_##tag = IceType_##tag; 826 ICETYPE_TABLE 827 #undef X 828 // Define a set of constants based on low-level table entries, and ensure the 829 // table entry keys are consistent. 830 #define X(tag, elty, cvt, sdss, pdps, spsd, int_, unpack, pack, width, fld) \ 831 static const int _table2_##tag = _tmp_##tag; \ 832 static_assert(_table1_##tag == _table2_##tag, \ 833 "Inconsistency between ICETYPEX8664_TABLE and ICETYPE_TABLE"); 834 ICETYPEX8664_TABLE 835 #undef X 836 // Repeat the static asserts with respect to the high-level table entries in 837 // case the high-level table has extra entries. 838 #define X(tag, sizeLog2, align, elts, elty, str, rcstr) \ 839 static_assert(_table1_##tag == _table2_##tag, \ 840 "Inconsistency between ICETYPEX8664_TABLE and ICETYPE_TABLE"); 841 ICETYPE_TABLE 842 #undef X 843 } // end of namespace dummy3 844 } // end of anonymous namespace 845 846 } // end of namespace X8664 847 } // end of namespace Ice 848