1 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the X86 implementation of TargetFrameLowering class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "X86FrameLowering.h" 15 #include "X86InstrBuilder.h" 16 #include "X86InstrInfo.h" 17 #include "X86MachineFunctionInfo.h" 18 #include "X86Subtarget.h" 19 #include "X86TargetMachine.h" 20 #include "llvm/ADT/SmallSet.h" 21 #include "llvm/Analysis/EHPersonalities.h" 22 #include "llvm/CodeGen/MachineFrameInfo.h" 23 #include "llvm/CodeGen/MachineFunction.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/MachineModuleInfo.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/WinEHFuncInfo.h" 28 #include "llvm/IR/DataLayout.h" 29 #include "llvm/IR/Function.h" 30 #include "llvm/MC/MCAsmInfo.h" 31 #include "llvm/MC/MCSymbol.h" 32 #include "llvm/Target/TargetOptions.h" 33 #include "llvm/Support/Debug.h" 34 #include <cstdlib> 35 36 using namespace llvm; 37 38 X86FrameLowering::X86FrameLowering(const X86Subtarget &STI, 39 unsigned StackAlignOverride) 40 : TargetFrameLowering(StackGrowsDown, StackAlignOverride, 41 STI.is64Bit() ? -8 : -4), 42 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) { 43 // Cache a bunch of frame-related predicates for this subtarget. 44 SlotSize = TRI->getSlotSize(); 45 Is64Bit = STI.is64Bit(); 46 IsLP64 = STI.isTarget64BitLP64(); 47 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit. 48 Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64(); 49 StackPtr = TRI->getStackRegister(); 50 } 51 52 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { 53 return !MF.getFrameInfo()->hasVarSizedObjects() && 54 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences(); 55 } 56 57 /// canSimplifyCallFramePseudos - If there is a reserved call frame, the 58 /// call frame pseudos can be simplified. Having a FP, as in the default 59 /// implementation, is not sufficient here since we can't always use it. 60 /// Use a more nuanced condition. 61 bool 62 X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const { 63 return hasReservedCallFrame(MF) || 64 (hasFP(MF) && !TRI->needsStackRealignment(MF)) || 65 TRI->hasBasePointer(MF); 66 } 67 68 // needsFrameIndexResolution - Do we need to perform FI resolution for 69 // this function. Normally, this is required only when the function 70 // has any stack objects. However, FI resolution actually has another job, 71 // not apparent from the title - it resolves callframesetup/destroy 72 // that were not simplified earlier. 73 // So, this is required for x86 functions that have push sequences even 74 // when there are no stack objects. 75 bool 76 X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const { 77 return MF.getFrameInfo()->hasStackObjects() || 78 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences(); 79 } 80 81 /// hasFP - Return true if the specified function should have a dedicated frame 82 /// pointer register. This is true if the function has variable sized allocas 83 /// or if frame pointer elimination is disabled. 84 bool X86FrameLowering::hasFP(const MachineFunction &MF) const { 85 const MachineFrameInfo *MFI = MF.getFrameInfo(); 86 const MachineModuleInfo &MMI = MF.getMMI(); 87 88 return (MF.getTarget().Options.DisableFramePointerElim(MF) || 89 TRI->needsStackRealignment(MF) || 90 MFI->hasVarSizedObjects() || 91 MFI->isFrameAddressTaken() || MFI->hasOpaqueSPAdjustment() || 92 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() || 93 MMI.callsUnwindInit() || MMI.hasEHFunclets() || MMI.callsEHReturn() || 94 MFI->hasStackMap() || MFI->hasPatchPoint() || 95 MFI->hasCopyImplyingStackAdjustment()); 96 } 97 98 static unsigned getSUBriOpcode(unsigned IsLP64, int64_t Imm) { 99 if (IsLP64) { 100 if (isInt<8>(Imm)) 101 return X86::SUB64ri8; 102 return X86::SUB64ri32; 103 } else { 104 if (isInt<8>(Imm)) 105 return X86::SUB32ri8; 106 return X86::SUB32ri; 107 } 108 } 109 110 static unsigned getADDriOpcode(unsigned IsLP64, int64_t Imm) { 111 if (IsLP64) { 112 if (isInt<8>(Imm)) 113 return X86::ADD64ri8; 114 return X86::ADD64ri32; 115 } else { 116 if (isInt<8>(Imm)) 117 return X86::ADD32ri8; 118 return X86::ADD32ri; 119 } 120 } 121 122 static unsigned getSUBrrOpcode(unsigned isLP64) { 123 return isLP64 ? X86::SUB64rr : X86::SUB32rr; 124 } 125 126 static unsigned getADDrrOpcode(unsigned isLP64) { 127 return isLP64 ? X86::ADD64rr : X86::ADD32rr; 128 } 129 130 static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) { 131 if (IsLP64) { 132 if (isInt<8>(Imm)) 133 return X86::AND64ri8; 134 return X86::AND64ri32; 135 } 136 if (isInt<8>(Imm)) 137 return X86::AND32ri8; 138 return X86::AND32ri; 139 } 140 141 static unsigned getLEArOpcode(unsigned IsLP64) { 142 return IsLP64 ? X86::LEA64r : X86::LEA32r; 143 } 144 145 /// findDeadCallerSavedReg - Return a caller-saved register that isn't live 146 /// when it reaches the "return" instruction. We can then pop a stack object 147 /// to this register without worry about clobbering it. 148 static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB, 149 MachineBasicBlock::iterator &MBBI, 150 const X86RegisterInfo *TRI, 151 bool Is64Bit) { 152 const MachineFunction *MF = MBB.getParent(); 153 const Function *F = MF->getFunction(); 154 if (!F || MF->getMMI().callsEHReturn()) 155 return 0; 156 157 const TargetRegisterClass &AvailableRegs = *TRI->getGPRsForTailCall(*MF); 158 159 unsigned Opc = MBBI->getOpcode(); 160 switch (Opc) { 161 default: return 0; 162 case TargetOpcode::PATCHABLE_RET: 163 case X86::RET: 164 case X86::RETL: 165 case X86::RETQ: 166 case X86::RETIL: 167 case X86::RETIQ: 168 case X86::TCRETURNdi: 169 case X86::TCRETURNri: 170 case X86::TCRETURNmi: 171 case X86::TCRETURNdi64: 172 case X86::TCRETURNri64: 173 case X86::TCRETURNmi64: 174 case X86::EH_RETURN: 175 case X86::EH_RETURN64: { 176 SmallSet<uint16_t, 8> Uses; 177 for (unsigned i = 0, e = MBBI->getNumOperands(); i != e; ++i) { 178 MachineOperand &MO = MBBI->getOperand(i); 179 if (!MO.isReg() || MO.isDef()) 180 continue; 181 unsigned Reg = MO.getReg(); 182 if (!Reg) 183 continue; 184 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) 185 Uses.insert(*AI); 186 } 187 188 for (auto CS : AvailableRegs) 189 if (!Uses.count(CS) && CS != X86::RIP) 190 return CS; 191 } 192 } 193 194 return 0; 195 } 196 197 static bool isEAXLiveIn(MachineBasicBlock &MBB) { 198 for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) { 199 unsigned Reg = RegMask.PhysReg; 200 201 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX || 202 Reg == X86::AH || Reg == X86::AL) 203 return true; 204 } 205 206 return false; 207 } 208 209 /// Check if the flags need to be preserved before the terminators. 210 /// This would be the case, if the eflags is live-in of the region 211 /// composed by the terminators or live-out of that region, without 212 /// being defined by a terminator. 213 static bool 214 flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) { 215 for (const MachineInstr &MI : MBB.terminators()) { 216 bool BreakNext = false; 217 for (const MachineOperand &MO : MI.operands()) { 218 if (!MO.isReg()) 219 continue; 220 unsigned Reg = MO.getReg(); 221 if (Reg != X86::EFLAGS) 222 continue; 223 224 // This terminator needs an eflags that is not defined 225 // by a previous another terminator: 226 // EFLAGS is live-in of the region composed by the terminators. 227 if (!MO.isDef()) 228 return true; 229 // This terminator defines the eflags, i.e., we don't need to preserve it. 230 // However, we still need to check this specific terminator does not 231 // read a live-in value. 232 BreakNext = true; 233 } 234 // We found a definition of the eflags, no need to preserve them. 235 if (BreakNext) 236 return false; 237 } 238 239 // None of the terminators use or define the eflags. 240 // Check if they are live-out, that would imply we need to preserve them. 241 for (const MachineBasicBlock *Succ : MBB.successors()) 242 if (Succ->isLiveIn(X86::EFLAGS)) 243 return true; 244 245 return false; 246 } 247 248 /// emitSPUpdate - Emit a series of instructions to increment / decrement the 249 /// stack pointer by a constant value. 250 void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB, 251 MachineBasicBlock::iterator &MBBI, 252 int64_t NumBytes, bool InEpilogue) const { 253 bool isSub = NumBytes < 0; 254 uint64_t Offset = isSub ? -NumBytes : NumBytes; 255 256 uint64_t Chunk = (1LL << 31) - 1; 257 DebugLoc DL = MBB.findDebugLoc(MBBI); 258 259 while (Offset) { 260 if (Offset > Chunk) { 261 // Rather than emit a long series of instructions for large offsets, 262 // load the offset into a register and do one sub/add 263 unsigned Reg = 0; 264 265 if (isSub && !isEAXLiveIn(MBB)) 266 Reg = (unsigned)(Is64Bit ? X86::RAX : X86::EAX); 267 else 268 Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit); 269 270 if (Reg) { 271 unsigned Opc = Is64Bit ? X86::MOV64ri : X86::MOV32ri; 272 BuildMI(MBB, MBBI, DL, TII.get(Opc), Reg) 273 .addImm(Offset); 274 Opc = isSub 275 ? getSUBrrOpcode(Is64Bit) 276 : getADDrrOpcode(Is64Bit); 277 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) 278 .addReg(StackPtr) 279 .addReg(Reg); 280 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. 281 Offset = 0; 282 continue; 283 } 284 } 285 286 uint64_t ThisVal = std::min(Offset, Chunk); 287 if (ThisVal == (Is64Bit ? 8 : 4)) { 288 // Use push / pop instead. 289 unsigned Reg = isSub 290 ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX) 291 : findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit); 292 if (Reg) { 293 unsigned Opc = isSub 294 ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r) 295 : (Is64Bit ? X86::POP64r : X86::POP32r); 296 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc)) 297 .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub)); 298 if (isSub) 299 MI->setFlag(MachineInstr::FrameSetup); 300 else 301 MI->setFlag(MachineInstr::FrameDestroy); 302 Offset -= ThisVal; 303 continue; 304 } 305 } 306 307 MachineInstrBuilder MI = BuildStackAdjustment( 308 MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue); 309 if (isSub) 310 MI.setMIFlag(MachineInstr::FrameSetup); 311 else 312 MI.setMIFlag(MachineInstr::FrameDestroy); 313 314 Offset -= ThisVal; 315 } 316 } 317 318 MachineInstrBuilder X86FrameLowering::BuildStackAdjustment( 319 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 320 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const { 321 assert(Offset != 0 && "zero offset stack adjustment requested"); 322 323 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue 324 // is tricky. 325 bool UseLEA; 326 if (!InEpilogue) { 327 // Check if inserting the prologue at the beginning 328 // of MBB would require to use LEA operations. 329 // We need to use LEA operations if EFLAGS is live in, because 330 // it means an instruction will read it before it gets defined. 331 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS); 332 } else { 333 // If we can use LEA for SP but we shouldn't, check that none 334 // of the terminators uses the eflags. Otherwise we will insert 335 // a ADD that will redefine the eflags and break the condition. 336 // Alternatively, we could move the ADD, but this may not be possible 337 // and is an optimization anyway. 338 UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent()); 339 if (UseLEA && !STI.useLeaForSP()) 340 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB); 341 // If that assert breaks, that means we do not do the right thing 342 // in canUseAsEpilogue. 343 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) && 344 "We shouldn't have allowed this insertion point"); 345 } 346 347 MachineInstrBuilder MI; 348 if (UseLEA) { 349 MI = addRegOffset(BuildMI(MBB, MBBI, DL, 350 TII.get(getLEArOpcode(Uses64BitFramePtr)), 351 StackPtr), 352 StackPtr, false, Offset); 353 } else { 354 bool IsSub = Offset < 0; 355 uint64_t AbsOffset = IsSub ? -Offset : Offset; 356 unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset) 357 : getADDriOpcode(Uses64BitFramePtr, AbsOffset); 358 MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) 359 .addReg(StackPtr) 360 .addImm(AbsOffset); 361 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. 362 } 363 return MI; 364 } 365 366 int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB, 367 MachineBasicBlock::iterator &MBBI, 368 bool doMergeWithPrevious) const { 369 if ((doMergeWithPrevious && MBBI == MBB.begin()) || 370 (!doMergeWithPrevious && MBBI == MBB.end())) 371 return 0; 372 373 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI; 374 MachineBasicBlock::iterator NI = doMergeWithPrevious ? nullptr 375 : std::next(MBBI); 376 unsigned Opc = PI->getOpcode(); 377 int Offset = 0; 378 379 if (!doMergeWithPrevious && NI != MBB.end() && 380 NI->getOpcode() == TargetOpcode::CFI_INSTRUCTION) { 381 // Don't merge with the next instruction if it has CFI. 382 return Offset; 383 } 384 385 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 || 386 Opc == X86::ADD32ri || Opc == X86::ADD32ri8) && 387 PI->getOperand(0).getReg() == StackPtr){ 388 assert(PI->getOperand(1).getReg() == StackPtr); 389 Offset += PI->getOperand(2).getImm(); 390 MBB.erase(PI); 391 if (!doMergeWithPrevious) MBBI = NI; 392 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) && 393 PI->getOperand(0).getReg() == StackPtr && 394 PI->getOperand(1).getReg() == StackPtr && 395 PI->getOperand(2).getImm() == 1 && 396 PI->getOperand(3).getReg() == X86::NoRegister && 397 PI->getOperand(5).getReg() == X86::NoRegister) { 398 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg. 399 Offset += PI->getOperand(4).getImm(); 400 MBB.erase(PI); 401 if (!doMergeWithPrevious) MBBI = NI; 402 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 || 403 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) && 404 PI->getOperand(0).getReg() == StackPtr) { 405 assert(PI->getOperand(1).getReg() == StackPtr); 406 Offset -= PI->getOperand(2).getImm(); 407 MBB.erase(PI); 408 if (!doMergeWithPrevious) MBBI = NI; 409 } 410 411 return Offset; 412 } 413 414 void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB, 415 MachineBasicBlock::iterator MBBI, 416 const DebugLoc &DL, 417 const MCCFIInstruction &CFIInst) const { 418 MachineFunction &MF = *MBB.getParent(); 419 unsigned CFIIndex = MF.getMMI().addFrameInst(CFIInst); 420 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION)) 421 .addCFIIndex(CFIIndex); 422 } 423 424 void X86FrameLowering::emitCalleeSavedFrameMoves( 425 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 426 const DebugLoc &DL) const { 427 MachineFunction &MF = *MBB.getParent(); 428 MachineFrameInfo *MFI = MF.getFrameInfo(); 429 MachineModuleInfo &MMI = MF.getMMI(); 430 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); 431 432 // Add callee saved registers to move list. 433 const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo(); 434 if (CSI.empty()) return; 435 436 // Calculate offsets. 437 for (std::vector<CalleeSavedInfo>::const_iterator 438 I = CSI.begin(), E = CSI.end(); I != E; ++I) { 439 int64_t Offset = MFI->getObjectOffset(I->getFrameIdx()); 440 unsigned Reg = I->getReg(); 441 442 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); 443 BuildCFI(MBB, MBBI, DL, 444 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); 445 } 446 } 447 448 MachineInstr *X86FrameLowering::emitStackProbe(MachineFunction &MF, 449 MachineBasicBlock &MBB, 450 MachineBasicBlock::iterator MBBI, 451 const DebugLoc &DL, 452 bool InProlog) const { 453 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 454 if (STI.isTargetWindowsCoreCLR()) { 455 if (InProlog) { 456 return emitStackProbeInlineStub(MF, MBB, MBBI, DL, true); 457 } else { 458 return emitStackProbeInline(MF, MBB, MBBI, DL, false); 459 } 460 } else { 461 return emitStackProbeCall(MF, MBB, MBBI, DL, InProlog); 462 } 463 } 464 465 void X86FrameLowering::inlineStackProbe(MachineFunction &MF, 466 MachineBasicBlock &PrologMBB) const { 467 const StringRef ChkStkStubSymbol = "__chkstk_stub"; 468 MachineInstr *ChkStkStub = nullptr; 469 470 for (MachineInstr &MI : PrologMBB) { 471 if (MI.isCall() && MI.getOperand(0).isSymbol() && 472 ChkStkStubSymbol == MI.getOperand(0).getSymbolName()) { 473 ChkStkStub = &MI; 474 break; 475 } 476 } 477 478 if (ChkStkStub != nullptr) { 479 assert(!ChkStkStub->isBundled() && 480 "Not expecting bundled instructions here"); 481 MachineBasicBlock::iterator MBBI = std::next(ChkStkStub->getIterator()); 482 assert(std::prev(MBBI).operator==(ChkStkStub) && 483 "MBBI expected after __chkstk_stub."); 484 DebugLoc DL = PrologMBB.findDebugLoc(MBBI); 485 emitStackProbeInline(MF, PrologMBB, MBBI, DL, true); 486 ChkStkStub->eraseFromParent(); 487 } 488 } 489 490 MachineInstr *X86FrameLowering::emitStackProbeInline( 491 MachineFunction &MF, MachineBasicBlock &MBB, 492 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const { 493 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 494 assert(STI.is64Bit() && "different expansion needed for 32 bit"); 495 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR"); 496 const TargetInstrInfo &TII = *STI.getInstrInfo(); 497 const BasicBlock *LLVM_BB = MBB.getBasicBlock(); 498 499 // RAX contains the number of bytes of desired stack adjustment. 500 // The handling here assumes this value has already been updated so as to 501 // maintain stack alignment. 502 // 503 // We need to exit with RSP modified by this amount and execute suitable 504 // page touches to notify the OS that we're growing the stack responsibly. 505 // All stack probing must be done without modifying RSP. 506 // 507 // MBB: 508 // SizeReg = RAX; 509 // ZeroReg = 0 510 // CopyReg = RSP 511 // Flags, TestReg = CopyReg - SizeReg 512 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg 513 // LimitReg = gs magic thread env access 514 // if FinalReg >= LimitReg goto ContinueMBB 515 // RoundBB: 516 // RoundReg = page address of FinalReg 517 // LoopMBB: 518 // LoopReg = PHI(LimitReg,ProbeReg) 519 // ProbeReg = LoopReg - PageSize 520 // [ProbeReg] = 0 521 // if (ProbeReg > RoundReg) goto LoopMBB 522 // ContinueMBB: 523 // RSP = RSP - RAX 524 // [rest of original MBB] 525 526 // Set up the new basic blocks 527 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB); 528 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB); 529 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB); 530 531 MachineFunction::iterator MBBIter = std::next(MBB.getIterator()); 532 MF.insert(MBBIter, RoundMBB); 533 MF.insert(MBBIter, LoopMBB); 534 MF.insert(MBBIter, ContinueMBB); 535 536 // Split MBB and move the tail portion down to ContinueMBB. 537 MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI); 538 ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end()); 539 ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB); 540 541 // Some useful constants 542 const int64_t ThreadEnvironmentStackLimit = 0x10; 543 const int64_t PageSize = 0x1000; 544 const int64_t PageMask = ~(PageSize - 1); 545 546 // Registers we need. For the normal case we use virtual 547 // registers. For the prolog expansion we use RAX, RCX and RDX. 548 MachineRegisterInfo &MRI = MF.getRegInfo(); 549 const TargetRegisterClass *RegClass = &X86::GR64RegClass; 550 const unsigned SizeReg = InProlog ? (unsigned)X86::RAX 551 : MRI.createVirtualRegister(RegClass), 552 ZeroReg = InProlog ? (unsigned)X86::RCX 553 : MRI.createVirtualRegister(RegClass), 554 CopyReg = InProlog ? (unsigned)X86::RDX 555 : MRI.createVirtualRegister(RegClass), 556 TestReg = InProlog ? (unsigned)X86::RDX 557 : MRI.createVirtualRegister(RegClass), 558 FinalReg = InProlog ? (unsigned)X86::RDX 559 : MRI.createVirtualRegister(RegClass), 560 RoundedReg = InProlog ? (unsigned)X86::RDX 561 : MRI.createVirtualRegister(RegClass), 562 LimitReg = InProlog ? (unsigned)X86::RCX 563 : MRI.createVirtualRegister(RegClass), 564 JoinReg = InProlog ? (unsigned)X86::RCX 565 : MRI.createVirtualRegister(RegClass), 566 ProbeReg = InProlog ? (unsigned)X86::RCX 567 : MRI.createVirtualRegister(RegClass); 568 569 // SP-relative offsets where we can save RCX and RDX. 570 int64_t RCXShadowSlot = 0; 571 int64_t RDXShadowSlot = 0; 572 573 // If inlining in the prolog, save RCX and RDX. 574 // Future optimization: don't save or restore if not live in. 575 if (InProlog) { 576 // Compute the offsets. We need to account for things already 577 // pushed onto the stack at this point: return address, frame 578 // pointer (if used), and callee saves. 579 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 580 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize(); 581 const bool HasFP = hasFP(MF); 582 RCXShadowSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0); 583 RDXShadowSlot = RCXShadowSlot + 8; 584 // Emit the saves. 585 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false, 586 RCXShadowSlot) 587 .addReg(X86::RCX); 588 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false, 589 RDXShadowSlot) 590 .addReg(X86::RDX); 591 } else { 592 // Not in the prolog. Copy RAX to a virtual reg. 593 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX); 594 } 595 596 // Add code to MBB to check for overflow and set the new target stack pointer 597 // to zero if so. 598 BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg) 599 .addReg(ZeroReg, RegState::Undef) 600 .addReg(ZeroReg, RegState::Undef); 601 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP); 602 BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg) 603 .addReg(CopyReg) 604 .addReg(SizeReg); 605 BuildMI(&MBB, DL, TII.get(X86::CMOVB64rr), FinalReg) 606 .addReg(TestReg) 607 .addReg(ZeroReg); 608 609 // FinalReg now holds final stack pointer value, or zero if 610 // allocation would overflow. Compare against the current stack 611 // limit from the thread environment block. Note this limit is the 612 // lowest touched page on the stack, not the point at which the OS 613 // will cause an overflow exception, so this is just an optimization 614 // to avoid unnecessarily touching pages that are below the current 615 // SP but already commited to the stack by the OS. 616 BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg) 617 .addReg(0) 618 .addImm(1) 619 .addReg(0) 620 .addImm(ThreadEnvironmentStackLimit) 621 .addReg(X86::GS); 622 BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg); 623 // Jump if the desired stack pointer is at or above the stack limit. 624 BuildMI(&MBB, DL, TII.get(X86::JAE_1)).addMBB(ContinueMBB); 625 626 // Add code to roundMBB to round the final stack pointer to a page boundary. 627 BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg) 628 .addReg(FinalReg) 629 .addImm(PageMask); 630 BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB); 631 632 // LimitReg now holds the current stack limit, RoundedReg page-rounded 633 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page 634 // and probe until we reach RoundedReg. 635 if (!InProlog) { 636 BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg) 637 .addReg(LimitReg) 638 .addMBB(RoundMBB) 639 .addReg(ProbeReg) 640 .addMBB(LoopMBB); 641 } 642 643 addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg, 644 false, -PageSize); 645 646 // Probe by storing a byte onto the stack. 647 BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi)) 648 .addReg(ProbeReg) 649 .addImm(1) 650 .addReg(0) 651 .addImm(0) 652 .addReg(0) 653 .addImm(0); 654 BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr)) 655 .addReg(RoundedReg) 656 .addReg(ProbeReg); 657 BuildMI(LoopMBB, DL, TII.get(X86::JNE_1)).addMBB(LoopMBB); 658 659 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI(); 660 661 // If in prolog, restore RDX and RCX. 662 if (InProlog) { 663 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::MOV64rm), 664 X86::RCX), 665 X86::RSP, false, RCXShadowSlot); 666 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::MOV64rm), 667 X86::RDX), 668 X86::RSP, false, RDXShadowSlot); 669 } 670 671 // Now that the probing is done, add code to continueMBB to update 672 // the stack pointer for real. 673 BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP) 674 .addReg(X86::RSP) 675 .addReg(SizeReg); 676 677 // Add the control flow edges we need. 678 MBB.addSuccessor(ContinueMBB); 679 MBB.addSuccessor(RoundMBB); 680 RoundMBB->addSuccessor(LoopMBB); 681 LoopMBB->addSuccessor(ContinueMBB); 682 LoopMBB->addSuccessor(LoopMBB); 683 684 // Mark all the instructions added to the prolog as frame setup. 685 if (InProlog) { 686 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) { 687 BeforeMBBI->setFlag(MachineInstr::FrameSetup); 688 } 689 for (MachineInstr &MI : *RoundMBB) { 690 MI.setFlag(MachineInstr::FrameSetup); 691 } 692 for (MachineInstr &MI : *LoopMBB) { 693 MI.setFlag(MachineInstr::FrameSetup); 694 } 695 for (MachineBasicBlock::iterator CMBBI = ContinueMBB->begin(); 696 CMBBI != ContinueMBBI; ++CMBBI) { 697 CMBBI->setFlag(MachineInstr::FrameSetup); 698 } 699 } 700 701 // Possible TODO: physreg liveness for InProlog case. 702 703 return &*ContinueMBBI; 704 } 705 706 MachineInstr *X86FrameLowering::emitStackProbeCall( 707 MachineFunction &MF, MachineBasicBlock &MBB, 708 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const { 709 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large; 710 711 unsigned CallOp; 712 if (Is64Bit) 713 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32; 714 else 715 CallOp = X86::CALLpcrel32; 716 717 const char *Symbol; 718 if (Is64Bit) { 719 if (STI.isTargetCygMing()) { 720 Symbol = "___chkstk_ms"; 721 } else { 722 Symbol = "__chkstk"; 723 } 724 } else if (STI.isTargetCygMing()) 725 Symbol = "_alloca"; 726 else 727 Symbol = "_chkstk"; 728 729 MachineInstrBuilder CI; 730 MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI); 731 732 // All current stack probes take AX and SP as input, clobber flags, and 733 // preserve all registers. x86_64 probes leave RSP unmodified. 734 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) { 735 // For the large code model, we have to call through a register. Use R11, 736 // as it is scratch in all supported calling conventions. 737 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11) 738 .addExternalSymbol(Symbol); 739 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11); 740 } else { 741 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addExternalSymbol(Symbol); 742 } 743 744 unsigned AX = Is64Bit ? X86::RAX : X86::EAX; 745 unsigned SP = Is64Bit ? X86::RSP : X86::ESP; 746 CI.addReg(AX, RegState::Implicit) 747 .addReg(SP, RegState::Implicit) 748 .addReg(AX, RegState::Define | RegState::Implicit) 749 .addReg(SP, RegState::Define | RegState::Implicit) 750 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 751 752 if (Is64Bit) { 753 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp 754 // themselves. It also does not clobber %rax so we can reuse it when 755 // adjusting %rsp. 756 BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64rr), X86::RSP) 757 .addReg(X86::RSP) 758 .addReg(X86::RAX); 759 } 760 761 if (InProlog) { 762 // Apply the frame setup flag to all inserted instrs. 763 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI) 764 ExpansionMBBI->setFlag(MachineInstr::FrameSetup); 765 } 766 767 return &*MBBI; 768 } 769 770 MachineInstr *X86FrameLowering::emitStackProbeInlineStub( 771 MachineFunction &MF, MachineBasicBlock &MBB, 772 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const { 773 774 assert(InProlog && "ChkStkStub called outside prolog!"); 775 776 BuildMI(MBB, MBBI, DL, TII.get(X86::CALLpcrel32)) 777 .addExternalSymbol("__chkstk_stub"); 778 779 return &*MBBI; 780 } 781 782 static unsigned calculateSetFPREG(uint64_t SPAdjust) { 783 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well 784 // and might require smaller successive adjustments. 785 const uint64_t Win64MaxSEHOffset = 128; 786 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset); 787 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode. 788 return SEHFrameOffset & -16; 789 } 790 791 // If we're forcing a stack realignment we can't rely on just the frame 792 // info, we need to know the ABI stack alignment as well in case we 793 // have a call out. Otherwise just make sure we have some alignment - we'll 794 // go with the minimum SlotSize. 795 uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const { 796 const MachineFrameInfo *MFI = MF.getFrameInfo(); 797 uint64_t MaxAlign = MFI->getMaxAlignment(); // Desired stack alignment. 798 unsigned StackAlign = getStackAlignment(); 799 if (MF.getFunction()->hasFnAttribute("stackrealign")) { 800 if (MFI->hasCalls()) 801 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign; 802 else if (MaxAlign < SlotSize) 803 MaxAlign = SlotSize; 804 } 805 return MaxAlign; 806 } 807 808 void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB, 809 MachineBasicBlock::iterator MBBI, 810 const DebugLoc &DL, unsigned Reg, 811 uint64_t MaxAlign) const { 812 uint64_t Val = -MaxAlign; 813 unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val); 814 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg) 815 .addReg(Reg) 816 .addImm(Val) 817 .setMIFlag(MachineInstr::FrameSetup); 818 819 // The EFLAGS implicit def is dead. 820 MI->getOperand(3).setIsDead(); 821 } 822 823 /// emitPrologue - Push callee-saved registers onto the stack, which 824 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate 825 /// space for local variables. Also emit labels used by the exception handler to 826 /// generate the exception handling frames. 827 828 /* 829 Here's a gist of what gets emitted: 830 831 ; Establish frame pointer, if needed 832 [if needs FP] 833 push %rbp 834 .cfi_def_cfa_offset 16 835 .cfi_offset %rbp, -16 836 .seh_pushreg %rpb 837 mov %rsp, %rbp 838 .cfi_def_cfa_register %rbp 839 840 ; Spill general-purpose registers 841 [for all callee-saved GPRs] 842 pushq %<reg> 843 [if not needs FP] 844 .cfi_def_cfa_offset (offset from RETADDR) 845 .seh_pushreg %<reg> 846 847 ; If the required stack alignment > default stack alignment 848 ; rsp needs to be re-aligned. This creates a "re-alignment gap" 849 ; of unknown size in the stack frame. 850 [if stack needs re-alignment] 851 and $MASK, %rsp 852 853 ; Allocate space for locals 854 [if target is Windows and allocated space > 4096 bytes] 855 ; Windows needs special care for allocations larger 856 ; than one page. 857 mov $NNN, %rax 858 call ___chkstk_ms/___chkstk 859 sub %rax, %rsp 860 [else] 861 sub $NNN, %rsp 862 863 [if needs FP] 864 .seh_stackalloc (size of XMM spill slots) 865 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots 866 [else] 867 .seh_stackalloc NNN 868 869 ; Spill XMMs 870 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved, 871 ; they may get spilled on any platform, if the current function 872 ; calls @llvm.eh.unwind.init 873 [if needs FP] 874 [for all callee-saved XMM registers] 875 movaps %<xmm reg>, -MMM(%rbp) 876 [for all callee-saved XMM registers] 877 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset) 878 ; i.e. the offset relative to (%rbp - SEHFrameOffset) 879 [else] 880 [for all callee-saved XMM registers] 881 movaps %<xmm reg>, KKK(%rsp) 882 [for all callee-saved XMM registers] 883 .seh_savexmm %<xmm reg>, KKK 884 885 .seh_endprologue 886 887 [if needs base pointer] 888 mov %rsp, %rbx 889 [if needs to restore base pointer] 890 mov %rsp, -MMM(%rbp) 891 892 ; Emit CFI info 893 [if needs FP] 894 [for all callee-saved registers] 895 .cfi_offset %<reg>, (offset from %rbp) 896 [else] 897 .cfi_def_cfa_offset (offset from RETADDR) 898 [for all callee-saved registers] 899 .cfi_offset %<reg>, (offset from %rsp) 900 901 Notes: 902 - .seh directives are emitted only for Windows 64 ABI 903 - .cfi directives are emitted for all other ABIs 904 - for 32-bit code, substitute %e?? registers for %r?? 905 */ 906 907 void X86FrameLowering::emitPrologue(MachineFunction &MF, 908 MachineBasicBlock &MBB) const { 909 assert(&STI == &MF.getSubtarget<X86Subtarget>() && 910 "MF used frame lowering for wrong subtarget"); 911 MachineBasicBlock::iterator MBBI = MBB.begin(); 912 MachineFrameInfo *MFI = MF.getFrameInfo(); 913 const Function *Fn = MF.getFunction(); 914 MachineModuleInfo &MMI = MF.getMMI(); 915 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 916 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment. 917 uint64_t StackSize = MFI->getStackSize(); // Number of bytes to allocate. 918 bool IsFunclet = MBB.isEHFuncletEntry(); 919 EHPersonality Personality = EHPersonality::Unknown; 920 if (Fn->hasPersonalityFn()) 921 Personality = classifyEHPersonality(Fn->getPersonalityFn()); 922 bool FnHasClrFunclet = 923 MMI.hasEHFunclets() && Personality == EHPersonality::CoreCLR; 924 bool IsClrFunclet = IsFunclet && FnHasClrFunclet; 925 bool HasFP = hasFP(MF); 926 bool IsWin64CC = STI.isCallingConvWin64(Fn->getCallingConv()); 927 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 928 bool NeedsWinCFI = IsWin64Prologue && Fn->needsUnwindTableEntry(); 929 bool NeedsDwarfCFI = 930 !IsWin64Prologue && (MMI.hasDebugInfo() || Fn->needsUnwindTableEntry()); 931 unsigned FramePtr = TRI->getFrameRegister(MF); 932 const unsigned MachineFramePtr = 933 STI.isTarget64BitILP32() 934 ? getX86SubSuperRegister(FramePtr, 64) : FramePtr; 935 unsigned BasePtr = TRI->getBaseRegister(); 936 937 // Debug location must be unknown since the first debug location is used 938 // to determine the end of the prologue. 939 DebugLoc DL; 940 941 // Add RETADDR move area to callee saved frame size. 942 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); 943 if (TailCallReturnAddrDelta && IsWin64Prologue) 944 report_fatal_error("Can't handle guaranteed tail call under win64 yet"); 945 946 if (TailCallReturnAddrDelta < 0) 947 X86FI->setCalleeSavedFrameSize( 948 X86FI->getCalleeSavedFrameSize() - TailCallReturnAddrDelta); 949 950 bool UseStackProbe = (STI.isOSWindows() && !STI.isTargetMachO()); 951 952 // The default stack probe size is 4096 if the function has no stackprobesize 953 // attribute. 954 unsigned StackProbeSize = 4096; 955 if (Fn->hasFnAttribute("stack-probe-size")) 956 Fn->getFnAttribute("stack-probe-size") 957 .getValueAsString() 958 .getAsInteger(0, StackProbeSize); 959 960 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf 961 // function, and use up to 128 bytes of stack space, don't have a frame 962 // pointer, calls, or dynamic alloca then we do not need to adjust the 963 // stack pointer (we fit in the Red Zone). We also check that we don't 964 // push and pop from the stack. 965 if (Is64Bit && !Fn->hasFnAttribute(Attribute::NoRedZone) && 966 !TRI->needsStackRealignment(MF) && 967 !MFI->hasVarSizedObjects() && // No dynamic alloca. 968 !MFI->adjustsStack() && // No calls. 969 !IsWin64CC && // Win64 has no Red Zone 970 !MFI->hasCopyImplyingStackAdjustment() && // Don't push and pop. 971 !MF.shouldSplitStack()) { // Regular stack 972 uint64_t MinSize = X86FI->getCalleeSavedFrameSize(); 973 if (HasFP) MinSize += SlotSize; 974 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0); 975 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0); 976 MFI->setStackSize(StackSize); 977 } 978 979 // Insert stack pointer adjustment for later moving of return addr. Only 980 // applies to tail call optimized functions where the callee argument stack 981 // size is bigger than the callers. 982 if (TailCallReturnAddrDelta < 0) { 983 BuildStackAdjustment(MBB, MBBI, DL, TailCallReturnAddrDelta, 984 /*InEpilogue=*/false) 985 .setMIFlag(MachineInstr::FrameSetup); 986 } 987 988 // Mapping for machine moves: 989 // 990 // DST: VirtualFP AND 991 // SRC: VirtualFP => DW_CFA_def_cfa_offset 992 // ELSE => DW_CFA_def_cfa 993 // 994 // SRC: VirtualFP AND 995 // DST: Register => DW_CFA_def_cfa_register 996 // 997 // ELSE 998 // OFFSET < 0 => DW_CFA_offset_extended_sf 999 // REG < 64 => DW_CFA_offset + Reg 1000 // ELSE => DW_CFA_offset_extended 1001 1002 uint64_t NumBytes = 0; 1003 int stackGrowth = -SlotSize; 1004 1005 // Find the funclet establisher parameter 1006 unsigned Establisher = X86::NoRegister; 1007 if (IsClrFunclet) 1008 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX; 1009 else if (IsFunclet) 1010 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX; 1011 1012 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) { 1013 // Immediately spill establisher into the home slot. 1014 // The runtime cares about this. 1015 // MOV64mr %rdx, 16(%rsp) 1016 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; 1017 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16) 1018 .addReg(Establisher) 1019 .setMIFlag(MachineInstr::FrameSetup); 1020 MBB.addLiveIn(Establisher); 1021 } 1022 1023 if (HasFP) { 1024 // Calculate required stack adjustment. 1025 uint64_t FrameSize = StackSize - SlotSize; 1026 // If required, include space for extra hidden slot for stashing base pointer. 1027 if (X86FI->getRestoreBasePointer()) 1028 FrameSize += SlotSize; 1029 1030 NumBytes = FrameSize - X86FI->getCalleeSavedFrameSize(); 1031 1032 // Callee-saved registers are pushed on stack before the stack is realigned. 1033 if (TRI->needsStackRealignment(MF) && !IsWin64Prologue) 1034 NumBytes = alignTo(NumBytes, MaxAlign); 1035 1036 // Get the offset of the stack slot for the EBP register, which is 1037 // guaranteed to be the last slot by processFunctionBeforeFrameFinalized. 1038 // Update the frame offset adjustment. 1039 if (!IsFunclet) 1040 MFI->setOffsetAdjustment(-NumBytes); 1041 else 1042 assert(MFI->getOffsetAdjustment() == -(int)NumBytes && 1043 "should calculate same local variable offset for funclets"); 1044 1045 // Save EBP/RBP into the appropriate stack slot. 1046 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r)) 1047 .addReg(MachineFramePtr, RegState::Kill) 1048 .setMIFlag(MachineInstr::FrameSetup); 1049 1050 if (NeedsDwarfCFI) { 1051 // Mark the place where EBP/RBP was saved. 1052 // Define the current CFA rule to use the provided offset. 1053 assert(StackSize); 1054 BuildCFI(MBB, MBBI, DL, 1055 MCCFIInstruction::createDefCfaOffset(nullptr, 2 * stackGrowth)); 1056 1057 // Change the rule for the FramePtr to be an "offset" rule. 1058 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); 1059 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset( 1060 nullptr, DwarfFramePtr, 2 * stackGrowth)); 1061 } 1062 1063 if (NeedsWinCFI) { 1064 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)) 1065 .addImm(FramePtr) 1066 .setMIFlag(MachineInstr::FrameSetup); 1067 } 1068 1069 if (!IsWin64Prologue && !IsFunclet) { 1070 // Update EBP with the new base value. 1071 BuildMI(MBB, MBBI, DL, 1072 TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), 1073 FramePtr) 1074 .addReg(StackPtr) 1075 .setMIFlag(MachineInstr::FrameSetup); 1076 1077 if (NeedsDwarfCFI) { 1078 // Mark effective beginning of when frame pointer becomes valid. 1079 // Define the current CFA to use the EBP/RBP register. 1080 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); 1081 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaRegister( 1082 nullptr, DwarfFramePtr)); 1083 } 1084 } 1085 1086 // Mark the FramePtr as live-in in every block. Don't do this again for 1087 // funclet prologues. 1088 if (!IsFunclet) { 1089 for (MachineBasicBlock &EveryMBB : MF) 1090 EveryMBB.addLiveIn(MachineFramePtr); 1091 } 1092 } else { 1093 assert(!IsFunclet && "funclets without FPs not yet implemented"); 1094 NumBytes = StackSize - X86FI->getCalleeSavedFrameSize(); 1095 } 1096 1097 // For EH funclets, only allocate enough space for outgoing calls. Save the 1098 // NumBytes value that we would've used for the parent frame. 1099 unsigned ParentFrameNumBytes = NumBytes; 1100 if (IsFunclet) 1101 NumBytes = getWinEHFuncletFrameSize(MF); 1102 1103 // Skip the callee-saved push instructions. 1104 bool PushedRegs = false; 1105 int StackOffset = 2 * stackGrowth; 1106 1107 while (MBBI != MBB.end() && 1108 MBBI->getFlag(MachineInstr::FrameSetup) && 1109 (MBBI->getOpcode() == X86::PUSH32r || 1110 MBBI->getOpcode() == X86::PUSH64r)) { 1111 PushedRegs = true; 1112 unsigned Reg = MBBI->getOperand(0).getReg(); 1113 ++MBBI; 1114 1115 if (!HasFP && NeedsDwarfCFI) { 1116 // Mark callee-saved push instruction. 1117 // Define the current CFA rule to use the provided offset. 1118 assert(StackSize); 1119 BuildCFI(MBB, MBBI, DL, 1120 MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset)); 1121 StackOffset += stackGrowth; 1122 } 1123 1124 if (NeedsWinCFI) { 1125 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)).addImm(Reg).setMIFlag( 1126 MachineInstr::FrameSetup); 1127 } 1128 } 1129 1130 // Realign stack after we pushed callee-saved registers (so that we'll be 1131 // able to calculate their offsets from the frame pointer). 1132 // Don't do this for Win64, it needs to realign the stack after the prologue. 1133 if (!IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF)) { 1134 assert(HasFP && "There should be a frame pointer if stack is realigned."); 1135 BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign); 1136 } 1137 1138 // If there is an SUB32ri of ESP immediately before this instruction, merge 1139 // the two. This can be the case when tail call elimination is enabled and 1140 // the callee has more arguments then the caller. 1141 NumBytes -= mergeSPUpdates(MBB, MBBI, true); 1142 1143 // Adjust stack pointer: ESP -= numbytes. 1144 1145 // Windows and cygwin/mingw require a prologue helper routine when allocating 1146 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw 1147 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the 1148 // stack and adjust the stack pointer in one go. The 64-bit version of 1149 // __chkstk is only responsible for probing the stack. The 64-bit prologue is 1150 // responsible for adjusting the stack pointer. Touching the stack at 4K 1151 // increments is necessary to ensure that the guard pages used by the OS 1152 // virtual memory manager are allocated in correct sequence. 1153 uint64_t AlignedNumBytes = NumBytes; 1154 if (IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF)) 1155 AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign); 1156 if (AlignedNumBytes >= StackProbeSize && UseStackProbe) { 1157 // Check whether EAX is livein for this block. 1158 bool isEAXAlive = isEAXLiveIn(MBB); 1159 1160 if (isEAXAlive) { 1161 // Sanity check that EAX is not livein for this function. 1162 // It should not be, so throw an assert. 1163 assert(!Is64Bit && "EAX is livein in x64 case!"); 1164 1165 // Save EAX 1166 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r)) 1167 .addReg(X86::EAX, RegState::Kill) 1168 .setMIFlag(MachineInstr::FrameSetup); 1169 } 1170 1171 if (Is64Bit) { 1172 // Handle the 64-bit Windows ABI case where we need to call __chkstk. 1173 // Function prologue is responsible for adjusting the stack pointer. 1174 if (isUInt<32>(NumBytes)) { 1175 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) 1176 .addImm(NumBytes) 1177 .setMIFlag(MachineInstr::FrameSetup); 1178 } else if (isInt<32>(NumBytes)) { 1179 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri32), X86::RAX) 1180 .addImm(NumBytes) 1181 .setMIFlag(MachineInstr::FrameSetup); 1182 } else { 1183 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX) 1184 .addImm(NumBytes) 1185 .setMIFlag(MachineInstr::FrameSetup); 1186 } 1187 } else { 1188 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive. 1189 // We'll also use 4 already allocated bytes for EAX. 1190 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) 1191 .addImm(isEAXAlive ? NumBytes - 4 : NumBytes) 1192 .setMIFlag(MachineInstr::FrameSetup); 1193 } 1194 1195 // Call __chkstk, __chkstk_ms, or __alloca. 1196 emitStackProbe(MF, MBB, MBBI, DL, true); 1197 1198 if (isEAXAlive) { 1199 // Restore EAX 1200 MachineInstr *MI = 1201 addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX), 1202 StackPtr, false, NumBytes - 4); 1203 MI->setFlag(MachineInstr::FrameSetup); 1204 MBB.insert(MBBI, MI); 1205 } 1206 } else if (NumBytes) { 1207 emitSPUpdate(MBB, MBBI, -(int64_t)NumBytes, /*InEpilogue=*/false); 1208 } 1209 1210 if (NeedsWinCFI && NumBytes) 1211 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc)) 1212 .addImm(NumBytes) 1213 .setMIFlag(MachineInstr::FrameSetup); 1214 1215 int SEHFrameOffset = 0; 1216 unsigned SPOrEstablisher; 1217 if (IsFunclet) { 1218 if (IsClrFunclet) { 1219 // The establisher parameter passed to a CLR funclet is actually a pointer 1220 // to the (mostly empty) frame of its nearest enclosing funclet; we have 1221 // to find the root function establisher frame by loading the PSPSym from 1222 // the intermediate frame. 1223 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF); 1224 MachinePointerInfo NoInfo; 1225 MBB.addLiveIn(Establisher); 1226 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher), 1227 Establisher, false, PSPSlotOffset) 1228 .addMemOperand(MF.getMachineMemOperand( 1229 NoInfo, MachineMemOperand::MOLoad, SlotSize, SlotSize)); 1230 ; 1231 // Save the root establisher back into the current funclet's (mostly 1232 // empty) frame, in case a sub-funclet or the GC needs it. 1233 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, 1234 false, PSPSlotOffset) 1235 .addReg(Establisher) 1236 .addMemOperand( 1237 MF.getMachineMemOperand(NoInfo, MachineMemOperand::MOStore | 1238 MachineMemOperand::MOVolatile, 1239 SlotSize, SlotSize)); 1240 } 1241 SPOrEstablisher = Establisher; 1242 } else { 1243 SPOrEstablisher = StackPtr; 1244 } 1245 1246 if (IsWin64Prologue && HasFP) { 1247 // Set RBP to a small fixed offset from RSP. In the funclet case, we base 1248 // this calculation on the incoming establisher, which holds the value of 1249 // RSP from the parent frame at the end of the prologue. 1250 SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes); 1251 if (SEHFrameOffset) 1252 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr), 1253 SPOrEstablisher, false, SEHFrameOffset); 1254 else 1255 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr) 1256 .addReg(SPOrEstablisher); 1257 1258 // If this is not a funclet, emit the CFI describing our frame pointer. 1259 if (NeedsWinCFI && !IsFunclet) { 1260 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame)) 1261 .addImm(FramePtr) 1262 .addImm(SEHFrameOffset) 1263 .setMIFlag(MachineInstr::FrameSetup); 1264 if (isAsynchronousEHPersonality(Personality)) 1265 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset; 1266 } 1267 } else if (IsFunclet && STI.is32Bit()) { 1268 // Reset EBP / ESI to something good for funclets. 1269 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL); 1270 // If we're a catch funclet, we can be returned to via catchret. Save ESP 1271 // into the registration node so that the runtime will restore it for us. 1272 if (!MBB.isCleanupFuncletEntry()) { 1273 assert(Personality == EHPersonality::MSVC_CXX); 1274 unsigned FrameReg; 1275 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex; 1276 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg); 1277 // ESP is the first field, so no extra displacement is needed. 1278 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg, 1279 false, EHRegOffset) 1280 .addReg(X86::ESP); 1281 } 1282 } 1283 1284 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) { 1285 const MachineInstr &FrameInstr = *MBBI; 1286 ++MBBI; 1287 1288 if (NeedsWinCFI) { 1289 int FI; 1290 if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) { 1291 if (X86::FR64RegClass.contains(Reg)) { 1292 unsigned IgnoredFrameReg; 1293 int Offset = getFrameIndexReference(MF, FI, IgnoredFrameReg); 1294 Offset += SEHFrameOffset; 1295 1296 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM)) 1297 .addImm(Reg) 1298 .addImm(Offset) 1299 .setMIFlag(MachineInstr::FrameSetup); 1300 } 1301 } 1302 } 1303 } 1304 1305 if (NeedsWinCFI) 1306 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue)) 1307 .setMIFlag(MachineInstr::FrameSetup); 1308 1309 if (FnHasClrFunclet && !IsFunclet) { 1310 // Save the so-called Initial-SP (i.e. the value of the stack pointer 1311 // immediately after the prolog) into the PSPSlot so that funclets 1312 // and the GC can recover it. 1313 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF); 1314 auto PSPInfo = MachinePointerInfo::getFixedStack( 1315 MF, MF.getWinEHFuncInfo()->PSPSymFrameIdx); 1316 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false, 1317 PSPSlotOffset) 1318 .addReg(StackPtr) 1319 .addMemOperand(MF.getMachineMemOperand( 1320 PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile, 1321 SlotSize, SlotSize)); 1322 } 1323 1324 // Realign stack after we spilled callee-saved registers (so that we'll be 1325 // able to calculate their offsets from the frame pointer). 1326 // Win64 requires aligning the stack after the prologue. 1327 if (IsWin64Prologue && TRI->needsStackRealignment(MF)) { 1328 assert(HasFP && "There should be a frame pointer if stack is realigned."); 1329 BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign); 1330 } 1331 1332 // We already dealt with stack realignment and funclets above. 1333 if (IsFunclet && STI.is32Bit()) 1334 return; 1335 1336 // If we need a base pointer, set it up here. It's whatever the value 1337 // of the stack pointer is at this point. Any variable size objects 1338 // will be allocated after this, so we can still use the base pointer 1339 // to reference locals. 1340 if (TRI->hasBasePointer(MF)) { 1341 // Update the base pointer with the current stack pointer. 1342 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr; 1343 BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr) 1344 .addReg(SPOrEstablisher) 1345 .setMIFlag(MachineInstr::FrameSetup); 1346 if (X86FI->getRestoreBasePointer()) { 1347 // Stash value of base pointer. Saving RSP instead of EBP shortens 1348 // dependence chain. Used by SjLj EH. 1349 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; 1350 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), 1351 FramePtr, true, X86FI->getRestoreBasePointerOffset()) 1352 .addReg(SPOrEstablisher) 1353 .setMIFlag(MachineInstr::FrameSetup); 1354 } 1355 1356 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) { 1357 // Stash the value of the frame pointer relative to the base pointer for 1358 // Win32 EH. This supports Win32 EH, which does the inverse of the above: 1359 // it recovers the frame pointer from the base pointer rather than the 1360 // other way around. 1361 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; 1362 unsigned UsedReg; 1363 int Offset = 1364 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg); 1365 assert(UsedReg == BasePtr); 1366 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset) 1367 .addReg(FramePtr) 1368 .setMIFlag(MachineInstr::FrameSetup); 1369 } 1370 } 1371 1372 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) { 1373 // Mark end of stack pointer adjustment. 1374 if (!HasFP && NumBytes) { 1375 // Define the current CFA rule to use the provided offset. 1376 assert(StackSize); 1377 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset( 1378 nullptr, -StackSize + stackGrowth)); 1379 } 1380 1381 // Emit DWARF info specifying the offsets of the callee-saved registers. 1382 if (PushedRegs) 1383 emitCalleeSavedFrameMoves(MBB, MBBI, DL); 1384 } 1385 1386 // X86 Interrupt handling function cannot assume anything about the direction 1387 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction 1388 // in each prologue of interrupt handler function. 1389 // 1390 // FIXME: Create "cld" instruction only in these cases: 1391 // 1. The interrupt handling function uses any of the "rep" instructions. 1392 // 2. Interrupt handling function calls another function. 1393 // 1394 if (Fn->getCallingConv() == CallingConv::X86_INTR) 1395 BuildMI(MBB, MBBI, DL, TII.get(X86::CLD)) 1396 .setMIFlag(MachineInstr::FrameSetup); 1397 } 1398 1399 bool X86FrameLowering::canUseLEAForSPInEpilogue( 1400 const MachineFunction &MF) const { 1401 // We can't use LEA instructions for adjusting the stack pointer if this is a 1402 // leaf function in the Win64 ABI. Only ADD instructions may be used to 1403 // deallocate the stack. 1404 // This means that we can use LEA for SP in two situations: 1405 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA. 1406 // 2. We *have* a frame pointer which means we are permitted to use LEA. 1407 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF); 1408 } 1409 1410 static bool isFuncletReturnInstr(MachineInstr &MI) { 1411 switch (MI.getOpcode()) { 1412 case X86::CATCHRET: 1413 case X86::CLEANUPRET: 1414 return true; 1415 default: 1416 return false; 1417 } 1418 llvm_unreachable("impossible"); 1419 } 1420 1421 // CLR funclets use a special "Previous Stack Pointer Symbol" slot on the 1422 // stack. It holds a pointer to the bottom of the root function frame. The 1423 // establisher frame pointer passed to a nested funclet may point to the 1424 // (mostly empty) frame of its parent funclet, but it will need to find 1425 // the frame of the root function to access locals. To facilitate this, 1426 // every funclet copies the pointer to the bottom of the root function 1427 // frame into a PSPSym slot in its own (mostly empty) stack frame. Using the 1428 // same offset for the PSPSym in the root function frame that's used in the 1429 // funclets' frames allows each funclet to dynamically accept any ancestor 1430 // frame as its establisher argument (the runtime doesn't guarantee the 1431 // immediate parent for some reason lost to history), and also allows the GC, 1432 // which uses the PSPSym for some bookkeeping, to find it in any funclet's 1433 // frame with only a single offset reported for the entire method. 1434 unsigned 1435 X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const { 1436 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo(); 1437 unsigned SPReg; 1438 int Offset = getFrameIndexReferencePreferSP(MF, Info.PSPSymFrameIdx, SPReg, 1439 /*IgnoreSPUpdates*/ true); 1440 assert(Offset >= 0 && SPReg == TRI->getStackRegister()); 1441 return static_cast<unsigned>(Offset); 1442 } 1443 1444 unsigned 1445 X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const { 1446 // This is the size of the pushed CSRs. 1447 unsigned CSSize = 1448 MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize(); 1449 // This is the amount of stack a funclet needs to allocate. 1450 unsigned UsedSize; 1451 EHPersonality Personality = 1452 classifyEHPersonality(MF.getFunction()->getPersonalityFn()); 1453 if (Personality == EHPersonality::CoreCLR) { 1454 // CLR funclets need to hold enough space to include the PSPSym, at the 1455 // same offset from the stack pointer (immediately after the prolog) as it 1456 // resides at in the main function. 1457 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize; 1458 } else { 1459 // Other funclets just need enough stack for outgoing call arguments. 1460 UsedSize = MF.getFrameInfo()->getMaxCallFrameSize(); 1461 } 1462 // RBP is not included in the callee saved register block. After pushing RBP, 1463 // everything is 16 byte aligned. Everything we allocate before an outgoing 1464 // call must also be 16 byte aligned. 1465 unsigned FrameSizeMinusRBP = alignTo(CSSize + UsedSize, getStackAlignment()); 1466 // Subtract out the size of the callee saved registers. This is how much stack 1467 // each funclet will allocate. 1468 return FrameSizeMinusRBP - CSSize; 1469 } 1470 1471 static bool isTailCallOpcode(unsigned Opc) { 1472 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi || 1473 Opc == X86::TCRETURNmi || 1474 Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 || 1475 Opc == X86::TCRETURNmi64; 1476 } 1477 1478 void X86FrameLowering::emitEpilogue(MachineFunction &MF, 1479 MachineBasicBlock &MBB) const { 1480 const MachineFrameInfo *MFI = MF.getFrameInfo(); 1481 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1482 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(); 1483 unsigned RetOpcode = MBBI->getOpcode(); 1484 DebugLoc DL; 1485 if (MBBI != MBB.end()) 1486 DL = MBBI->getDebugLoc(); 1487 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit. 1488 const bool Is64BitILP32 = STI.isTarget64BitILP32(); 1489 unsigned FramePtr = TRI->getFrameRegister(MF); 1490 unsigned MachineFramePtr = 1491 Is64BitILP32 ? getX86SubSuperRegister(FramePtr, 64) : FramePtr; 1492 1493 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 1494 bool NeedsWinCFI = 1495 IsWin64Prologue && MF.getFunction()->needsUnwindTableEntry(); 1496 bool IsFunclet = isFuncletReturnInstr(*MBBI); 1497 MachineBasicBlock *TargetMBB = nullptr; 1498 1499 // Get the number of bytes to allocate from the FrameInfo. 1500 uint64_t StackSize = MFI->getStackSize(); 1501 uint64_t MaxAlign = calculateMaxStackAlign(MF); 1502 unsigned CSSize = X86FI->getCalleeSavedFrameSize(); 1503 uint64_t NumBytes = 0; 1504 1505 if (MBBI->getOpcode() == X86::CATCHRET) { 1506 // SEH shouldn't use catchret. 1507 assert(!isAsynchronousEHPersonality( 1508 classifyEHPersonality(MF.getFunction()->getPersonalityFn())) && 1509 "SEH should not use CATCHRET"); 1510 1511 NumBytes = getWinEHFuncletFrameSize(MF); 1512 assert(hasFP(MF) && "EH funclets without FP not yet implemented"); 1513 TargetMBB = MBBI->getOperand(0).getMBB(); 1514 1515 // Pop EBP. 1516 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r), 1517 MachineFramePtr) 1518 .setMIFlag(MachineInstr::FrameDestroy); 1519 } else if (MBBI->getOpcode() == X86::CLEANUPRET) { 1520 NumBytes = getWinEHFuncletFrameSize(MF); 1521 assert(hasFP(MF) && "EH funclets without FP not yet implemented"); 1522 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r), 1523 MachineFramePtr) 1524 .setMIFlag(MachineInstr::FrameDestroy); 1525 } else if (hasFP(MF)) { 1526 // Calculate required stack adjustment. 1527 uint64_t FrameSize = StackSize - SlotSize; 1528 NumBytes = FrameSize - CSSize; 1529 1530 // Callee-saved registers were pushed on stack before the stack was 1531 // realigned. 1532 if (TRI->needsStackRealignment(MF) && !IsWin64Prologue) 1533 NumBytes = alignTo(FrameSize, MaxAlign); 1534 1535 // Pop EBP. 1536 BuildMI(MBB, MBBI, DL, 1537 TII.get(Is64Bit ? X86::POP64r : X86::POP32r), MachineFramePtr) 1538 .setMIFlag(MachineInstr::FrameDestroy); 1539 } else { 1540 NumBytes = StackSize - CSSize; 1541 } 1542 uint64_t SEHStackAllocAmt = NumBytes; 1543 1544 // Skip the callee-saved pop instructions. 1545 while (MBBI != MBB.begin()) { 1546 MachineBasicBlock::iterator PI = std::prev(MBBI); 1547 unsigned Opc = PI->getOpcode(); 1548 1549 if ((Opc != X86::POP32r || !PI->getFlag(MachineInstr::FrameDestroy)) && 1550 (Opc != X86::POP64r || !PI->getFlag(MachineInstr::FrameDestroy)) && 1551 Opc != X86::DBG_VALUE && !PI->isTerminator()) 1552 break; 1553 1554 --MBBI; 1555 } 1556 MachineBasicBlock::iterator FirstCSPop = MBBI; 1557 1558 if (TargetMBB) { 1559 // Fill EAX/RAX with the address of the target block. 1560 unsigned ReturnReg = STI.is64Bit() ? X86::RAX : X86::EAX; 1561 if (STI.is64Bit()) { 1562 // LEA64r TargetMBB(%rip), %rax 1563 BuildMI(MBB, FirstCSPop, DL, TII.get(X86::LEA64r), ReturnReg) 1564 .addReg(X86::RIP) 1565 .addImm(0) 1566 .addReg(0) 1567 .addMBB(TargetMBB) 1568 .addReg(0); 1569 } else { 1570 // MOV32ri $TargetMBB, %eax 1571 BuildMI(MBB, FirstCSPop, DL, TII.get(X86::MOV32ri), ReturnReg) 1572 .addMBB(TargetMBB); 1573 } 1574 // Record that we've taken the address of TargetMBB and no longer just 1575 // reference it in a terminator. 1576 TargetMBB->setHasAddressTaken(); 1577 } 1578 1579 if (MBBI != MBB.end()) 1580 DL = MBBI->getDebugLoc(); 1581 1582 // If there is an ADD32ri or SUB32ri of ESP immediately before this 1583 // instruction, merge the two instructions. 1584 if (NumBytes || MFI->hasVarSizedObjects()) 1585 NumBytes += mergeSPUpdates(MBB, MBBI, true); 1586 1587 // If dynamic alloca is used, then reset esp to point to the last callee-saved 1588 // slot before popping them off! Same applies for the case, when stack was 1589 // realigned. Don't do this if this was a funclet epilogue, since the funclets 1590 // will not do realignment or dynamic stack allocation. 1591 if ((TRI->needsStackRealignment(MF) || MFI->hasVarSizedObjects()) && 1592 !IsFunclet) { 1593 if (TRI->needsStackRealignment(MF)) 1594 MBBI = FirstCSPop; 1595 unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt); 1596 uint64_t LEAAmount = 1597 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize; 1598 1599 // There are only two legal forms of epilogue: 1600 // - add SEHAllocationSize, %rsp 1601 // - lea SEHAllocationSize(%FramePtr), %rsp 1602 // 1603 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence. 1604 // However, we may use this sequence if we have a frame pointer because the 1605 // effects of the prologue can safely be undone. 1606 if (LEAAmount != 0) { 1607 unsigned Opc = getLEArOpcode(Uses64BitFramePtr); 1608 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr), 1609 FramePtr, false, LEAAmount); 1610 --MBBI; 1611 } else { 1612 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr); 1613 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) 1614 .addReg(FramePtr); 1615 --MBBI; 1616 } 1617 } else if (NumBytes) { 1618 // Adjust stack pointer back: ESP += numbytes. 1619 emitSPUpdate(MBB, MBBI, NumBytes, /*InEpilogue=*/true); 1620 --MBBI; 1621 } 1622 1623 // Windows unwinder will not invoke function's exception handler if IP is 1624 // either in prologue or in epilogue. This behavior causes a problem when a 1625 // call immediately precedes an epilogue, because the return address points 1626 // into the epilogue. To cope with that, we insert an epilogue marker here, 1627 // then replace it with a 'nop' if it ends up immediately after a CALL in the 1628 // final emitted code. 1629 if (NeedsWinCFI) 1630 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue)); 1631 1632 if (!isTailCallOpcode(RetOpcode)) { 1633 // Add the return addr area delta back since we are not tail calling. 1634 int Offset = -1 * X86FI->getTCReturnAddrDelta(); 1635 assert(Offset >= 0 && "TCDelta should never be positive"); 1636 if (Offset) { 1637 MBBI = MBB.getFirstTerminator(); 1638 1639 // Check for possible merge with preceding ADD instruction. 1640 Offset += mergeSPUpdates(MBB, MBBI, true); 1641 emitSPUpdate(MBB, MBBI, Offset, /*InEpilogue=*/true); 1642 } 1643 } 1644 } 1645 1646 // NOTE: this only has a subset of the full frame index logic. In 1647 // particular, the FI < 0 and AfterFPPop logic is handled in 1648 // X86RegisterInfo::eliminateFrameIndex, but not here. Possibly 1649 // (probably?) it should be moved into here. 1650 int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, 1651 unsigned &FrameReg) const { 1652 const MachineFrameInfo *MFI = MF.getFrameInfo(); 1653 1654 // We can't calculate offset from frame pointer if the stack is realigned, 1655 // so enforce usage of stack/base pointer. The base pointer is used when we 1656 // have dynamic allocas in addition to dynamic realignment. 1657 if (TRI->hasBasePointer(MF)) 1658 FrameReg = TRI->getBaseRegister(); 1659 else if (TRI->needsStackRealignment(MF)) 1660 FrameReg = TRI->getStackRegister(); 1661 else 1662 FrameReg = TRI->getFrameRegister(MF); 1663 1664 // Offset will hold the offset from the stack pointer at function entry to the 1665 // object. 1666 // We need to factor in additional offsets applied during the prologue to the 1667 // frame, base, and stack pointer depending on which is used. 1668 int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea(); 1669 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1670 unsigned CSSize = X86FI->getCalleeSavedFrameSize(); 1671 uint64_t StackSize = MFI->getStackSize(); 1672 bool HasFP = hasFP(MF); 1673 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 1674 int64_t FPDelta = 0; 1675 1676 if (IsWin64Prologue) { 1677 assert(!MFI->hasCalls() || (StackSize % 16) == 8); 1678 1679 // Calculate required stack adjustment. 1680 uint64_t FrameSize = StackSize - SlotSize; 1681 // If required, include space for extra hidden slot for stashing base pointer. 1682 if (X86FI->getRestoreBasePointer()) 1683 FrameSize += SlotSize; 1684 uint64_t NumBytes = FrameSize - CSSize; 1685 1686 uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes); 1687 if (FI && FI == X86FI->getFAIndex()) 1688 return -SEHFrameOffset; 1689 1690 // FPDelta is the offset from the "traditional" FP location of the old base 1691 // pointer followed by return address and the location required by the 1692 // restricted Win64 prologue. 1693 // Add FPDelta to all offsets below that go through the frame pointer. 1694 FPDelta = FrameSize - SEHFrameOffset; 1695 assert((!MFI->hasCalls() || (FPDelta % 16) == 0) && 1696 "FPDelta isn't aligned per the Win64 ABI!"); 1697 } 1698 1699 1700 if (TRI->hasBasePointer(MF)) { 1701 assert(HasFP && "VLAs and dynamic stack realign, but no FP?!"); 1702 if (FI < 0) { 1703 // Skip the saved EBP. 1704 return Offset + SlotSize + FPDelta; 1705 } else { 1706 assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0); 1707 return Offset + StackSize; 1708 } 1709 } else if (TRI->needsStackRealignment(MF)) { 1710 if (FI < 0) { 1711 // Skip the saved EBP. 1712 return Offset + SlotSize + FPDelta; 1713 } else { 1714 assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0); 1715 return Offset + StackSize; 1716 } 1717 // FIXME: Support tail calls 1718 } else { 1719 if (!HasFP) 1720 return Offset + StackSize; 1721 1722 // Skip the saved EBP. 1723 Offset += SlotSize; 1724 1725 // Skip the RETADDR move area 1726 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); 1727 if (TailCallReturnAddrDelta < 0) 1728 Offset -= TailCallReturnAddrDelta; 1729 } 1730 1731 return Offset + FPDelta; 1732 } 1733 1734 int 1735 X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF, 1736 int FI, unsigned &FrameReg, 1737 bool IgnoreSPUpdates) const { 1738 1739 const MachineFrameInfo *MFI = MF.getFrameInfo(); 1740 // Does not include any dynamic realign. 1741 const uint64_t StackSize = MFI->getStackSize(); 1742 // LLVM arranges the stack as follows: 1743 // ... 1744 // ARG2 1745 // ARG1 1746 // RETADDR 1747 // PUSH RBP <-- RBP points here 1748 // PUSH CSRs 1749 // ~~~~~~~ <-- possible stack realignment (non-win64) 1750 // ... 1751 // STACK OBJECTS 1752 // ... <-- RSP after prologue points here 1753 // ~~~~~~~ <-- possible stack realignment (win64) 1754 // 1755 // if (hasVarSizedObjects()): 1756 // ... <-- "base pointer" (ESI/RBX) points here 1757 // DYNAMIC ALLOCAS 1758 // ... <-- RSP points here 1759 // 1760 // Case 1: In the simple case of no stack realignment and no dynamic 1761 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable 1762 // with fixed offsets from RSP. 1763 // 1764 // Case 2: In the case of stack realignment with no dynamic allocas, fixed 1765 // stack objects are addressed with RBP and regular stack objects with RSP. 1766 // 1767 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used 1768 // to address stack arguments for outgoing calls and nothing else. The "base 1769 // pointer" points to local variables, and RBP points to fixed objects. 1770 // 1771 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the 1772 // answer we give is relative to the SP after the prologue, and not the 1773 // SP in the middle of the function. 1774 1775 if (MFI->isFixedObjectIndex(FI) && TRI->needsStackRealignment(MF) && 1776 !STI.isTargetWin64()) 1777 return getFrameIndexReference(MF, FI, FrameReg); 1778 1779 // If !hasReservedCallFrame the function might have SP adjustement in the 1780 // body. So, even though the offset is statically known, it depends on where 1781 // we are in the function. 1782 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); 1783 if (!IgnoreSPUpdates && !TFI->hasReservedCallFrame(MF)) 1784 return getFrameIndexReference(MF, FI, FrameReg); 1785 1786 // We don't handle tail calls, and shouldn't be seeing them either. 1787 assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 && 1788 "we don't handle this case!"); 1789 1790 // Fill in FrameReg output argument. 1791 FrameReg = TRI->getStackRegister(); 1792 1793 // This is how the math works out: 1794 // 1795 // %rsp grows (i.e. gets lower) left to right. Each box below is 1796 // one word (eight bytes). Obj0 is the stack slot we're trying to 1797 // get to. 1798 // 1799 // ---------------------------------- 1800 // | BP | Obj0 | Obj1 | ... | ObjN | 1801 // ---------------------------------- 1802 // ^ ^ ^ ^ 1803 // A B C E 1804 // 1805 // A is the incoming stack pointer. 1806 // (B - A) is the local area offset (-8 for x86-64) [1] 1807 // (C - A) is the Offset returned by MFI->getObjectOffset for Obj0 [2] 1808 // 1809 // |(E - B)| is the StackSize (absolute value, positive). For a 1810 // stack that grown down, this works out to be (B - E). [3] 1811 // 1812 // E is also the value of %rsp after stack has been set up, and we 1813 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now 1814 // (C - E) == (C - A) - (B - A) + (B - E) 1815 // { Using [1], [2] and [3] above } 1816 // == getObjectOffset - LocalAreaOffset + StackSize 1817 // 1818 1819 // Get the Offset from the StackPointer 1820 int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea(); 1821 1822 return Offset + StackSize; 1823 } 1824 1825 bool X86FrameLowering::assignCalleeSavedSpillSlots( 1826 MachineFunction &MF, const TargetRegisterInfo *TRI, 1827 std::vector<CalleeSavedInfo> &CSI) const { 1828 MachineFrameInfo *MFI = MF.getFrameInfo(); 1829 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1830 1831 unsigned CalleeSavedFrameSize = 0; 1832 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta(); 1833 1834 if (hasFP(MF)) { 1835 // emitPrologue always spills frame register the first thing. 1836 SpillSlotOffset -= SlotSize; 1837 MFI->CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); 1838 1839 // Since emitPrologue and emitEpilogue will handle spilling and restoring of 1840 // the frame register, we can delete it from CSI list and not have to worry 1841 // about avoiding it later. 1842 unsigned FPReg = TRI->getFrameRegister(MF); 1843 for (unsigned i = 0; i < CSI.size(); ++i) { 1844 if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) { 1845 CSI.erase(CSI.begin() + i); 1846 break; 1847 } 1848 } 1849 } 1850 1851 // Assign slots for GPRs. It increases frame size. 1852 for (unsigned i = CSI.size(); i != 0; --i) { 1853 unsigned Reg = CSI[i - 1].getReg(); 1854 1855 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) 1856 continue; 1857 1858 SpillSlotOffset -= SlotSize; 1859 CalleeSavedFrameSize += SlotSize; 1860 1861 int SlotIndex = MFI->CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); 1862 CSI[i - 1].setFrameIdx(SlotIndex); 1863 } 1864 1865 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize); 1866 1867 // Assign slots for XMMs. 1868 for (unsigned i = CSI.size(); i != 0; --i) { 1869 unsigned Reg = CSI[i - 1].getReg(); 1870 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) 1871 continue; 1872 1873 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); 1874 // ensure alignment 1875 SpillSlotOffset -= std::abs(SpillSlotOffset) % RC->getAlignment(); 1876 // spill into slot 1877 SpillSlotOffset -= RC->getSize(); 1878 int SlotIndex = 1879 MFI->CreateFixedSpillStackObject(RC->getSize(), SpillSlotOffset); 1880 CSI[i - 1].setFrameIdx(SlotIndex); 1881 MFI->ensureMaxAlignment(RC->getAlignment()); 1882 } 1883 1884 return true; 1885 } 1886 1887 bool X86FrameLowering::spillCalleeSavedRegisters( 1888 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, 1889 const std::vector<CalleeSavedInfo> &CSI, 1890 const TargetRegisterInfo *TRI) const { 1891 DebugLoc DL = MBB.findDebugLoc(MI); 1892 1893 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI 1894 // for us, and there are no XMM CSRs on Win32. 1895 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows()) 1896 return true; 1897 1898 // Push GPRs. It increases frame size. 1899 const MachineFunction &MF = *MBB.getParent(); 1900 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r; 1901 for (unsigned i = CSI.size(); i != 0; --i) { 1902 unsigned Reg = CSI[i - 1].getReg(); 1903 1904 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) 1905 continue; 1906 1907 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1908 bool isLiveIn = MRI.isLiveIn(Reg); 1909 if (!isLiveIn) 1910 MBB.addLiveIn(Reg); 1911 1912 // Decide whether we can add a kill flag to the use. 1913 bool CanKill = !isLiveIn; 1914 // Check if any subregister is live-in 1915 if (CanKill) { 1916 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg) { 1917 if (MRI.isLiveIn(*AReg)) { 1918 CanKill = false; 1919 break; 1920 } 1921 } 1922 } 1923 1924 // Do not set a kill flag on values that are also marked as live-in. This 1925 // happens with the @llvm-returnaddress intrinsic and with arguments 1926 // passed in callee saved registers. 1927 // Omitting the kill flags is conservatively correct even if the live-in 1928 // is not used after all. 1929 BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, getKillRegState(CanKill)) 1930 .setMIFlag(MachineInstr::FrameSetup); 1931 } 1932 1933 // Make XMM regs spilled. X86 does not have ability of push/pop XMM. 1934 // It can be done by spilling XMMs to stack frame. 1935 for (unsigned i = CSI.size(); i != 0; --i) { 1936 unsigned Reg = CSI[i-1].getReg(); 1937 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) 1938 continue; 1939 // Add the callee-saved register as live-in. It's killed at the spill. 1940 MBB.addLiveIn(Reg); 1941 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); 1942 1943 TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i - 1].getFrameIdx(), RC, 1944 TRI); 1945 --MI; 1946 MI->setFlag(MachineInstr::FrameSetup); 1947 ++MI; 1948 } 1949 1950 return true; 1951 } 1952 1953 bool X86FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB, 1954 MachineBasicBlock::iterator MI, 1955 const std::vector<CalleeSavedInfo> &CSI, 1956 const TargetRegisterInfo *TRI) const { 1957 if (CSI.empty()) 1958 return false; 1959 1960 if (isFuncletReturnInstr(*MI) && STI.isOSWindows()) { 1961 // Don't restore CSRs in 32-bit EH funclets. Matches 1962 // spillCalleeSavedRegisters. 1963 if (STI.is32Bit()) 1964 return true; 1965 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form 1966 // funclets. emitEpilogue transforms these to normal jumps. 1967 if (MI->getOpcode() == X86::CATCHRET) { 1968 const Function *Func = MBB.getParent()->getFunction(); 1969 bool IsSEH = isAsynchronousEHPersonality( 1970 classifyEHPersonality(Func->getPersonalityFn())); 1971 if (IsSEH) 1972 return true; 1973 } 1974 } 1975 1976 DebugLoc DL = MBB.findDebugLoc(MI); 1977 1978 // Reload XMMs from stack frame. 1979 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 1980 unsigned Reg = CSI[i].getReg(); 1981 if (X86::GR64RegClass.contains(Reg) || 1982 X86::GR32RegClass.contains(Reg)) 1983 continue; 1984 1985 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); 1986 TII.loadRegFromStackSlot(MBB, MI, Reg, CSI[i].getFrameIdx(), RC, TRI); 1987 } 1988 1989 // POP GPRs. 1990 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r; 1991 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 1992 unsigned Reg = CSI[i].getReg(); 1993 if (!X86::GR64RegClass.contains(Reg) && 1994 !X86::GR32RegClass.contains(Reg)) 1995 continue; 1996 1997 BuildMI(MBB, MI, DL, TII.get(Opc), Reg) 1998 .setMIFlag(MachineInstr::FrameDestroy); 1999 } 2000 return true; 2001 } 2002 2003 void X86FrameLowering::determineCalleeSaves(MachineFunction &MF, 2004 BitVector &SavedRegs, 2005 RegScavenger *RS) const { 2006 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); 2007 2008 MachineFrameInfo *MFI = MF.getFrameInfo(); 2009 2010 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 2011 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); 2012 2013 if (TailCallReturnAddrDelta < 0) { 2014 // create RETURNADDR area 2015 // arg 2016 // arg 2017 // RETADDR 2018 // { ... 2019 // RETADDR area 2020 // ... 2021 // } 2022 // [EBP] 2023 MFI->CreateFixedObject(-TailCallReturnAddrDelta, 2024 TailCallReturnAddrDelta - SlotSize, true); 2025 } 2026 2027 // Spill the BasePtr if it's used. 2028 if (TRI->hasBasePointer(MF)) { 2029 SavedRegs.set(TRI->getBaseRegister()); 2030 2031 // Allocate a spill slot for EBP if we have a base pointer and EH funclets. 2032 if (MF.getMMI().hasEHFunclets()) { 2033 int FI = MFI->CreateSpillStackObject(SlotSize, SlotSize); 2034 X86FI->setHasSEHFramePtrSave(true); 2035 X86FI->setSEHFramePtrSaveIndex(FI); 2036 } 2037 } 2038 } 2039 2040 static bool 2041 HasNestArgument(const MachineFunction *MF) { 2042 const Function *F = MF->getFunction(); 2043 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end(); 2044 I != E; I++) { 2045 if (I->hasNestAttr()) 2046 return true; 2047 } 2048 return false; 2049 } 2050 2051 /// GetScratchRegister - Get a temp register for performing work in the 2052 /// segmented stack and the Erlang/HiPE stack prologue. Depending on platform 2053 /// and the properties of the function either one or two registers will be 2054 /// needed. Set primary to true for the first register, false for the second. 2055 static unsigned 2056 GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary) { 2057 CallingConv::ID CallingConvention = MF.getFunction()->getCallingConv(); 2058 2059 // Erlang stuff. 2060 if (CallingConvention == CallingConv::HiPE) { 2061 if (Is64Bit) 2062 return Primary ? X86::R14 : X86::R13; 2063 else 2064 return Primary ? X86::EBX : X86::EDI; 2065 } 2066 2067 if (Is64Bit) { 2068 if (IsLP64) 2069 return Primary ? X86::R11 : X86::R12; 2070 else 2071 return Primary ? X86::R11D : X86::R12D; 2072 } 2073 2074 bool IsNested = HasNestArgument(&MF); 2075 2076 if (CallingConvention == CallingConv::X86_FastCall || 2077 CallingConvention == CallingConv::Fast) { 2078 if (IsNested) 2079 report_fatal_error("Segmented stacks does not support fastcall with " 2080 "nested function."); 2081 return Primary ? X86::EAX : X86::ECX; 2082 } 2083 if (IsNested) 2084 return Primary ? X86::EDX : X86::EAX; 2085 return Primary ? X86::ECX : X86::EAX; 2086 } 2087 2088 // The stack limit in the TCB is set to this many bytes above the actual stack 2089 // limit. 2090 static const uint64_t kSplitStackAvailable = 256; 2091 2092 void X86FrameLowering::adjustForSegmentedStacks( 2093 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { 2094 MachineFrameInfo *MFI = MF.getFrameInfo(); 2095 uint64_t StackSize; 2096 unsigned TlsReg, TlsOffset; 2097 DebugLoc DL; 2098 2099 // To support shrink-wrapping we would need to insert the new blocks 2100 // at the right place and update the branches to PrologueMBB. 2101 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet"); 2102 2103 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true); 2104 assert(!MF.getRegInfo().isLiveIn(ScratchReg) && 2105 "Scratch register is live-in"); 2106 2107 if (MF.getFunction()->isVarArg()) 2108 report_fatal_error("Segmented stacks do not support vararg functions."); 2109 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() && 2110 !STI.isTargetWin64() && !STI.isTargetFreeBSD() && 2111 !STI.isTargetDragonFly()) 2112 report_fatal_error("Segmented stacks not supported on this platform."); 2113 2114 // Eventually StackSize will be calculated by a link-time pass; which will 2115 // also decide whether checking code needs to be injected into this particular 2116 // prologue. 2117 StackSize = MFI->getStackSize(); 2118 2119 // Do not generate a prologue for functions with a stack of size zero 2120 if (StackSize == 0) 2121 return; 2122 2123 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock(); 2124 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock(); 2125 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 2126 bool IsNested = false; 2127 2128 // We need to know if the function has a nest argument only in 64 bit mode. 2129 if (Is64Bit) 2130 IsNested = HasNestArgument(&MF); 2131 2132 // The MOV R10, RAX needs to be in a different block, since the RET we emit in 2133 // allocMBB needs to be last (terminating) instruction. 2134 2135 for (const auto &LI : PrologueMBB.liveins()) { 2136 allocMBB->addLiveIn(LI); 2137 checkMBB->addLiveIn(LI); 2138 } 2139 2140 if (IsNested) 2141 allocMBB->addLiveIn(IsLP64 ? X86::R10 : X86::R10D); 2142 2143 MF.push_front(allocMBB); 2144 MF.push_front(checkMBB); 2145 2146 // When the frame size is less than 256 we just compare the stack 2147 // boundary directly to the value of the stack pointer, per gcc. 2148 bool CompareStackPointer = StackSize < kSplitStackAvailable; 2149 2150 // Read the limit off the current stacklet off the stack_guard location. 2151 if (Is64Bit) { 2152 if (STI.isTargetLinux()) { 2153 TlsReg = X86::FS; 2154 TlsOffset = IsLP64 ? 0x70 : 0x40; 2155 } else if (STI.isTargetDarwin()) { 2156 TlsReg = X86::GS; 2157 TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90. 2158 } else if (STI.isTargetWin64()) { 2159 TlsReg = X86::GS; 2160 TlsOffset = 0x28; // pvArbitrary, reserved for application use 2161 } else if (STI.isTargetFreeBSD()) { 2162 TlsReg = X86::FS; 2163 TlsOffset = 0x18; 2164 } else if (STI.isTargetDragonFly()) { 2165 TlsReg = X86::FS; 2166 TlsOffset = 0x20; // use tls_tcb.tcb_segstack 2167 } else { 2168 report_fatal_error("Segmented stacks not supported on this platform."); 2169 } 2170 2171 if (CompareStackPointer) 2172 ScratchReg = IsLP64 ? X86::RSP : X86::ESP; 2173 else 2174 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::LEA64r : X86::LEA64_32r), ScratchReg).addReg(X86::RSP) 2175 .addImm(1).addReg(0).addImm(-StackSize).addReg(0); 2176 2177 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::CMP64rm : X86::CMP32rm)).addReg(ScratchReg) 2178 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg); 2179 } else { 2180 if (STI.isTargetLinux()) { 2181 TlsReg = X86::GS; 2182 TlsOffset = 0x30; 2183 } else if (STI.isTargetDarwin()) { 2184 TlsReg = X86::GS; 2185 TlsOffset = 0x48 + 90*4; 2186 } else if (STI.isTargetWin32()) { 2187 TlsReg = X86::FS; 2188 TlsOffset = 0x14; // pvArbitrary, reserved for application use 2189 } else if (STI.isTargetDragonFly()) { 2190 TlsReg = X86::FS; 2191 TlsOffset = 0x10; // use tls_tcb.tcb_segstack 2192 } else if (STI.isTargetFreeBSD()) { 2193 report_fatal_error("Segmented stacks not supported on FreeBSD i386."); 2194 } else { 2195 report_fatal_error("Segmented stacks not supported on this platform."); 2196 } 2197 2198 if (CompareStackPointer) 2199 ScratchReg = X86::ESP; 2200 else 2201 BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP) 2202 .addImm(1).addReg(0).addImm(-StackSize).addReg(0); 2203 2204 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() || 2205 STI.isTargetDragonFly()) { 2206 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg) 2207 .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg); 2208 } else if (STI.isTargetDarwin()) { 2209 2210 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register. 2211 unsigned ScratchReg2; 2212 bool SaveScratch2; 2213 if (CompareStackPointer) { 2214 // The primary scratch register is available for holding the TLS offset. 2215 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, true); 2216 SaveScratch2 = false; 2217 } else { 2218 // Need to use a second register to hold the TLS offset 2219 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, false); 2220 2221 // Unfortunately, with fastcc the second scratch register may hold an 2222 // argument. 2223 SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2); 2224 } 2225 2226 // If Scratch2 is live-in then it needs to be saved. 2227 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) && 2228 "Scratch register is live-in and not saved"); 2229 2230 if (SaveScratch2) 2231 BuildMI(checkMBB, DL, TII.get(X86::PUSH32r)) 2232 .addReg(ScratchReg2, RegState::Kill); 2233 2234 BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2) 2235 .addImm(TlsOffset); 2236 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)) 2237 .addReg(ScratchReg) 2238 .addReg(ScratchReg2).addImm(1).addReg(0) 2239 .addImm(0) 2240 .addReg(TlsReg); 2241 2242 if (SaveScratch2) 2243 BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2); 2244 } 2245 } 2246 2247 // This jump is taken if SP >= (Stacklet Limit + Stack Space required). 2248 // It jumps to normal execution of the function body. 2249 BuildMI(checkMBB, DL, TII.get(X86::JA_1)).addMBB(&PrologueMBB); 2250 2251 // On 32 bit we first push the arguments size and then the frame size. On 64 2252 // bit, we pass the stack frame size in r10 and the argument size in r11. 2253 if (Is64Bit) { 2254 // Functions with nested arguments use R10, so it needs to be saved across 2255 // the call to _morestack 2256 2257 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX; 2258 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D; 2259 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D; 2260 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr; 2261 const unsigned MOVri = IsLP64 ? X86::MOV64ri : X86::MOV32ri; 2262 2263 if (IsNested) 2264 BuildMI(allocMBB, DL, TII.get(MOVrr), RegAX).addReg(Reg10); 2265 2266 BuildMI(allocMBB, DL, TII.get(MOVri), Reg10) 2267 .addImm(StackSize); 2268 BuildMI(allocMBB, DL, TII.get(MOVri), Reg11) 2269 .addImm(X86FI->getArgumentStackSize()); 2270 } else { 2271 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32)) 2272 .addImm(X86FI->getArgumentStackSize()); 2273 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32)) 2274 .addImm(StackSize); 2275 } 2276 2277 // __morestack is in libgcc 2278 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) { 2279 // Under the large code model, we cannot assume that __morestack lives 2280 // within 2^31 bytes of the call site, so we cannot use pc-relative 2281 // addressing. We cannot perform the call via a temporary register, 2282 // as the rax register may be used to store the static chain, and all 2283 // other suitable registers may be either callee-save or used for 2284 // parameter passing. We cannot use the stack at this point either 2285 // because __morestack manipulates the stack directly. 2286 // 2287 // To avoid these issues, perform an indirect call via a read-only memory 2288 // location containing the address. 2289 // 2290 // This solution is not perfect, as it assumes that the .rodata section 2291 // is laid out within 2^31 bytes of each function body, but this seems 2292 // to be sufficient for JIT. 2293 BuildMI(allocMBB, DL, TII.get(X86::CALL64m)) 2294 .addReg(X86::RIP) 2295 .addImm(0) 2296 .addReg(0) 2297 .addExternalSymbol("__morestack_addr") 2298 .addReg(0); 2299 MF.getMMI().setUsesMorestackAddr(true); 2300 } else { 2301 if (Is64Bit) 2302 BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32)) 2303 .addExternalSymbol("__morestack"); 2304 else 2305 BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32)) 2306 .addExternalSymbol("__morestack"); 2307 } 2308 2309 if (IsNested) 2310 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10)); 2311 else 2312 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET)); 2313 2314 allocMBB->addSuccessor(&PrologueMBB); 2315 2316 checkMBB->addSuccessor(allocMBB); 2317 checkMBB->addSuccessor(&PrologueMBB); 2318 2319 #ifdef EXPENSIVE_CHECKS 2320 MF.verify(); 2321 #endif 2322 } 2323 2324 /// Lookup an ERTS parameter in the !hipe.literals named metadata node. 2325 /// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets 2326 /// to fields it needs, through a named metadata node "hipe.literals" containing 2327 /// name-value pairs. 2328 static unsigned getHiPELiteral( 2329 NamedMDNode *HiPELiteralsMD, const StringRef LiteralName) { 2330 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) { 2331 MDNode *Node = HiPELiteralsMD->getOperand(i); 2332 if (Node->getNumOperands() != 2) continue; 2333 MDString *NodeName = dyn_cast<MDString>(Node->getOperand(0)); 2334 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Node->getOperand(1)); 2335 if (!NodeName || !NodeVal) continue; 2336 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(NodeVal->getValue()); 2337 if (ValConst && NodeName->getString() == LiteralName) { 2338 return ValConst->getZExtValue(); 2339 } 2340 } 2341 2342 report_fatal_error("HiPE literal " + LiteralName 2343 + " required but not provided"); 2344 } 2345 2346 /// Erlang programs may need a special prologue to handle the stack size they 2347 /// might need at runtime. That is because Erlang/OTP does not implement a C 2348 /// stack but uses a custom implementation of hybrid stack/heap architecture. 2349 /// (for more information see Eric Stenman's Ph.D. thesis: 2350 /// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf) 2351 /// 2352 /// CheckStack: 2353 /// temp0 = sp - MaxStack 2354 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart 2355 /// OldStart: 2356 /// ... 2357 /// IncStack: 2358 /// call inc_stack # doubles the stack space 2359 /// temp0 = sp - MaxStack 2360 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart 2361 void X86FrameLowering::adjustForHiPEPrologue( 2362 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { 2363 MachineFrameInfo *MFI = MF.getFrameInfo(); 2364 DebugLoc DL; 2365 2366 // To support shrink-wrapping we would need to insert the new blocks 2367 // at the right place and update the branches to PrologueMBB. 2368 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet"); 2369 2370 // HiPE-specific values 2371 NamedMDNode *HiPELiteralsMD = MF.getMMI().getModule() 2372 ->getNamedMetadata("hipe.literals"); 2373 if (!HiPELiteralsMD) 2374 report_fatal_error( 2375 "Can't generate HiPE prologue without runtime parameters"); 2376 const unsigned HipeLeafWords 2377 = getHiPELiteral(HiPELiteralsMD, 2378 Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS"); 2379 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5; 2380 const unsigned Guaranteed = HipeLeafWords * SlotSize; 2381 unsigned CallerStkArity = MF.getFunction()->arg_size() > CCRegisteredArgs ? 2382 MF.getFunction()->arg_size() - CCRegisteredArgs : 0; 2383 unsigned MaxStack = MFI->getStackSize() + CallerStkArity*SlotSize + SlotSize; 2384 2385 assert(STI.isTargetLinux() && 2386 "HiPE prologue is only supported on Linux operating systems."); 2387 2388 // Compute the largest caller's frame that is needed to fit the callees' 2389 // frames. This 'MaxStack' is computed from: 2390 // 2391 // a) the fixed frame size, which is the space needed for all spilled temps, 2392 // b) outgoing on-stack parameter areas, and 2393 // c) the minimum stack space this function needs to make available for the 2394 // functions it calls (a tunable ABI property). 2395 if (MFI->hasCalls()) { 2396 unsigned MoreStackForCalls = 0; 2397 2398 for (auto &MBB : MF) { 2399 for (auto &MI : MBB) { 2400 if (!MI.isCall()) 2401 continue; 2402 2403 // Get callee operand. 2404 const MachineOperand &MO = MI.getOperand(0); 2405 2406 // Only take account of global function calls (no closures etc.). 2407 if (!MO.isGlobal()) 2408 continue; 2409 2410 const Function *F = dyn_cast<Function>(MO.getGlobal()); 2411 if (!F) 2412 continue; 2413 2414 // Do not update 'MaxStack' for primitive and built-in functions 2415 // (encoded with names either starting with "erlang."/"bif_" or not 2416 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an 2417 // "_", such as the BIF "suspend_0") as they are executed on another 2418 // stack. 2419 if (F->getName().find("erlang.") != StringRef::npos || 2420 F->getName().find("bif_") != StringRef::npos || 2421 F->getName().find_first_of("._") == StringRef::npos) 2422 continue; 2423 2424 unsigned CalleeStkArity = 2425 F->arg_size() > CCRegisteredArgs ? F->arg_size()-CCRegisteredArgs : 0; 2426 if (HipeLeafWords - 1 > CalleeStkArity) 2427 MoreStackForCalls = std::max(MoreStackForCalls, 2428 (HipeLeafWords - 1 - CalleeStkArity) * SlotSize); 2429 } 2430 } 2431 MaxStack += MoreStackForCalls; 2432 } 2433 2434 // If the stack frame needed is larger than the guaranteed then runtime checks 2435 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue. 2436 if (MaxStack > Guaranteed) { 2437 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock(); 2438 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock(); 2439 2440 for (const auto &LI : PrologueMBB.liveins()) { 2441 stackCheckMBB->addLiveIn(LI); 2442 incStackMBB->addLiveIn(LI); 2443 } 2444 2445 MF.push_front(incStackMBB); 2446 MF.push_front(stackCheckMBB); 2447 2448 unsigned ScratchReg, SPReg, PReg, SPLimitOffset; 2449 unsigned LEAop, CMPop, CALLop; 2450 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, "P_NSP_LIMIT"); 2451 if (Is64Bit) { 2452 SPReg = X86::RSP; 2453 PReg = X86::RBP; 2454 LEAop = X86::LEA64r; 2455 CMPop = X86::CMP64rm; 2456 CALLop = X86::CALL64pcrel32; 2457 } else { 2458 SPReg = X86::ESP; 2459 PReg = X86::EBP; 2460 LEAop = X86::LEA32r; 2461 CMPop = X86::CMP32rm; 2462 CALLop = X86::CALLpcrel32; 2463 } 2464 2465 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true); 2466 assert(!MF.getRegInfo().isLiveIn(ScratchReg) && 2467 "HiPE prologue scratch register is live-in"); 2468 2469 // Create new MBB for StackCheck: 2470 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg), 2471 SPReg, false, -MaxStack); 2472 // SPLimitOffset is in a fixed heap location (pointed by BP). 2473 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop)) 2474 .addReg(ScratchReg), PReg, false, SPLimitOffset); 2475 BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_1)).addMBB(&PrologueMBB); 2476 2477 // Create new MBB for IncStack: 2478 BuildMI(incStackMBB, DL, TII.get(CALLop)). 2479 addExternalSymbol("inc_stack_0"); 2480 addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg), 2481 SPReg, false, -MaxStack); 2482 addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop)) 2483 .addReg(ScratchReg), PReg, false, SPLimitOffset); 2484 BuildMI(incStackMBB, DL, TII.get(X86::JLE_1)).addMBB(incStackMBB); 2485 2486 stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100}); 2487 stackCheckMBB->addSuccessor(incStackMBB, {1, 100}); 2488 incStackMBB->addSuccessor(&PrologueMBB, {99, 100}); 2489 incStackMBB->addSuccessor(incStackMBB, {1, 100}); 2490 } 2491 #ifdef EXPENSIVE_CHECKS 2492 MF.verify(); 2493 #endif 2494 } 2495 2496 bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB, 2497 MachineBasicBlock::iterator MBBI, 2498 const DebugLoc &DL, 2499 int Offset) const { 2500 2501 if (Offset <= 0) 2502 return false; 2503 2504 if (Offset % SlotSize) 2505 return false; 2506 2507 int NumPops = Offset / SlotSize; 2508 // This is only worth it if we have at most 2 pops. 2509 if (NumPops != 1 && NumPops != 2) 2510 return false; 2511 2512 // Handle only the trivial case where the adjustment directly follows 2513 // a call. This is the most common one, anyway. 2514 if (MBBI == MBB.begin()) 2515 return false; 2516 MachineBasicBlock::iterator Prev = std::prev(MBBI); 2517 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask()) 2518 return false; 2519 2520 unsigned Regs[2]; 2521 unsigned FoundRegs = 0; 2522 2523 auto RegMask = Prev->getOperand(1); 2524 2525 auto &RegClass = 2526 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass; 2527 // Try to find up to NumPops free registers. 2528 for (auto Candidate : RegClass) { 2529 2530 // Poor man's liveness: 2531 // Since we're immediately after a call, any register that is clobbered 2532 // by the call and not defined by it can be considered dead. 2533 if (!RegMask.clobbersPhysReg(Candidate)) 2534 continue; 2535 2536 bool IsDef = false; 2537 for (const MachineOperand &MO : Prev->implicit_operands()) { 2538 if (MO.isReg() && MO.isDef() && 2539 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) { 2540 IsDef = true; 2541 break; 2542 } 2543 } 2544 2545 if (IsDef) 2546 continue; 2547 2548 Regs[FoundRegs++] = Candidate; 2549 if (FoundRegs == (unsigned)NumPops) 2550 break; 2551 } 2552 2553 if (FoundRegs == 0) 2554 return false; 2555 2556 // If we found only one free register, but need two, reuse the same one twice. 2557 while (FoundRegs < (unsigned)NumPops) 2558 Regs[FoundRegs++] = Regs[0]; 2559 2560 for (int i = 0; i < NumPops; ++i) 2561 BuildMI(MBB, MBBI, DL, 2562 TII.get(STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]); 2563 2564 return true; 2565 } 2566 2567 MachineBasicBlock::iterator X86FrameLowering:: 2568 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, 2569 MachineBasicBlock::iterator I) const { 2570 bool reserveCallFrame = hasReservedCallFrame(MF); 2571 unsigned Opcode = I->getOpcode(); 2572 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode(); 2573 DebugLoc DL = I->getDebugLoc(); 2574 uint64_t Amount = !reserveCallFrame ? I->getOperand(0).getImm() : 0; 2575 uint64_t InternalAmt = (isDestroy || Amount) ? I->getOperand(1).getImm() : 0; 2576 I = MBB.erase(I); 2577 2578 if (!reserveCallFrame) { 2579 // If the stack pointer can be changed after prologue, turn the 2580 // adjcallstackup instruction into a 'sub ESP, <amt>' and the 2581 // adjcallstackdown instruction into 'add ESP, <amt>' 2582 2583 // We need to keep the stack aligned properly. To do this, we round the 2584 // amount of space needed for the outgoing arguments up to the next 2585 // alignment boundary. 2586 unsigned StackAlign = getStackAlignment(); 2587 Amount = alignTo(Amount, StackAlign); 2588 2589 MachineModuleInfo &MMI = MF.getMMI(); 2590 const Function *Fn = MF.getFunction(); 2591 bool WindowsCFI = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 2592 bool DwarfCFI = !WindowsCFI && 2593 (MMI.hasDebugInfo() || Fn->needsUnwindTableEntry()); 2594 2595 // If we have any exception handlers in this function, and we adjust 2596 // the SP before calls, we may need to indicate this to the unwinder 2597 // using GNU_ARGS_SIZE. Note that this may be necessary even when 2598 // Amount == 0, because the preceding function may have set a non-0 2599 // GNU_ARGS_SIZE. 2600 // TODO: We don't need to reset this between subsequent functions, 2601 // if it didn't change. 2602 bool HasDwarfEHHandlers = !WindowsCFI && 2603 !MF.getMMI().getLandingPads().empty(); 2604 2605 if (HasDwarfEHHandlers && !isDestroy && 2606 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences()) 2607 BuildCFI(MBB, I, DL, 2608 MCCFIInstruction::createGnuArgsSize(nullptr, Amount)); 2609 2610 if (Amount == 0) 2611 return I; 2612 2613 // Factor out the amount that gets handled inside the sequence 2614 // (Pushes of argument for frame setup, callee pops for frame destroy) 2615 Amount -= InternalAmt; 2616 2617 // TODO: This is needed only if we require precise CFA. 2618 // If this is a callee-pop calling convention, emit a CFA adjust for 2619 // the amount the callee popped. 2620 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF)) 2621 BuildCFI(MBB, I, DL, 2622 MCCFIInstruction::createAdjustCfaOffset(nullptr, -InternalAmt)); 2623 2624 // Add Amount to SP to destroy a frame, or subtract to setup. 2625 int64_t StackAdjustment = isDestroy ? Amount : -Amount; 2626 int64_t CfaAdjustment = -StackAdjustment; 2627 2628 if (StackAdjustment) { 2629 // Merge with any previous or following adjustment instruction. Note: the 2630 // instructions merged with here do not have CFI, so their stack 2631 // adjustments do not feed into CfaAdjustment. 2632 StackAdjustment += mergeSPUpdates(MBB, I, true); 2633 StackAdjustment += mergeSPUpdates(MBB, I, false); 2634 2635 if (StackAdjustment) { 2636 if (!(Fn->optForMinSize() && 2637 adjustStackWithPops(MBB, I, DL, StackAdjustment))) 2638 BuildStackAdjustment(MBB, I, DL, StackAdjustment, 2639 /*InEpilogue=*/false); 2640 } 2641 } 2642 2643 if (DwarfCFI && !hasFP(MF)) { 2644 // If we don't have FP, but need to generate unwind information, 2645 // we need to set the correct CFA offset after the stack adjustment. 2646 // How much we adjust the CFA offset depends on whether we're emitting 2647 // CFI only for EH purposes or for debugging. EH only requires the CFA 2648 // offset to be correct at each call site, while for debugging we want 2649 // it to be more precise. 2650 2651 // TODO: When not using precise CFA, we also need to adjust for the 2652 // InternalAmt here. 2653 if (CfaAdjustment) { 2654 BuildCFI(MBB, I, DL, MCCFIInstruction::createAdjustCfaOffset( 2655 nullptr, CfaAdjustment)); 2656 } 2657 } 2658 2659 return I; 2660 } 2661 2662 if (isDestroy && InternalAmt) { 2663 // If we are performing frame pointer elimination and if the callee pops 2664 // something off the stack pointer, add it back. We do this until we have 2665 // more advanced stack pointer tracking ability. 2666 // We are not tracking the stack pointer adjustment by the callee, so make 2667 // sure we restore the stack pointer immediately after the call, there may 2668 // be spill code inserted between the CALL and ADJCALLSTACKUP instructions. 2669 MachineBasicBlock::iterator CI = I; 2670 MachineBasicBlock::iterator B = MBB.begin(); 2671 while (CI != B && !std::prev(CI)->isCall()) 2672 --CI; 2673 BuildStackAdjustment(MBB, CI, DL, -InternalAmt, /*InEpilogue=*/false); 2674 } 2675 2676 return I; 2677 } 2678 2679 bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const { 2680 assert(MBB.getParent() && "Block is not attached to a function!"); 2681 const MachineFunction &MF = *MBB.getParent(); 2682 return !TRI->needsStackRealignment(MF) || !MBB.isLiveIn(X86::EFLAGS); 2683 } 2684 2685 bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const { 2686 assert(MBB.getParent() && "Block is not attached to a function!"); 2687 2688 // Win64 has strict requirements in terms of epilogue and we are 2689 // not taking a chance at messing with them. 2690 // I.e., unless this block is already an exit block, we can't use 2691 // it as an epilogue. 2692 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock()) 2693 return false; 2694 2695 if (canUseLEAForSPInEpilogue(*MBB.getParent())) 2696 return true; 2697 2698 // If we cannot use LEA to adjust SP, we may need to use ADD, which 2699 // clobbers the EFLAGS. Check that we do not need to preserve it, 2700 // otherwise, conservatively assume this is not 2701 // safe to insert the epilogue here. 2702 return !flagsNeedToBePreservedBeforeTheTerminators(MBB); 2703 } 2704 2705 bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { 2706 // If we may need to emit frameless compact unwind information, give 2707 // up as this is currently broken: PR25614. 2708 return (MF.getFunction()->hasFnAttribute(Attribute::NoUnwind) || hasFP(MF)) && 2709 // The lowering of segmented stack and HiPE only support entry blocks 2710 // as prologue blocks: PR26107. 2711 // This limitation may be lifted if we fix: 2712 // - adjustForSegmentedStacks 2713 // - adjustForHiPEPrologue 2714 MF.getFunction()->getCallingConv() != CallingConv::HiPE && 2715 !MF.shouldSplitStack(); 2716 } 2717 2718 MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers( 2719 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 2720 const DebugLoc &DL, bool RestoreSP) const { 2721 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env"); 2722 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32"); 2723 assert(STI.is32Bit() && !Uses64BitFramePtr && 2724 "restoring EBP/ESI on non-32-bit target"); 2725 2726 MachineFunction &MF = *MBB.getParent(); 2727 unsigned FramePtr = TRI->getFrameRegister(MF); 2728 unsigned BasePtr = TRI->getBaseRegister(); 2729 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo(); 2730 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 2731 MachineFrameInfo *MFI = MF.getFrameInfo(); 2732 2733 // FIXME: Don't set FrameSetup flag in catchret case. 2734 2735 int FI = FuncInfo.EHRegNodeFrameIndex; 2736 int EHRegSize = MFI->getObjectSize(FI); 2737 2738 if (RestoreSP) { 2739 // MOV32rm -EHRegSize(%ebp), %esp 2740 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), X86::ESP), 2741 X86::EBP, true, -EHRegSize) 2742 .setMIFlag(MachineInstr::FrameSetup); 2743 } 2744 2745 unsigned UsedReg; 2746 int EHRegOffset = getFrameIndexReference(MF, FI, UsedReg); 2747 int EndOffset = -EHRegOffset - EHRegSize; 2748 FuncInfo.EHRegNodeEndOffset = EndOffset; 2749 2750 if (UsedReg == FramePtr) { 2751 // ADD $offset, %ebp 2752 unsigned ADDri = getADDriOpcode(false, EndOffset); 2753 BuildMI(MBB, MBBI, DL, TII.get(ADDri), FramePtr) 2754 .addReg(FramePtr) 2755 .addImm(EndOffset) 2756 .setMIFlag(MachineInstr::FrameSetup) 2757 ->getOperand(3) 2758 .setIsDead(); 2759 assert(EndOffset >= 0 && 2760 "end of registration object above normal EBP position!"); 2761 } else if (UsedReg == BasePtr) { 2762 // LEA offset(%ebp), %esi 2763 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA32r), BasePtr), 2764 FramePtr, false, EndOffset) 2765 .setMIFlag(MachineInstr::FrameSetup); 2766 // MOV32rm SavedEBPOffset(%esi), %ebp 2767 assert(X86FI->getHasSEHFramePtrSave()); 2768 int Offset = 2769 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg); 2770 assert(UsedReg == BasePtr); 2771 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), FramePtr), 2772 UsedReg, true, Offset) 2773 .setMIFlag(MachineInstr::FrameSetup); 2774 } else { 2775 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr"); 2776 } 2777 return MBBI; 2778 } 2779 2780 namespace { 2781 // Struct used by orderFrameObjects to help sort the stack objects. 2782 struct X86FrameSortingObject { 2783 bool IsValid = false; // true if we care about this Object. 2784 unsigned ObjectIndex = 0; // Index of Object into MFI list. 2785 unsigned ObjectSize = 0; // Size of Object in bytes. 2786 unsigned ObjectAlignment = 1; // Alignment of Object in bytes. 2787 unsigned ObjectNumUses = 0; // Object static number of uses. 2788 }; 2789 2790 // The comparison function we use for std::sort to order our local 2791 // stack symbols. The current algorithm is to use an estimated 2792 // "density". This takes into consideration the size and number of 2793 // uses each object has in order to roughly minimize code size. 2794 // So, for example, an object of size 16B that is referenced 5 times 2795 // will get higher priority than 4 4B objects referenced 1 time each. 2796 // It's not perfect and we may be able to squeeze a few more bytes out of 2797 // it (for example : 0(esp) requires fewer bytes, symbols allocated at the 2798 // fringe end can have special consideration, given their size is less 2799 // important, etc.), but the algorithmic complexity grows too much to be 2800 // worth the extra gains we get. This gets us pretty close. 2801 // The final order leaves us with objects with highest priority going 2802 // at the end of our list. 2803 struct X86FrameSortingComparator { 2804 inline bool operator()(const X86FrameSortingObject &A, 2805 const X86FrameSortingObject &B) { 2806 uint64_t DensityAScaled, DensityBScaled; 2807 2808 // For consistency in our comparison, all invalid objects are placed 2809 // at the end. This also allows us to stop walking when we hit the 2810 // first invalid item after it's all sorted. 2811 if (!A.IsValid) 2812 return false; 2813 if (!B.IsValid) 2814 return true; 2815 2816 // The density is calculated by doing : 2817 // (double)DensityA = A.ObjectNumUses / A.ObjectSize 2818 // (double)DensityB = B.ObjectNumUses / B.ObjectSize 2819 // Since this approach may cause inconsistencies in 2820 // the floating point <, >, == comparisons, depending on the floating 2821 // point model with which the compiler was built, we're going 2822 // to scale both sides by multiplying with 2823 // A.ObjectSize * B.ObjectSize. This ends up factoring away 2824 // the division and, with it, the need for any floating point 2825 // arithmetic. 2826 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) * 2827 static_cast<uint64_t>(B.ObjectSize); 2828 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) * 2829 static_cast<uint64_t>(A.ObjectSize); 2830 2831 // If the two densities are equal, prioritize highest alignment 2832 // objects. This allows for similar alignment objects 2833 // to be packed together (given the same density). 2834 // There's room for improvement here, also, since we can pack 2835 // similar alignment (different density) objects next to each 2836 // other to save padding. This will also require further 2837 // complexity/iterations, and the overall gain isn't worth it, 2838 // in general. Something to keep in mind, though. 2839 if (DensityAScaled == DensityBScaled) 2840 return A.ObjectAlignment < B.ObjectAlignment; 2841 2842 return DensityAScaled < DensityBScaled; 2843 } 2844 }; 2845 } // namespace 2846 2847 // Order the symbols in the local stack. 2848 // We want to place the local stack objects in some sort of sensible order. 2849 // The heuristic we use is to try and pack them according to static number 2850 // of uses and size of object in order to minimize code size. 2851 void X86FrameLowering::orderFrameObjects( 2852 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const { 2853 const MachineFrameInfo *MFI = MF.getFrameInfo(); 2854 2855 // Don't waste time if there's nothing to do. 2856 if (ObjectsToAllocate.empty()) 2857 return; 2858 2859 // Create an array of all MFI objects. We won't need all of these 2860 // objects, but we're going to create a full array of them to make 2861 // it easier to index into when we're counting "uses" down below. 2862 // We want to be able to easily/cheaply access an object by simply 2863 // indexing into it, instead of having to search for it every time. 2864 std::vector<X86FrameSortingObject> SortingObjects(MFI->getObjectIndexEnd()); 2865 2866 // Walk the objects we care about and mark them as such in our working 2867 // struct. 2868 for (auto &Obj : ObjectsToAllocate) { 2869 SortingObjects[Obj].IsValid = true; 2870 SortingObjects[Obj].ObjectIndex = Obj; 2871 SortingObjects[Obj].ObjectAlignment = MFI->getObjectAlignment(Obj); 2872 // Set the size. 2873 int ObjectSize = MFI->getObjectSize(Obj); 2874 if (ObjectSize == 0) 2875 // Variable size. Just use 4. 2876 SortingObjects[Obj].ObjectSize = 4; 2877 else 2878 SortingObjects[Obj].ObjectSize = ObjectSize; 2879 } 2880 2881 // Count the number of uses for each object. 2882 for (auto &MBB : MF) { 2883 for (auto &MI : MBB) { 2884 if (MI.isDebugValue()) 2885 continue; 2886 for (const MachineOperand &MO : MI.operands()) { 2887 // Check to see if it's a local stack symbol. 2888 if (!MO.isFI()) 2889 continue; 2890 int Index = MO.getIndex(); 2891 // Check to see if it falls within our range, and is tagged 2892 // to require ordering. 2893 if (Index >= 0 && Index < MFI->getObjectIndexEnd() && 2894 SortingObjects[Index].IsValid) 2895 SortingObjects[Index].ObjectNumUses++; 2896 } 2897 } 2898 } 2899 2900 // Sort the objects using X86FrameSortingAlgorithm (see its comment for 2901 // info). 2902 std::stable_sort(SortingObjects.begin(), SortingObjects.end(), 2903 X86FrameSortingComparator()); 2904 2905 // Now modify the original list to represent the final order that 2906 // we want. The order will depend on whether we're going to access them 2907 // from the stack pointer or the frame pointer. For SP, the list should 2908 // end up with the END containing objects that we want with smaller offsets. 2909 // For FP, it should be flipped. 2910 int i = 0; 2911 for (auto &Obj : SortingObjects) { 2912 // All invalid items are sorted at the end, so it's safe to stop. 2913 if (!Obj.IsValid) 2914 break; 2915 ObjectsToAllocate[i++] = Obj.ObjectIndex; 2916 } 2917 2918 // Flip it if we're accessing off of the FP. 2919 if (!TRI->needsStackRealignment(MF) && hasFP(MF)) 2920 std::reverse(ObjectsToAllocate.begin(), ObjectsToAllocate.end()); 2921 } 2922 2923 2924 unsigned X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const { 2925 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue. 2926 unsigned Offset = 16; 2927 // RBP is immediately pushed. 2928 Offset += SlotSize; 2929 // All callee-saved registers are then pushed. 2930 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize(); 2931 // Every funclet allocates enough stack space for the largest outgoing call. 2932 Offset += getWinEHFuncletFrameSize(MF); 2933 return Offset; 2934 } 2935 2936 void X86FrameLowering::processFunctionBeforeFrameFinalized( 2937 MachineFunction &MF, RegScavenger *RS) const { 2938 // If this function isn't doing Win64-style C++ EH, we don't need to do 2939 // anything. 2940 const Function *Fn = MF.getFunction(); 2941 if (!STI.is64Bit() || !MF.getMMI().hasEHFunclets() || 2942 classifyEHPersonality(Fn->getPersonalityFn()) != EHPersonality::MSVC_CXX) 2943 return; 2944 2945 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset 2946 // relative to RSP after the prologue. Find the offset of the last fixed 2947 // object, so that we can allocate a slot immediately following it. If there 2948 // were no fixed objects, use offset -SlotSize, which is immediately after the 2949 // return address. Fixed objects have negative frame indices. 2950 MachineFrameInfo *MFI = MF.getFrameInfo(); 2951 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo(); 2952 int64_t MinFixedObjOffset = -SlotSize; 2953 for (int I = MFI->getObjectIndexBegin(); I < 0; ++I) 2954 MinFixedObjOffset = std::min(MinFixedObjOffset, MFI->getObjectOffset(I)); 2955 2956 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) { 2957 for (WinEHHandlerType &H : TBME.HandlerArray) { 2958 int FrameIndex = H.CatchObj.FrameIndex; 2959 if (FrameIndex != INT_MAX) { 2960 // Ensure alignment. 2961 unsigned Align = MFI->getObjectAlignment(FrameIndex); 2962 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % Align; 2963 MinFixedObjOffset -= MFI->getObjectSize(FrameIndex); 2964 MFI->setObjectOffset(FrameIndex, MinFixedObjOffset); 2965 } 2966 } 2967 } 2968 2969 // Ensure alignment. 2970 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8; 2971 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize; 2972 int UnwindHelpFI = 2973 MFI->CreateFixedObject(SlotSize, UnwindHelpOffset, /*Immutable=*/false); 2974 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI; 2975 2976 // Store -2 into UnwindHelp on function entry. We have to scan forwards past 2977 // other frame setup instructions. 2978 MachineBasicBlock &MBB = MF.front(); 2979 auto MBBI = MBB.begin(); 2980 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) 2981 ++MBBI; 2982 2983 DebugLoc DL = MBB.findDebugLoc(MBBI); 2984 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mi32)), 2985 UnwindHelpFI) 2986 .addImm(-2); 2987 } 2988