1 //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains code to lower X86 MachineInstrs to their corresponding 11 // MCInst records. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86AsmPrinter.h" 16 #include "X86RegisterInfo.h" 17 #include "InstPrinter/X86ATTInstPrinter.h" 18 #include "MCTargetDesc/X86BaseInfo.h" 19 #include "Utils/X86ShuffleDecode.h" 20 #include "llvm/ADT/Optional.h" 21 #include "llvm/ADT/SmallString.h" 22 #include "llvm/CodeGen/MachineFunction.h" 23 #include "llvm/CodeGen/MachineConstantPool.h" 24 #include "llvm/CodeGen/MachineOperand.h" 25 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 26 #include "llvm/CodeGen/StackMaps.h" 27 #include "llvm/IR/DataLayout.h" 28 #include "llvm/IR/GlobalValue.h" 29 #include "llvm/IR/Mangler.h" 30 #include "llvm/MC/MCAsmInfo.h" 31 #include "llvm/MC/MCCodeEmitter.h" 32 #include "llvm/MC/MCContext.h" 33 #include "llvm/MC/MCExpr.h" 34 #include "llvm/MC/MCFixup.h" 35 #include "llvm/MC/MCInst.h" 36 #include "llvm/MC/MCInstBuilder.h" 37 #include "llvm/MC/MCStreamer.h" 38 #include "llvm/MC/MCSymbol.h" 39 #include "llvm/Support/TargetRegistry.h" 40 using namespace llvm; 41 42 namespace { 43 44 /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst. 45 class X86MCInstLower { 46 MCContext &Ctx; 47 const MachineFunction &MF; 48 const TargetMachine &TM; 49 const MCAsmInfo &MAI; 50 X86AsmPrinter &AsmPrinter; 51 public: 52 X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter); 53 54 Optional<MCOperand> LowerMachineOperand(const MachineInstr *MI, 55 const MachineOperand &MO) const; 56 void Lower(const MachineInstr *MI, MCInst &OutMI) const; 57 58 MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const; 59 MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const; 60 61 private: 62 MachineModuleInfoMachO &getMachOMMI() const; 63 Mangler *getMang() const { 64 return AsmPrinter.Mang; 65 } 66 }; 67 68 } // end anonymous namespace 69 70 // Emit a minimal sequence of nops spanning NumBytes bytes. 71 static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, 72 const MCSubtargetInfo &STI); 73 74 namespace llvm { 75 X86AsmPrinter::StackMapShadowTracker::StackMapShadowTracker(TargetMachine &TM) 76 : TM(TM), InShadow(false), RequiredShadowSize(0), CurrentShadowSize(0) {} 77 78 X86AsmPrinter::StackMapShadowTracker::~StackMapShadowTracker() {} 79 80 void 81 X86AsmPrinter::StackMapShadowTracker::startFunction(MachineFunction &F) { 82 MF = &F; 83 CodeEmitter.reset(TM.getTarget().createMCCodeEmitter( 84 *MF->getSubtarget().getInstrInfo(), 85 *MF->getSubtarget().getRegisterInfo(), MF->getContext())); 86 } 87 88 void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst, 89 const MCSubtargetInfo &STI) { 90 if (InShadow) { 91 SmallString<256> Code; 92 SmallVector<MCFixup, 4> Fixups; 93 raw_svector_ostream VecOS(Code); 94 CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI); 95 CurrentShadowSize += Code.size(); 96 if (CurrentShadowSize >= RequiredShadowSize) 97 InShadow = false; // The shadow is big enough. Stop counting. 98 } 99 } 100 101 void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding( 102 MCStreamer &OutStreamer, const MCSubtargetInfo &STI) { 103 if (InShadow && CurrentShadowSize < RequiredShadowSize) { 104 InShadow = false; 105 EmitNops(OutStreamer, RequiredShadowSize - CurrentShadowSize, 106 MF->getSubtarget<X86Subtarget>().is64Bit(), STI); 107 } 108 } 109 110 void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) { 111 OutStreamer->EmitInstruction(Inst, getSubtargetInfo()); 112 SMShadowTracker.count(Inst, getSubtargetInfo()); 113 } 114 } // end llvm namespace 115 116 X86MCInstLower::X86MCInstLower(const MachineFunction &mf, 117 X86AsmPrinter &asmprinter) 118 : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()), 119 AsmPrinter(asmprinter) {} 120 121 MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const { 122 return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>(); 123 } 124 125 126 /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol 127 /// operand to an MCSymbol. 128 MCSymbol *X86MCInstLower:: 129 GetSymbolFromOperand(const MachineOperand &MO) const { 130 const DataLayout &DL = MF.getDataLayout(); 131 assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && "Isn't a symbol reference"); 132 133 MCSymbol *Sym = nullptr; 134 SmallString<128> Name; 135 StringRef Suffix; 136 137 switch (MO.getTargetFlags()) { 138 case X86II::MO_DLLIMPORT: 139 // Handle dllimport linkage. 140 Name += "__imp_"; 141 break; 142 case X86II::MO_DARWIN_STUB: 143 Suffix = "$stub"; 144 break; 145 case X86II::MO_DARWIN_NONLAZY: 146 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: 147 case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: 148 Suffix = "$non_lazy_ptr"; 149 break; 150 } 151 152 if (!Suffix.empty()) 153 Name += DL.getPrivateGlobalPrefix(); 154 155 unsigned PrefixLen = Name.size(); 156 157 if (MO.isGlobal()) { 158 const GlobalValue *GV = MO.getGlobal(); 159 AsmPrinter.getNameWithPrefix(Name, GV); 160 } else if (MO.isSymbol()) { 161 Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL); 162 } else if (MO.isMBB()) { 163 assert(Suffix.empty()); 164 Sym = MO.getMBB()->getSymbol(); 165 } 166 unsigned OrigLen = Name.size() - PrefixLen; 167 168 Name += Suffix; 169 if (!Sym) 170 Sym = Ctx.getOrCreateSymbol(Name); 171 172 StringRef OrigName = StringRef(Name).substr(PrefixLen, OrigLen); 173 174 // If the target flags on the operand changes the name of the symbol, do that 175 // before we return the symbol. 176 switch (MO.getTargetFlags()) { 177 default: break; 178 case X86II::MO_DARWIN_NONLAZY: 179 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: { 180 MachineModuleInfoImpl::StubValueTy &StubSym = 181 getMachOMMI().getGVStubEntry(Sym); 182 if (!StubSym.getPointer()) { 183 assert(MO.isGlobal() && "Extern symbol not handled yet"); 184 StubSym = 185 MachineModuleInfoImpl:: 186 StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), 187 !MO.getGlobal()->hasInternalLinkage()); 188 } 189 break; 190 } 191 case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: { 192 MachineModuleInfoImpl::StubValueTy &StubSym = 193 getMachOMMI().getHiddenGVStubEntry(Sym); 194 if (!StubSym.getPointer()) { 195 assert(MO.isGlobal() && "Extern symbol not handled yet"); 196 StubSym = 197 MachineModuleInfoImpl:: 198 StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), 199 !MO.getGlobal()->hasInternalLinkage()); 200 } 201 break; 202 } 203 case X86II::MO_DARWIN_STUB: { 204 MachineModuleInfoImpl::StubValueTy &StubSym = 205 getMachOMMI().getFnStubEntry(Sym); 206 if (StubSym.getPointer()) 207 return Sym; 208 209 if (MO.isGlobal()) { 210 StubSym = 211 MachineModuleInfoImpl:: 212 StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), 213 !MO.getGlobal()->hasInternalLinkage()); 214 } else { 215 StubSym = 216 MachineModuleInfoImpl:: 217 StubValueTy(Ctx.getOrCreateSymbol(OrigName), false); 218 } 219 break; 220 } 221 } 222 223 return Sym; 224 } 225 226 MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, 227 MCSymbol *Sym) const { 228 // FIXME: We would like an efficient form for this, so we don't have to do a 229 // lot of extra uniquing. 230 const MCExpr *Expr = nullptr; 231 MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None; 232 233 switch (MO.getTargetFlags()) { 234 default: llvm_unreachable("Unknown target flag on GV operand"); 235 case X86II::MO_NO_FLAG: // No flag. 236 // These affect the name of the symbol, not any suffix. 237 case X86II::MO_DARWIN_NONLAZY: 238 case X86II::MO_DLLIMPORT: 239 case X86II::MO_DARWIN_STUB: 240 break; 241 242 case X86II::MO_TLVP: RefKind = MCSymbolRefExpr::VK_TLVP; break; 243 case X86II::MO_TLVP_PIC_BASE: 244 Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx); 245 // Subtract the pic base. 246 Expr = MCBinaryExpr::createSub(Expr, 247 MCSymbolRefExpr::create(MF.getPICBaseSymbol(), 248 Ctx), 249 Ctx); 250 break; 251 case X86II::MO_SECREL: RefKind = MCSymbolRefExpr::VK_SECREL; break; 252 case X86II::MO_TLSGD: RefKind = MCSymbolRefExpr::VK_TLSGD; break; 253 case X86II::MO_TLSLD: RefKind = MCSymbolRefExpr::VK_TLSLD; break; 254 case X86II::MO_TLSLDM: RefKind = MCSymbolRefExpr::VK_TLSLDM; break; 255 case X86II::MO_GOTTPOFF: RefKind = MCSymbolRefExpr::VK_GOTTPOFF; break; 256 case X86II::MO_INDNTPOFF: RefKind = MCSymbolRefExpr::VK_INDNTPOFF; break; 257 case X86II::MO_TPOFF: RefKind = MCSymbolRefExpr::VK_TPOFF; break; 258 case X86II::MO_DTPOFF: RefKind = MCSymbolRefExpr::VK_DTPOFF; break; 259 case X86II::MO_NTPOFF: RefKind = MCSymbolRefExpr::VK_NTPOFF; break; 260 case X86II::MO_GOTNTPOFF: RefKind = MCSymbolRefExpr::VK_GOTNTPOFF; break; 261 case X86II::MO_GOTPCREL: RefKind = MCSymbolRefExpr::VK_GOTPCREL; break; 262 case X86II::MO_GOT: RefKind = MCSymbolRefExpr::VK_GOT; break; 263 case X86II::MO_GOTOFF: RefKind = MCSymbolRefExpr::VK_GOTOFF; break; 264 case X86II::MO_PLT: RefKind = MCSymbolRefExpr::VK_PLT; break; 265 case X86II::MO_PIC_BASE_OFFSET: 266 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: 267 case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: 268 Expr = MCSymbolRefExpr::create(Sym, Ctx); 269 // Subtract the pic base. 270 Expr = MCBinaryExpr::createSub(Expr, 271 MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), 272 Ctx); 273 if (MO.isJTI()) { 274 assert(MAI.doesSetDirectiveSuppressesReloc()); 275 // If .set directive is supported, use it to reduce the number of 276 // relocations the assembler will generate for differences between 277 // local labels. This is only safe when the symbols are in the same 278 // section so we are restricting it to jumptable references. 279 MCSymbol *Label = Ctx.createTempSymbol(); 280 AsmPrinter.OutStreamer->EmitAssignment(Label, Expr); 281 Expr = MCSymbolRefExpr::create(Label, Ctx); 282 } 283 break; 284 } 285 286 if (!Expr) 287 Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx); 288 289 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset()) 290 Expr = MCBinaryExpr::createAdd(Expr, 291 MCConstantExpr::create(MO.getOffset(), Ctx), 292 Ctx); 293 return MCOperand::createExpr(Expr); 294 } 295 296 297 /// \brief Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with 298 /// a short fixed-register form. 299 static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) { 300 unsigned ImmOp = Inst.getNumOperands() - 1; 301 assert(Inst.getOperand(0).isReg() && 302 (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && 303 ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && 304 Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || 305 Inst.getNumOperands() == 2) && "Unexpected instruction!"); 306 307 // Check whether the destination register can be fixed. 308 unsigned Reg = Inst.getOperand(0).getReg(); 309 if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX) 310 return; 311 312 // If so, rewrite the instruction. 313 MCOperand Saved = Inst.getOperand(ImmOp); 314 Inst = MCInst(); 315 Inst.setOpcode(Opcode); 316 Inst.addOperand(Saved); 317 } 318 319 /// \brief If a movsx instruction has a shorter encoding for the used register 320 /// simplify the instruction to use it instead. 321 static void SimplifyMOVSX(MCInst &Inst) { 322 unsigned NewOpcode = 0; 323 unsigned Op0 = Inst.getOperand(0).getReg(), Op1 = Inst.getOperand(1).getReg(); 324 switch (Inst.getOpcode()) { 325 default: 326 llvm_unreachable("Unexpected instruction!"); 327 case X86::MOVSX16rr8: // movsbw %al, %ax --> cbtw 328 if (Op0 == X86::AX && Op1 == X86::AL) 329 NewOpcode = X86::CBW; 330 break; 331 case X86::MOVSX32rr16: // movswl %ax, %eax --> cwtl 332 if (Op0 == X86::EAX && Op1 == X86::AX) 333 NewOpcode = X86::CWDE; 334 break; 335 case X86::MOVSX64rr32: // movslq %eax, %rax --> cltq 336 if (Op0 == X86::RAX && Op1 == X86::EAX) 337 NewOpcode = X86::CDQE; 338 break; 339 } 340 341 if (NewOpcode != 0) { 342 Inst = MCInst(); 343 Inst.setOpcode(NewOpcode); 344 } 345 } 346 347 /// \brief Simplify things like MOV32rm to MOV32o32a. 348 static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst, 349 unsigned Opcode) { 350 // Don't make these simplifications in 64-bit mode; other assemblers don't 351 // perform them because they make the code larger. 352 if (Printer.getSubtarget().is64Bit()) 353 return; 354 355 bool IsStore = Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg(); 356 unsigned AddrBase = IsStore; 357 unsigned RegOp = IsStore ? 0 : 5; 358 unsigned AddrOp = AddrBase + 3; 359 assert(Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && 360 Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && 361 Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && 362 Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && 363 Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && 364 (Inst.getOperand(AddrOp).isExpr() || 365 Inst.getOperand(AddrOp).isImm()) && 366 "Unexpected instruction!"); 367 368 // Check whether the destination register can be fixed. 369 unsigned Reg = Inst.getOperand(RegOp).getReg(); 370 if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX) 371 return; 372 373 // Check whether this is an absolute address. 374 // FIXME: We know TLVP symbol refs aren't, but there should be a better way 375 // to do this here. 376 bool Absolute = true; 377 if (Inst.getOperand(AddrOp).isExpr()) { 378 const MCExpr *MCE = Inst.getOperand(AddrOp).getExpr(); 379 if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE)) 380 if (SRE->getKind() == MCSymbolRefExpr::VK_TLVP) 381 Absolute = false; 382 } 383 384 if (Absolute && 385 (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 || 386 Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 || 387 Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0)) 388 return; 389 390 // If so, rewrite the instruction. 391 MCOperand Saved = Inst.getOperand(AddrOp); 392 MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg); 393 Inst = MCInst(); 394 Inst.setOpcode(Opcode); 395 Inst.addOperand(Saved); 396 Inst.addOperand(Seg); 397 } 398 399 static unsigned getRetOpcode(const X86Subtarget &Subtarget) { 400 return Subtarget.is64Bit() ? X86::RETQ : X86::RETL; 401 } 402 403 Optional<MCOperand> 404 X86MCInstLower::LowerMachineOperand(const MachineInstr *MI, 405 const MachineOperand &MO) const { 406 switch (MO.getType()) { 407 default: 408 MI->dump(); 409 llvm_unreachable("unknown operand type"); 410 case MachineOperand::MO_Register: 411 // Ignore all implicit register operands. 412 if (MO.isImplicit()) 413 return None; 414 return MCOperand::createReg(MO.getReg()); 415 case MachineOperand::MO_Immediate: 416 return MCOperand::createImm(MO.getImm()); 417 case MachineOperand::MO_MachineBasicBlock: 418 case MachineOperand::MO_GlobalAddress: 419 case MachineOperand::MO_ExternalSymbol: 420 return LowerSymbolOperand(MO, GetSymbolFromOperand(MO)); 421 case MachineOperand::MO_MCSymbol: 422 return LowerSymbolOperand(MO, MO.getMCSymbol()); 423 case MachineOperand::MO_JumpTableIndex: 424 return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex())); 425 case MachineOperand::MO_ConstantPoolIndex: 426 return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex())); 427 case MachineOperand::MO_BlockAddress: 428 return LowerSymbolOperand( 429 MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress())); 430 case MachineOperand::MO_RegisterMask: 431 // Ignore call clobbers. 432 return None; 433 } 434 } 435 436 void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { 437 OutMI.setOpcode(MI->getOpcode()); 438 439 for (const MachineOperand &MO : MI->operands()) 440 if (auto MaybeMCOp = LowerMachineOperand(MI, MO)) 441 OutMI.addOperand(MaybeMCOp.getValue()); 442 443 // Handle a few special cases to eliminate operand modifiers. 444 ReSimplify: 445 switch (OutMI.getOpcode()) { 446 case X86::LEA64_32r: 447 case X86::LEA64r: 448 case X86::LEA16r: 449 case X86::LEA32r: 450 // LEA should have a segment register, but it must be empty. 451 assert(OutMI.getNumOperands() == 1+X86::AddrNumOperands && 452 "Unexpected # of LEA operands"); 453 assert(OutMI.getOperand(1+X86::AddrSegmentReg).getReg() == 0 && 454 "LEA has segment specified!"); 455 break; 456 457 case X86::MOV32ri64: 458 OutMI.setOpcode(X86::MOV32ri); 459 break; 460 461 // Commute operands to get a smaller encoding by using VEX.R instead of VEX.B 462 // if one of the registers is extended, but other isn't. 463 case X86::VMOVZPQILo2PQIrr: 464 case X86::VMOVAPDrr: 465 case X86::VMOVAPDYrr: 466 case X86::VMOVAPSrr: 467 case X86::VMOVAPSYrr: 468 case X86::VMOVDQArr: 469 case X86::VMOVDQAYrr: 470 case X86::VMOVDQUrr: 471 case X86::VMOVDQUYrr: 472 case X86::VMOVUPDrr: 473 case X86::VMOVUPDYrr: 474 case X86::VMOVUPSrr: 475 case X86::VMOVUPSYrr: { 476 if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) && 477 X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg())) { 478 unsigned NewOpc; 479 switch (OutMI.getOpcode()) { 480 default: llvm_unreachable("Invalid opcode"); 481 case X86::VMOVZPQILo2PQIrr: NewOpc = X86::VMOVPQI2QIrr; break; 482 case X86::VMOVAPDrr: NewOpc = X86::VMOVAPDrr_REV; break; 483 case X86::VMOVAPDYrr: NewOpc = X86::VMOVAPDYrr_REV; break; 484 case X86::VMOVAPSrr: NewOpc = X86::VMOVAPSrr_REV; break; 485 case X86::VMOVAPSYrr: NewOpc = X86::VMOVAPSYrr_REV; break; 486 case X86::VMOVDQArr: NewOpc = X86::VMOVDQArr_REV; break; 487 case X86::VMOVDQAYrr: NewOpc = X86::VMOVDQAYrr_REV; break; 488 case X86::VMOVDQUrr: NewOpc = X86::VMOVDQUrr_REV; break; 489 case X86::VMOVDQUYrr: NewOpc = X86::VMOVDQUYrr_REV; break; 490 case X86::VMOVUPDrr: NewOpc = X86::VMOVUPDrr_REV; break; 491 case X86::VMOVUPDYrr: NewOpc = X86::VMOVUPDYrr_REV; break; 492 case X86::VMOVUPSrr: NewOpc = X86::VMOVUPSrr_REV; break; 493 case X86::VMOVUPSYrr: NewOpc = X86::VMOVUPSYrr_REV; break; 494 } 495 OutMI.setOpcode(NewOpc); 496 } 497 break; 498 } 499 case X86::VMOVSDrr: 500 case X86::VMOVSSrr: { 501 if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) && 502 X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) { 503 unsigned NewOpc; 504 switch (OutMI.getOpcode()) { 505 default: llvm_unreachable("Invalid opcode"); 506 case X86::VMOVSDrr: NewOpc = X86::VMOVSDrr_REV; break; 507 case X86::VMOVSSrr: NewOpc = X86::VMOVSSrr_REV; break; 508 } 509 OutMI.setOpcode(NewOpc); 510 } 511 break; 512 } 513 514 // TAILJMPr64, CALL64r, CALL64pcrel32 - These instructions have register 515 // inputs modeled as normal uses instead of implicit uses. As such, truncate 516 // off all but the first operand (the callee). FIXME: Change isel. 517 case X86::TAILJMPr64: 518 case X86::TAILJMPr64_REX: 519 case X86::CALL64r: 520 case X86::CALL64pcrel32: { 521 unsigned Opcode = OutMI.getOpcode(); 522 MCOperand Saved = OutMI.getOperand(0); 523 OutMI = MCInst(); 524 OutMI.setOpcode(Opcode); 525 OutMI.addOperand(Saved); 526 break; 527 } 528 529 case X86::EH_RETURN: 530 case X86::EH_RETURN64: { 531 OutMI = MCInst(); 532 OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); 533 break; 534 } 535 536 case X86::CLEANUPRET: { 537 // Replace CATCHRET with the appropriate RET. 538 OutMI = MCInst(); 539 OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); 540 break; 541 } 542 543 case X86::CATCHRET: { 544 // Replace CATCHRET with the appropriate RET. 545 const X86Subtarget &Subtarget = AsmPrinter.getSubtarget(); 546 unsigned ReturnReg = Subtarget.is64Bit() ? X86::RAX : X86::EAX; 547 OutMI = MCInst(); 548 OutMI.setOpcode(getRetOpcode(Subtarget)); 549 OutMI.addOperand(MCOperand::createReg(ReturnReg)); 550 break; 551 } 552 553 // TAILJMPd, TAILJMPd64 - Lower to the correct jump instructions. 554 case X86::TAILJMPr: 555 case X86::TAILJMPd: 556 case X86::TAILJMPd64: { 557 unsigned Opcode; 558 switch (OutMI.getOpcode()) { 559 default: llvm_unreachable("Invalid opcode"); 560 case X86::TAILJMPr: Opcode = X86::JMP32r; break; 561 case X86::TAILJMPd: 562 case X86::TAILJMPd64: Opcode = X86::JMP_1; break; 563 } 564 565 MCOperand Saved = OutMI.getOperand(0); 566 OutMI = MCInst(); 567 OutMI.setOpcode(Opcode); 568 OutMI.addOperand(Saved); 569 break; 570 } 571 572 case X86::DEC16r: 573 case X86::DEC32r: 574 case X86::INC16r: 575 case X86::INC32r: 576 // If we aren't in 64-bit mode we can use the 1-byte inc/dec instructions. 577 if (!AsmPrinter.getSubtarget().is64Bit()) { 578 unsigned Opcode; 579 switch (OutMI.getOpcode()) { 580 default: llvm_unreachable("Invalid opcode"); 581 case X86::DEC16r: Opcode = X86::DEC16r_alt; break; 582 case X86::DEC32r: Opcode = X86::DEC32r_alt; break; 583 case X86::INC16r: Opcode = X86::INC16r_alt; break; 584 case X86::INC32r: Opcode = X86::INC32r_alt; break; 585 } 586 OutMI.setOpcode(Opcode); 587 } 588 break; 589 590 // These are pseudo-ops for OR to help with the OR->ADD transformation. We do 591 // this with an ugly goto in case the resultant OR uses EAX and needs the 592 // short form. 593 case X86::ADD16rr_DB: OutMI.setOpcode(X86::OR16rr); goto ReSimplify; 594 case X86::ADD32rr_DB: OutMI.setOpcode(X86::OR32rr); goto ReSimplify; 595 case X86::ADD64rr_DB: OutMI.setOpcode(X86::OR64rr); goto ReSimplify; 596 case X86::ADD16ri_DB: OutMI.setOpcode(X86::OR16ri); goto ReSimplify; 597 case X86::ADD32ri_DB: OutMI.setOpcode(X86::OR32ri); goto ReSimplify; 598 case X86::ADD64ri32_DB: OutMI.setOpcode(X86::OR64ri32); goto ReSimplify; 599 case X86::ADD16ri8_DB: OutMI.setOpcode(X86::OR16ri8); goto ReSimplify; 600 case X86::ADD32ri8_DB: OutMI.setOpcode(X86::OR32ri8); goto ReSimplify; 601 case X86::ADD64ri8_DB: OutMI.setOpcode(X86::OR64ri8); goto ReSimplify; 602 603 // Atomic load and store require a separate pseudo-inst because Acquire 604 // implies mayStore and Release implies mayLoad; fix these to regular MOV 605 // instructions here 606 case X86::ACQUIRE_MOV8rm: OutMI.setOpcode(X86::MOV8rm); goto ReSimplify; 607 case X86::ACQUIRE_MOV16rm: OutMI.setOpcode(X86::MOV16rm); goto ReSimplify; 608 case X86::ACQUIRE_MOV32rm: OutMI.setOpcode(X86::MOV32rm); goto ReSimplify; 609 case X86::ACQUIRE_MOV64rm: OutMI.setOpcode(X86::MOV64rm); goto ReSimplify; 610 case X86::RELEASE_MOV8mr: OutMI.setOpcode(X86::MOV8mr); goto ReSimplify; 611 case X86::RELEASE_MOV16mr: OutMI.setOpcode(X86::MOV16mr); goto ReSimplify; 612 case X86::RELEASE_MOV32mr: OutMI.setOpcode(X86::MOV32mr); goto ReSimplify; 613 case X86::RELEASE_MOV64mr: OutMI.setOpcode(X86::MOV64mr); goto ReSimplify; 614 case X86::RELEASE_MOV8mi: OutMI.setOpcode(X86::MOV8mi); goto ReSimplify; 615 case X86::RELEASE_MOV16mi: OutMI.setOpcode(X86::MOV16mi); goto ReSimplify; 616 case X86::RELEASE_MOV32mi: OutMI.setOpcode(X86::MOV32mi); goto ReSimplify; 617 case X86::RELEASE_MOV64mi32: OutMI.setOpcode(X86::MOV64mi32); goto ReSimplify; 618 case X86::RELEASE_ADD8mi: OutMI.setOpcode(X86::ADD8mi); goto ReSimplify; 619 case X86::RELEASE_ADD8mr: OutMI.setOpcode(X86::ADD8mr); goto ReSimplify; 620 case X86::RELEASE_ADD32mi: OutMI.setOpcode(X86::ADD32mi); goto ReSimplify; 621 case X86::RELEASE_ADD32mr: OutMI.setOpcode(X86::ADD32mr); goto ReSimplify; 622 case X86::RELEASE_ADD64mi32: OutMI.setOpcode(X86::ADD64mi32); goto ReSimplify; 623 case X86::RELEASE_ADD64mr: OutMI.setOpcode(X86::ADD64mr); goto ReSimplify; 624 case X86::RELEASE_AND8mi: OutMI.setOpcode(X86::AND8mi); goto ReSimplify; 625 case X86::RELEASE_AND8mr: OutMI.setOpcode(X86::AND8mr); goto ReSimplify; 626 case X86::RELEASE_AND32mi: OutMI.setOpcode(X86::AND32mi); goto ReSimplify; 627 case X86::RELEASE_AND32mr: OutMI.setOpcode(X86::AND32mr); goto ReSimplify; 628 case X86::RELEASE_AND64mi32: OutMI.setOpcode(X86::AND64mi32); goto ReSimplify; 629 case X86::RELEASE_AND64mr: OutMI.setOpcode(X86::AND64mr); goto ReSimplify; 630 case X86::RELEASE_OR8mi: OutMI.setOpcode(X86::OR8mi); goto ReSimplify; 631 case X86::RELEASE_OR8mr: OutMI.setOpcode(X86::OR8mr); goto ReSimplify; 632 case X86::RELEASE_OR32mi: OutMI.setOpcode(X86::OR32mi); goto ReSimplify; 633 case X86::RELEASE_OR32mr: OutMI.setOpcode(X86::OR32mr); goto ReSimplify; 634 case X86::RELEASE_OR64mi32: OutMI.setOpcode(X86::OR64mi32); goto ReSimplify; 635 case X86::RELEASE_OR64mr: OutMI.setOpcode(X86::OR64mr); goto ReSimplify; 636 case X86::RELEASE_XOR8mi: OutMI.setOpcode(X86::XOR8mi); goto ReSimplify; 637 case X86::RELEASE_XOR8mr: OutMI.setOpcode(X86::XOR8mr); goto ReSimplify; 638 case X86::RELEASE_XOR32mi: OutMI.setOpcode(X86::XOR32mi); goto ReSimplify; 639 case X86::RELEASE_XOR32mr: OutMI.setOpcode(X86::XOR32mr); goto ReSimplify; 640 case X86::RELEASE_XOR64mi32: OutMI.setOpcode(X86::XOR64mi32); goto ReSimplify; 641 case X86::RELEASE_XOR64mr: OutMI.setOpcode(X86::XOR64mr); goto ReSimplify; 642 case X86::RELEASE_INC8m: OutMI.setOpcode(X86::INC8m); goto ReSimplify; 643 case X86::RELEASE_INC16m: OutMI.setOpcode(X86::INC16m); goto ReSimplify; 644 case X86::RELEASE_INC32m: OutMI.setOpcode(X86::INC32m); goto ReSimplify; 645 case X86::RELEASE_INC64m: OutMI.setOpcode(X86::INC64m); goto ReSimplify; 646 case X86::RELEASE_DEC8m: OutMI.setOpcode(X86::DEC8m); goto ReSimplify; 647 case X86::RELEASE_DEC16m: OutMI.setOpcode(X86::DEC16m); goto ReSimplify; 648 case X86::RELEASE_DEC32m: OutMI.setOpcode(X86::DEC32m); goto ReSimplify; 649 case X86::RELEASE_DEC64m: OutMI.setOpcode(X86::DEC64m); goto ReSimplify; 650 651 // We don't currently select the correct instruction form for instructions 652 // which have a short %eax, etc. form. Handle this by custom lowering, for 653 // now. 654 // 655 // Note, we are currently not handling the following instructions: 656 // MOV64ao8, MOV64o8a 657 // XCHG16ar, XCHG32ar, XCHG64ar 658 case X86::MOV8mr_NOREX: 659 case X86::MOV8mr: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV8o32a); break; 660 case X86::MOV8rm_NOREX: 661 case X86::MOV8rm: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV8ao32); break; 662 case X86::MOV16mr: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV16o32a); break; 663 case X86::MOV16rm: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV16ao32); break; 664 case X86::MOV32mr: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV32o32a); break; 665 case X86::MOV32rm: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV32ao32); break; 666 667 case X86::ADC8ri: SimplifyShortImmForm(OutMI, X86::ADC8i8); break; 668 case X86::ADC16ri: SimplifyShortImmForm(OutMI, X86::ADC16i16); break; 669 case X86::ADC32ri: SimplifyShortImmForm(OutMI, X86::ADC32i32); break; 670 case X86::ADC64ri32: SimplifyShortImmForm(OutMI, X86::ADC64i32); break; 671 case X86::ADD8ri: SimplifyShortImmForm(OutMI, X86::ADD8i8); break; 672 case X86::ADD16ri: SimplifyShortImmForm(OutMI, X86::ADD16i16); break; 673 case X86::ADD32ri: SimplifyShortImmForm(OutMI, X86::ADD32i32); break; 674 case X86::ADD64ri32: SimplifyShortImmForm(OutMI, X86::ADD64i32); break; 675 case X86::AND8ri: SimplifyShortImmForm(OutMI, X86::AND8i8); break; 676 case X86::AND16ri: SimplifyShortImmForm(OutMI, X86::AND16i16); break; 677 case X86::AND32ri: SimplifyShortImmForm(OutMI, X86::AND32i32); break; 678 case X86::AND64ri32: SimplifyShortImmForm(OutMI, X86::AND64i32); break; 679 case X86::CMP8ri: SimplifyShortImmForm(OutMI, X86::CMP8i8); break; 680 case X86::CMP16ri: SimplifyShortImmForm(OutMI, X86::CMP16i16); break; 681 case X86::CMP32ri: SimplifyShortImmForm(OutMI, X86::CMP32i32); break; 682 case X86::CMP64ri32: SimplifyShortImmForm(OutMI, X86::CMP64i32); break; 683 case X86::OR8ri: SimplifyShortImmForm(OutMI, X86::OR8i8); break; 684 case X86::OR16ri: SimplifyShortImmForm(OutMI, X86::OR16i16); break; 685 case X86::OR32ri: SimplifyShortImmForm(OutMI, X86::OR32i32); break; 686 case X86::OR64ri32: SimplifyShortImmForm(OutMI, X86::OR64i32); break; 687 case X86::SBB8ri: SimplifyShortImmForm(OutMI, X86::SBB8i8); break; 688 case X86::SBB16ri: SimplifyShortImmForm(OutMI, X86::SBB16i16); break; 689 case X86::SBB32ri: SimplifyShortImmForm(OutMI, X86::SBB32i32); break; 690 case X86::SBB64ri32: SimplifyShortImmForm(OutMI, X86::SBB64i32); break; 691 case X86::SUB8ri: SimplifyShortImmForm(OutMI, X86::SUB8i8); break; 692 case X86::SUB16ri: SimplifyShortImmForm(OutMI, X86::SUB16i16); break; 693 case X86::SUB32ri: SimplifyShortImmForm(OutMI, X86::SUB32i32); break; 694 case X86::SUB64ri32: SimplifyShortImmForm(OutMI, X86::SUB64i32); break; 695 case X86::TEST8ri: SimplifyShortImmForm(OutMI, X86::TEST8i8); break; 696 case X86::TEST16ri: SimplifyShortImmForm(OutMI, X86::TEST16i16); break; 697 case X86::TEST32ri: SimplifyShortImmForm(OutMI, X86::TEST32i32); break; 698 case X86::TEST64ri32: SimplifyShortImmForm(OutMI, X86::TEST64i32); break; 699 case X86::XOR8ri: SimplifyShortImmForm(OutMI, X86::XOR8i8); break; 700 case X86::XOR16ri: SimplifyShortImmForm(OutMI, X86::XOR16i16); break; 701 case X86::XOR32ri: SimplifyShortImmForm(OutMI, X86::XOR32i32); break; 702 case X86::XOR64ri32: SimplifyShortImmForm(OutMI, X86::XOR64i32); break; 703 704 // Try to shrink some forms of movsx. 705 case X86::MOVSX16rr8: 706 case X86::MOVSX32rr16: 707 case X86::MOVSX64rr32: 708 SimplifyMOVSX(OutMI); 709 break; 710 } 711 } 712 713 void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, 714 const MachineInstr &MI) { 715 716 bool is64Bits = MI.getOpcode() == X86::TLS_addr64 || 717 MI.getOpcode() == X86::TLS_base_addr64; 718 719 bool needsPadding = MI.getOpcode() == X86::TLS_addr64; 720 721 MCContext &context = OutStreamer->getContext(); 722 723 if (needsPadding) 724 EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); 725 726 MCSymbolRefExpr::VariantKind SRVK; 727 switch (MI.getOpcode()) { 728 case X86::TLS_addr32: 729 case X86::TLS_addr64: 730 SRVK = MCSymbolRefExpr::VK_TLSGD; 731 break; 732 case X86::TLS_base_addr32: 733 SRVK = MCSymbolRefExpr::VK_TLSLDM; 734 break; 735 case X86::TLS_base_addr64: 736 SRVK = MCSymbolRefExpr::VK_TLSLD; 737 break; 738 default: 739 llvm_unreachable("unexpected opcode"); 740 } 741 742 MCSymbol *sym = MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)); 743 const MCSymbolRefExpr *symRef = MCSymbolRefExpr::create(sym, SRVK, context); 744 745 MCInst LEA; 746 if (is64Bits) { 747 LEA.setOpcode(X86::LEA64r); 748 LEA.addOperand(MCOperand::createReg(X86::RDI)); // dest 749 LEA.addOperand(MCOperand::createReg(X86::RIP)); // base 750 LEA.addOperand(MCOperand::createImm(1)); // scale 751 LEA.addOperand(MCOperand::createReg(0)); // index 752 LEA.addOperand(MCOperand::createExpr(symRef)); // disp 753 LEA.addOperand(MCOperand::createReg(0)); // seg 754 } else if (SRVK == MCSymbolRefExpr::VK_TLSLDM) { 755 LEA.setOpcode(X86::LEA32r); 756 LEA.addOperand(MCOperand::createReg(X86::EAX)); // dest 757 LEA.addOperand(MCOperand::createReg(X86::EBX)); // base 758 LEA.addOperand(MCOperand::createImm(1)); // scale 759 LEA.addOperand(MCOperand::createReg(0)); // index 760 LEA.addOperand(MCOperand::createExpr(symRef)); // disp 761 LEA.addOperand(MCOperand::createReg(0)); // seg 762 } else { 763 LEA.setOpcode(X86::LEA32r); 764 LEA.addOperand(MCOperand::createReg(X86::EAX)); // dest 765 LEA.addOperand(MCOperand::createReg(0)); // base 766 LEA.addOperand(MCOperand::createImm(1)); // scale 767 LEA.addOperand(MCOperand::createReg(X86::EBX)); // index 768 LEA.addOperand(MCOperand::createExpr(symRef)); // disp 769 LEA.addOperand(MCOperand::createReg(0)); // seg 770 } 771 EmitAndCountInstruction(LEA); 772 773 if (needsPadding) { 774 EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); 775 EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); 776 EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX)); 777 } 778 779 StringRef name = is64Bits ? "__tls_get_addr" : "___tls_get_addr"; 780 MCSymbol *tlsGetAddr = context.getOrCreateSymbol(name); 781 const MCSymbolRefExpr *tlsRef = 782 MCSymbolRefExpr::create(tlsGetAddr, 783 MCSymbolRefExpr::VK_PLT, 784 context); 785 786 EmitAndCountInstruction(MCInstBuilder(is64Bits ? X86::CALL64pcrel32 787 : X86::CALLpcrel32) 788 .addExpr(tlsRef)); 789 } 790 791 /// \brief Emit the optimal amount of multi-byte nops on X86. 792 static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSubtargetInfo &STI) { 793 // This works only for 64bit. For 32bit we have to do additional checking if 794 // the CPU supports multi-byte nops. 795 assert(Is64Bit && "EmitNops only supports X86-64"); 796 while (NumBytes) { 797 unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg; 798 Opc = IndexReg = Displacement = SegmentReg = 0; 799 BaseReg = X86::RAX; ScaleVal = 1; 800 switch (NumBytes) { 801 case 0: llvm_unreachable("Zero nops?"); break; 802 case 1: NumBytes -= 1; Opc = X86::NOOP; break; 803 case 2: NumBytes -= 2; Opc = X86::XCHG16ar; break; 804 case 3: NumBytes -= 3; Opc = X86::NOOPL; break; 805 case 4: NumBytes -= 4; Opc = X86::NOOPL; Displacement = 8; break; 806 case 5: NumBytes -= 5; Opc = X86::NOOPL; Displacement = 8; 807 IndexReg = X86::RAX; break; 808 case 6: NumBytes -= 6; Opc = X86::NOOPW; Displacement = 8; 809 IndexReg = X86::RAX; break; 810 case 7: NumBytes -= 7; Opc = X86::NOOPL; Displacement = 512; break; 811 case 8: NumBytes -= 8; Opc = X86::NOOPL; Displacement = 512; 812 IndexReg = X86::RAX; break; 813 case 9: NumBytes -= 9; Opc = X86::NOOPW; Displacement = 512; 814 IndexReg = X86::RAX; break; 815 default: NumBytes -= 10; Opc = X86::NOOPW; Displacement = 512; 816 IndexReg = X86::RAX; SegmentReg = X86::CS; break; 817 } 818 819 unsigned NumPrefixes = std::min(NumBytes, 5U); 820 NumBytes -= NumPrefixes; 821 for (unsigned i = 0; i != NumPrefixes; ++i) 822 OS.EmitBytes("\x66"); 823 824 switch (Opc) { 825 default: llvm_unreachable("Unexpected opcode"); break; 826 case X86::NOOP: 827 OS.EmitInstruction(MCInstBuilder(Opc), STI); 828 break; 829 case X86::XCHG16ar: 830 OS.EmitInstruction(MCInstBuilder(Opc).addReg(X86::AX), STI); 831 break; 832 case X86::NOOPL: 833 case X86::NOOPW: 834 OS.EmitInstruction(MCInstBuilder(Opc).addReg(BaseReg) 835 .addImm(ScaleVal).addReg(IndexReg) 836 .addImm(Displacement).addReg(SegmentReg), STI); 837 break; 838 } 839 } // while (NumBytes) 840 } 841 842 void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI, 843 X86MCInstLower &MCIL) { 844 assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64"); 845 846 StatepointOpers SOpers(&MI); 847 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 848 EmitNops(*OutStreamer, PatchBytes, Subtarget->is64Bit(), 849 getSubtargetInfo()); 850 } else { 851 // Lower call target and choose correct opcode 852 const MachineOperand &CallTarget = SOpers.getCallTarget(); 853 MCOperand CallTargetMCOp; 854 unsigned CallOpcode; 855 switch (CallTarget.getType()) { 856 case MachineOperand::MO_GlobalAddress: 857 case MachineOperand::MO_ExternalSymbol: 858 CallTargetMCOp = MCIL.LowerSymbolOperand( 859 CallTarget, MCIL.GetSymbolFromOperand(CallTarget)); 860 CallOpcode = X86::CALL64pcrel32; 861 // Currently, we only support relative addressing with statepoints. 862 // Otherwise, we'll need a scratch register to hold the target 863 // address. You'll fail asserts during load & relocation if this 864 // symbol is to far away. (TODO: support non-relative addressing) 865 break; 866 case MachineOperand::MO_Immediate: 867 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm()); 868 CallOpcode = X86::CALL64pcrel32; 869 // Currently, we only support relative addressing with statepoints. 870 // Otherwise, we'll need a scratch register to hold the target 871 // immediate. You'll fail asserts during load & relocation if this 872 // address is to far away. (TODO: support non-relative addressing) 873 break; 874 case MachineOperand::MO_Register: 875 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg()); 876 CallOpcode = X86::CALL64r; 877 break; 878 default: 879 llvm_unreachable("Unsupported operand type in statepoint call target"); 880 break; 881 } 882 883 // Emit call 884 MCInst CallInst; 885 CallInst.setOpcode(CallOpcode); 886 CallInst.addOperand(CallTargetMCOp); 887 OutStreamer->EmitInstruction(CallInst, getSubtargetInfo()); 888 } 889 890 // Record our statepoint node in the same section used by STACKMAP 891 // and PATCHPOINT 892 SM.recordStatepoint(MI); 893 } 894 895 void X86AsmPrinter::LowerFAULTING_LOAD_OP(const MachineInstr &MI, 896 X86MCInstLower &MCIL) { 897 // FAULTING_LOAD_OP <def>, <handler label>, <load opcode>, <load operands> 898 899 unsigned LoadDefRegister = MI.getOperand(0).getReg(); 900 MCSymbol *HandlerLabel = MI.getOperand(1).getMCSymbol(); 901 unsigned LoadOpcode = MI.getOperand(2).getImm(); 902 unsigned LoadOperandsBeginIdx = 3; 903 904 FM.recordFaultingOp(FaultMaps::FaultingLoad, HandlerLabel); 905 906 MCInst LoadMI; 907 LoadMI.setOpcode(LoadOpcode); 908 909 if (LoadDefRegister != X86::NoRegister) 910 LoadMI.addOperand(MCOperand::createReg(LoadDefRegister)); 911 912 for (auto I = MI.operands_begin() + LoadOperandsBeginIdx, 913 E = MI.operands_end(); 914 I != E; ++I) 915 if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, *I)) 916 LoadMI.addOperand(MaybeOperand.getValue()); 917 918 OutStreamer->EmitInstruction(LoadMI, getSubtargetInfo()); 919 } 920 921 // Lower a stackmap of the form: 922 // <id>, <shadowBytes>, ... 923 void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) { 924 SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); 925 SM.recordStackMap(MI); 926 unsigned NumShadowBytes = MI.getOperand(1).getImm(); 927 SMShadowTracker.reset(NumShadowBytes); 928 } 929 930 // Lower a patchpoint of the form: 931 // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ... 932 void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, 933 X86MCInstLower &MCIL) { 934 assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64"); 935 936 SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); 937 938 SM.recordPatchPoint(MI); 939 940 PatchPointOpers opers(&MI); 941 unsigned ScratchIdx = opers.getNextScratchIdx(); 942 unsigned EncodedBytes = 0; 943 const MachineOperand &CalleeMO = 944 opers.getMetaOper(PatchPointOpers::TargetPos); 945 946 // Check for null target. If target is non-null (i.e. is non-zero or is 947 // symbolic) then emit a call. 948 if (!(CalleeMO.isImm() && !CalleeMO.getImm())) { 949 MCOperand CalleeMCOp; 950 switch (CalleeMO.getType()) { 951 default: 952 /// FIXME: Add a verifier check for bad callee types. 953 llvm_unreachable("Unrecognized callee operand type."); 954 case MachineOperand::MO_Immediate: 955 if (CalleeMO.getImm()) 956 CalleeMCOp = MCOperand::createImm(CalleeMO.getImm()); 957 break; 958 case MachineOperand::MO_ExternalSymbol: 959 case MachineOperand::MO_GlobalAddress: 960 CalleeMCOp = 961 MCIL.LowerSymbolOperand(CalleeMO, 962 MCIL.GetSymbolFromOperand(CalleeMO)); 963 break; 964 } 965 966 // Emit MOV to materialize the target address and the CALL to target. 967 // This is encoded with 12-13 bytes, depending on which register is used. 968 unsigned ScratchReg = MI.getOperand(ScratchIdx).getReg(); 969 if (X86II::isX86_64ExtendedReg(ScratchReg)) 970 EncodedBytes = 13; 971 else 972 EncodedBytes = 12; 973 974 EmitAndCountInstruction( 975 MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp)); 976 EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg)); 977 } 978 979 // Emit padding. 980 unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm(); 981 assert(NumBytes >= EncodedBytes && 982 "Patchpoint can't request size less than the length of a call."); 983 984 EmitNops(*OutStreamer, NumBytes - EncodedBytes, Subtarget->is64Bit(), 985 getSubtargetInfo()); 986 } 987 988 // Returns instruction preceding MBBI in MachineFunction. 989 // If MBBI is the first instruction of the first basic block, returns null. 990 static MachineBasicBlock::const_iterator 991 PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) { 992 const MachineBasicBlock *MBB = MBBI->getParent(); 993 while (MBBI == MBB->begin()) { 994 if (MBB == MBB->getParent()->begin()) 995 return nullptr; 996 MBB = MBB->getPrevNode(); 997 MBBI = MBB->end(); 998 } 999 return --MBBI; 1000 } 1001 1002 static const Constant *getConstantFromPool(const MachineInstr &MI, 1003 const MachineOperand &Op) { 1004 if (!Op.isCPI()) 1005 return nullptr; 1006 1007 ArrayRef<MachineConstantPoolEntry> Constants = 1008 MI.getParent()->getParent()->getConstantPool()->getConstants(); 1009 const MachineConstantPoolEntry &ConstantEntry = 1010 Constants[Op.getIndex()]; 1011 1012 // Bail if this is a machine constant pool entry, we won't be able to dig out 1013 // anything useful. 1014 if (ConstantEntry.isMachineConstantPoolEntry()) 1015 return nullptr; 1016 1017 auto *C = dyn_cast<Constant>(ConstantEntry.Val.ConstVal); 1018 assert((!C || ConstantEntry.getType() == C->getType()) && 1019 "Expected a constant of the same type!"); 1020 return C; 1021 } 1022 1023 static std::string getShuffleComment(const MachineOperand &DstOp, 1024 const MachineOperand &SrcOp, 1025 ArrayRef<int> Mask) { 1026 std::string Comment; 1027 1028 // Compute the name for a register. This is really goofy because we have 1029 // multiple instruction printers that could (in theory) use different 1030 // names. Fortunately most people use the ATT style (outside of Windows) 1031 // and they actually agree on register naming here. Ultimately, this is 1032 // a comment, and so its OK if it isn't perfect. 1033 auto GetRegisterName = [](unsigned RegNum) -> StringRef { 1034 return X86ATTInstPrinter::getRegisterName(RegNum); 1035 }; 1036 1037 StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem"; 1038 StringRef SrcName = SrcOp.isReg() ? GetRegisterName(SrcOp.getReg()) : "mem"; 1039 1040 raw_string_ostream CS(Comment); 1041 CS << DstName << " = "; 1042 bool NeedComma = false; 1043 bool InSrc = false; 1044 for (int M : Mask) { 1045 // Wrap up any prior entry... 1046 if (M == SM_SentinelZero && InSrc) { 1047 InSrc = false; 1048 CS << "]"; 1049 } 1050 if (NeedComma) 1051 CS << ","; 1052 else 1053 NeedComma = true; 1054 1055 // Print this shuffle... 1056 if (M == SM_SentinelZero) { 1057 CS << "zero"; 1058 } else { 1059 if (!InSrc) { 1060 InSrc = true; 1061 CS << SrcName << "["; 1062 } 1063 if (M == SM_SentinelUndef) 1064 CS << "u"; 1065 else 1066 CS << M; 1067 } 1068 } 1069 if (InSrc) 1070 CS << "]"; 1071 CS.flush(); 1072 1073 return Comment; 1074 } 1075 1076 void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { 1077 X86MCInstLower MCInstLowering(*MF, *this); 1078 const X86RegisterInfo *RI = MF->getSubtarget<X86Subtarget>().getRegisterInfo(); 1079 1080 switch (MI->getOpcode()) { 1081 case TargetOpcode::DBG_VALUE: 1082 llvm_unreachable("Should be handled target independently"); 1083 1084 // Emit nothing here but a comment if we can. 1085 case X86::Int_MemBarrier: 1086 OutStreamer->emitRawComment("MEMBARRIER"); 1087 return; 1088 1089 1090 case X86::EH_RETURN: 1091 case X86::EH_RETURN64: { 1092 // Lower these as normal, but add some comments. 1093 unsigned Reg = MI->getOperand(0).getReg(); 1094 OutStreamer->AddComment(StringRef("eh_return, addr: %") + 1095 X86ATTInstPrinter::getRegisterName(Reg)); 1096 break; 1097 } 1098 case X86::CLEANUPRET: { 1099 // Lower these as normal, but add some comments. 1100 OutStreamer->AddComment("CLEANUPRET"); 1101 break; 1102 } 1103 1104 case X86::CATCHRET: { 1105 // Lower these as normal, but add some comments. 1106 OutStreamer->AddComment("CATCHRET"); 1107 break; 1108 } 1109 1110 case X86::TAILJMPr: 1111 case X86::TAILJMPm: 1112 case X86::TAILJMPd: 1113 case X86::TAILJMPr64: 1114 case X86::TAILJMPm64: 1115 case X86::TAILJMPd64: 1116 case X86::TAILJMPr64_REX: 1117 case X86::TAILJMPm64_REX: 1118 case X86::TAILJMPd64_REX: 1119 // Lower these as normal, but add some comments. 1120 OutStreamer->AddComment("TAILCALL"); 1121 break; 1122 1123 case X86::TLS_addr32: 1124 case X86::TLS_addr64: 1125 case X86::TLS_base_addr32: 1126 case X86::TLS_base_addr64: 1127 return LowerTlsAddr(MCInstLowering, *MI); 1128 1129 case X86::MOVPC32r: { 1130 // This is a pseudo op for a two instruction sequence with a label, which 1131 // looks like: 1132 // call "L1$pb" 1133 // "L1$pb": 1134 // popl %esi 1135 1136 // Emit the call. 1137 MCSymbol *PICBase = MF->getPICBaseSymbol(); 1138 // FIXME: We would like an efficient form for this, so we don't have to do a 1139 // lot of extra uniquing. 1140 EmitAndCountInstruction(MCInstBuilder(X86::CALLpcrel32) 1141 .addExpr(MCSymbolRefExpr::create(PICBase, OutContext))); 1142 1143 const X86FrameLowering* FrameLowering = 1144 MF->getSubtarget<X86Subtarget>().getFrameLowering(); 1145 bool hasFP = FrameLowering->hasFP(*MF); 1146 1147 // TODO: This is needed only if we require precise CFA. 1148 bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() && 1149 !OutStreamer->getDwarfFrameInfos().back().End; 1150 1151 int stackGrowth = -RI->getSlotSize(); 1152 1153 if (HasActiveDwarfFrame && !hasFP) { 1154 OutStreamer->EmitCFIAdjustCfaOffset(-stackGrowth); 1155 } 1156 1157 // Emit the label. 1158 OutStreamer->EmitLabel(PICBase); 1159 1160 // popl $reg 1161 EmitAndCountInstruction(MCInstBuilder(X86::POP32r) 1162 .addReg(MI->getOperand(0).getReg())); 1163 1164 if (HasActiveDwarfFrame && !hasFP) { 1165 OutStreamer->EmitCFIAdjustCfaOffset(stackGrowth); 1166 } 1167 return; 1168 } 1169 1170 case X86::ADD32ri: { 1171 // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri. 1172 if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS) 1173 break; 1174 1175 // Okay, we have something like: 1176 // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL) 1177 1178 // For this, we want to print something like: 1179 // MYGLOBAL + (. - PICBASE) 1180 // However, we can't generate a ".", so just emit a new label here and refer 1181 // to it. 1182 MCSymbol *DotSym = OutContext.createTempSymbol(); 1183 OutStreamer->EmitLabel(DotSym); 1184 1185 // Now that we have emitted the label, lower the complex operand expression. 1186 MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2)); 1187 1188 const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext); 1189 const MCExpr *PICBase = 1190 MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext); 1191 DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext); 1192 1193 DotExpr = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(OpSym,OutContext), 1194 DotExpr, OutContext); 1195 1196 EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri) 1197 .addReg(MI->getOperand(0).getReg()) 1198 .addReg(MI->getOperand(1).getReg()) 1199 .addExpr(DotExpr)); 1200 return; 1201 } 1202 case TargetOpcode::STATEPOINT: 1203 return LowerSTATEPOINT(*MI, MCInstLowering); 1204 1205 case TargetOpcode::FAULTING_LOAD_OP: 1206 return LowerFAULTING_LOAD_OP(*MI, MCInstLowering); 1207 1208 case TargetOpcode::STACKMAP: 1209 return LowerSTACKMAP(*MI); 1210 1211 case TargetOpcode::PATCHPOINT: 1212 return LowerPATCHPOINT(*MI, MCInstLowering); 1213 1214 case X86::MORESTACK_RET: 1215 EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); 1216 return; 1217 1218 case X86::MORESTACK_RET_RESTORE_R10: 1219 // Return, then restore R10. 1220 EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); 1221 EmitAndCountInstruction(MCInstBuilder(X86::MOV64rr) 1222 .addReg(X86::R10) 1223 .addReg(X86::RAX)); 1224 return; 1225 1226 case X86::SEH_PushReg: 1227 OutStreamer->EmitWinCFIPushReg(RI->getSEHRegNum(MI->getOperand(0).getImm())); 1228 return; 1229 1230 case X86::SEH_SaveReg: 1231 OutStreamer->EmitWinCFISaveReg(RI->getSEHRegNum(MI->getOperand(0).getImm()), 1232 MI->getOperand(1).getImm()); 1233 return; 1234 1235 case X86::SEH_SaveXMM: 1236 OutStreamer->EmitWinCFISaveXMM(RI->getSEHRegNum(MI->getOperand(0).getImm()), 1237 MI->getOperand(1).getImm()); 1238 return; 1239 1240 case X86::SEH_StackAlloc: 1241 OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm()); 1242 return; 1243 1244 case X86::SEH_SetFrame: 1245 OutStreamer->EmitWinCFISetFrame(RI->getSEHRegNum(MI->getOperand(0).getImm()), 1246 MI->getOperand(1).getImm()); 1247 return; 1248 1249 case X86::SEH_PushFrame: 1250 OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm()); 1251 return; 1252 1253 case X86::SEH_EndPrologue: 1254 OutStreamer->EmitWinCFIEndProlog(); 1255 return; 1256 1257 case X86::SEH_Epilogue: { 1258 MachineBasicBlock::const_iterator MBBI(MI); 1259 // Check if preceded by a call and emit nop if so. 1260 for (MBBI = PrevCrossBBInst(MBBI); MBBI; MBBI = PrevCrossBBInst(MBBI)) { 1261 // Conservatively assume that pseudo instructions don't emit code and keep 1262 // looking for a call. We may emit an unnecessary nop in some cases. 1263 if (!MBBI->isPseudo()) { 1264 if (MBBI->isCall()) 1265 EmitAndCountInstruction(MCInstBuilder(X86::NOOP)); 1266 break; 1267 } 1268 } 1269 return; 1270 } 1271 1272 // Lower PSHUFB and VPERMILP normally but add a comment if we can find 1273 // a constant shuffle mask. We won't be able to do this at the MC layer 1274 // because the mask isn't an immediate. 1275 case X86::PSHUFBrm: 1276 case X86::VPSHUFBrm: 1277 case X86::VPSHUFBYrm: { 1278 if (!OutStreamer->isVerboseAsm()) 1279 break; 1280 assert(MI->getNumOperands() > 5 && 1281 "We should always have at least 5 operands!"); 1282 const MachineOperand &DstOp = MI->getOperand(0); 1283 const MachineOperand &SrcOp = MI->getOperand(1); 1284 const MachineOperand &MaskOp = MI->getOperand(5); 1285 1286 if (auto *C = getConstantFromPool(*MI, MaskOp)) { 1287 SmallVector<int, 16> Mask; 1288 DecodePSHUFBMask(C, Mask); 1289 if (!Mask.empty()) 1290 OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, Mask)); 1291 } 1292 break; 1293 } 1294 case X86::VPERMILPSrm: 1295 case X86::VPERMILPDrm: 1296 case X86::VPERMILPSYrm: 1297 case X86::VPERMILPDYrm: { 1298 if (!OutStreamer->isVerboseAsm()) 1299 break; 1300 assert(MI->getNumOperands() > 5 && 1301 "We should always have at least 5 operands!"); 1302 const MachineOperand &DstOp = MI->getOperand(0); 1303 const MachineOperand &SrcOp = MI->getOperand(1); 1304 const MachineOperand &MaskOp = MI->getOperand(5); 1305 1306 if (auto *C = getConstantFromPool(*MI, MaskOp)) { 1307 SmallVector<int, 16> Mask; 1308 DecodeVPERMILPMask(C, Mask); 1309 if (!Mask.empty()) 1310 OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, Mask)); 1311 } 1312 break; 1313 } 1314 1315 #define MOV_CASE(Prefix, Suffix) \ 1316 case X86::Prefix##MOVAPD##Suffix##rm: \ 1317 case X86::Prefix##MOVAPS##Suffix##rm: \ 1318 case X86::Prefix##MOVUPD##Suffix##rm: \ 1319 case X86::Prefix##MOVUPS##Suffix##rm: \ 1320 case X86::Prefix##MOVDQA##Suffix##rm: \ 1321 case X86::Prefix##MOVDQU##Suffix##rm: 1322 1323 #define MOV_AVX512_CASE(Suffix) \ 1324 case X86::VMOVDQA64##Suffix##rm: \ 1325 case X86::VMOVDQA32##Suffix##rm: \ 1326 case X86::VMOVDQU64##Suffix##rm: \ 1327 case X86::VMOVDQU32##Suffix##rm: \ 1328 case X86::VMOVDQU16##Suffix##rm: \ 1329 case X86::VMOVDQU8##Suffix##rm: \ 1330 case X86::VMOVAPS##Suffix##rm: \ 1331 case X86::VMOVAPD##Suffix##rm: \ 1332 case X86::VMOVUPS##Suffix##rm: \ 1333 case X86::VMOVUPD##Suffix##rm: 1334 1335 #define CASE_ALL_MOV_RM() \ 1336 MOV_CASE(, ) /* SSE */ \ 1337 MOV_CASE(V, ) /* AVX-128 */ \ 1338 MOV_CASE(V, Y) /* AVX-256 */ \ 1339 MOV_AVX512_CASE(Z) \ 1340 MOV_AVX512_CASE(Z256) \ 1341 MOV_AVX512_CASE(Z128) 1342 1343 // For loads from a constant pool to a vector register, print the constant 1344 // loaded. 1345 CASE_ALL_MOV_RM() 1346 if (!OutStreamer->isVerboseAsm()) 1347 break; 1348 if (MI->getNumOperands() > 4) 1349 if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { 1350 std::string Comment; 1351 raw_string_ostream CS(Comment); 1352 const MachineOperand &DstOp = MI->getOperand(0); 1353 CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; 1354 if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) { 1355 CS << "["; 1356 for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements; ++i) { 1357 if (i != 0) 1358 CS << ","; 1359 if (CDS->getElementType()->isIntegerTy()) 1360 CS << CDS->getElementAsInteger(i); 1361 else if (CDS->getElementType()->isFloatTy()) 1362 CS << CDS->getElementAsFloat(i); 1363 else if (CDS->getElementType()->isDoubleTy()) 1364 CS << CDS->getElementAsDouble(i); 1365 else 1366 CS << "?"; 1367 } 1368 CS << "]"; 1369 OutStreamer->AddComment(CS.str()); 1370 } else if (auto *CV = dyn_cast<ConstantVector>(C)) { 1371 CS << "<"; 1372 for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands; ++i) { 1373 if (i != 0) 1374 CS << ","; 1375 Constant *COp = CV->getOperand(i); 1376 if (isa<UndefValue>(COp)) { 1377 CS << "u"; 1378 } else if (auto *CI = dyn_cast<ConstantInt>(COp)) { 1379 if (CI->getBitWidth() <= 64) { 1380 CS << CI->getZExtValue(); 1381 } else { 1382 // print multi-word constant as (w0,w1) 1383 auto Val = CI->getValue(); 1384 CS << "("; 1385 for (int i = 0, N = Val.getNumWords(); i < N; ++i) { 1386 if (i > 0) 1387 CS << ","; 1388 CS << Val.getRawData()[i]; 1389 } 1390 CS << ")"; 1391 } 1392 } else if (auto *CF = dyn_cast<ConstantFP>(COp)) { 1393 SmallString<32> Str; 1394 CF->getValueAPF().toString(Str); 1395 CS << Str; 1396 } else { 1397 CS << "?"; 1398 } 1399 } 1400 CS << ">"; 1401 OutStreamer->AddComment(CS.str()); 1402 } 1403 } 1404 break; 1405 } 1406 1407 MCInst TmpInst; 1408 MCInstLowering.Lower(MI, TmpInst); 1409 1410 // Stackmap shadows cannot include branch targets, so we can count the bytes 1411 // in a call towards the shadow, but must ensure that the no thread returns 1412 // in to the stackmap shadow. The only way to achieve this is if the call 1413 // is at the end of the shadow. 1414 if (MI->isCall()) { 1415 // Count then size of the call towards the shadow 1416 SMShadowTracker.count(TmpInst, getSubtargetInfo()); 1417 // Then flush the shadow so that we fill with nops before the call, not 1418 // after it. 1419 SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); 1420 // Then emit the call 1421 OutStreamer->EmitInstruction(TmpInst, getSubtargetInfo()); 1422 return; 1423 } 1424 1425 EmitAndCountInstruction(TmpInst); 1426 } 1427
