1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines the interfaces that X86 uses to lower LLVM code into a 11 // selection DAG. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86ISelLowering.h" 16 #include "Utils/X86ShuffleDecode.h" 17 #include "X86CallingConv.h" 18 #include "X86FrameLowering.h" 19 #include "X86InstrBuilder.h" 20 #include "X86MachineFunctionInfo.h" 21 #include "X86ShuffleDecodeConstantPool.h" 22 #include "X86TargetMachine.h" 23 #include "X86TargetObjectFile.h" 24 #include "llvm/ADT/SmallBitVector.h" 25 #include "llvm/ADT/SmallSet.h" 26 #include "llvm/ADT/Statistic.h" 27 #include "llvm/ADT/StringExtras.h" 28 #include "llvm/ADT/StringSwitch.h" 29 #include "llvm/Analysis/EHPersonalities.h" 30 #include "llvm/CodeGen/IntrinsicLowering.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstrBuilder.h" 34 #include "llvm/CodeGen/MachineJumpTableInfo.h" 35 #include "llvm/CodeGen/MachineModuleInfo.h" 36 #include "llvm/CodeGen/MachineRegisterInfo.h" 37 #include "llvm/CodeGen/WinEHFuncInfo.h" 38 #include "llvm/IR/CallSite.h" 39 #include "llvm/IR/CallingConv.h" 40 #include "llvm/IR/Constants.h" 41 #include "llvm/IR/DerivedTypes.h" 42 #include "llvm/IR/Function.h" 43 #include "llvm/IR/GlobalAlias.h" 44 #include "llvm/IR/GlobalVariable.h" 45 #include "llvm/IR/Instructions.h" 46 #include "llvm/IR/Intrinsics.h" 47 #include "llvm/MC/MCAsmInfo.h" 48 #include "llvm/MC/MCContext.h" 49 #include "llvm/MC/MCExpr.h" 50 #include "llvm/MC/MCSymbol.h" 51 #include "llvm/Support/CommandLine.h" 52 #include "llvm/Support/Debug.h" 53 #include "llvm/Support/ErrorHandling.h" 54 #include "llvm/Support/MathExtras.h" 55 #include "llvm/Target/TargetOptions.h" 56 #include "X86IntrinsicsInfo.h" 57 #include <bitset> 58 #include <numeric> 59 #include <cctype> 60 using namespace llvm; 61 62 #define DEBUG_TYPE "x86-isel" 63 64 STATISTIC(NumTailCalls, "Number of tail calls"); 65 66 static cl::opt<bool> ExperimentalVectorWideningLegalization( 67 "x86-experimental-vector-widening-legalization", cl::init(false), 68 cl::desc("Enable an experimental vector type legalization through widening " 69 "rather than promotion."), 70 cl::Hidden); 71 72 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, 73 const X86Subtarget &STI) 74 : TargetLowering(TM), Subtarget(STI) { 75 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87(); 76 X86ScalarSSEf64 = Subtarget.hasSSE2(); 77 X86ScalarSSEf32 = Subtarget.hasSSE1(); 78 MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize()); 79 80 // Set up the TargetLowering object. 81 82 // X86 is weird. It always uses i8 for shift amounts and setcc results. 83 setBooleanContents(ZeroOrOneBooleanContent); 84 // X86-SSE is even stranger. It uses -1 or 0 for vector masks. 85 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 86 87 // For 64-bit, since we have so many registers, use the ILP scheduler. 88 // For 32-bit, use the register pressure specific scheduling. 89 // For Atom, always use ILP scheduling. 90 if (Subtarget.isAtom()) 91 setSchedulingPreference(Sched::ILP); 92 else if (Subtarget.is64Bit()) 93 setSchedulingPreference(Sched::ILP); 94 else 95 setSchedulingPreference(Sched::RegPressure); 96 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); 97 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister()); 98 99 // Bypass expensive divides on Atom when compiling with O2. 100 if (TM.getOptLevel() >= CodeGenOpt::Default) { 101 if (Subtarget.hasSlowDivide32()) 102 addBypassSlowDiv(32, 8); 103 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit()) 104 addBypassSlowDiv(64, 16); 105 } 106 107 if (Subtarget.isTargetKnownWindowsMSVC()) { 108 // Setup Windows compiler runtime calls. 109 setLibcallName(RTLIB::SDIV_I64, "_alldiv"); 110 setLibcallName(RTLIB::UDIV_I64, "_aulldiv"); 111 setLibcallName(RTLIB::SREM_I64, "_allrem"); 112 setLibcallName(RTLIB::UREM_I64, "_aullrem"); 113 setLibcallName(RTLIB::MUL_I64, "_allmul"); 114 setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall); 115 setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall); 116 setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall); 117 setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall); 118 setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall); 119 } 120 121 if (Subtarget.isTargetDarwin()) { 122 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 123 setUseUnderscoreSetJmp(false); 124 setUseUnderscoreLongJmp(false); 125 } else if (Subtarget.isTargetWindowsGNU()) { 126 // MS runtime is weird: it exports _setjmp, but longjmp! 127 setUseUnderscoreSetJmp(true); 128 setUseUnderscoreLongJmp(false); 129 } else { 130 setUseUnderscoreSetJmp(true); 131 setUseUnderscoreLongJmp(true); 132 } 133 134 // Set up the register classes. 135 addRegisterClass(MVT::i8, &X86::GR8RegClass); 136 addRegisterClass(MVT::i16, &X86::GR16RegClass); 137 addRegisterClass(MVT::i32, &X86::GR32RegClass); 138 if (Subtarget.is64Bit()) 139 addRegisterClass(MVT::i64, &X86::GR64RegClass); 140 141 for (MVT VT : MVT::integer_valuetypes()) 142 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 143 144 // We don't accept any truncstore of integer registers. 145 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 146 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 147 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 148 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 149 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 150 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 151 152 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 153 154 // SETOEQ and SETUNE require checking two conditions. 155 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); 156 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); 157 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); 158 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); 159 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); 160 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); 161 162 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 163 // operation. 164 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 165 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 166 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 167 168 if (Subtarget.is64Bit()) { 169 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) 170 // f32/f64 are legal, f80 is custom. 171 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); 172 else 173 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 174 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); 175 } else if (!Subtarget.useSoftFloat()) { 176 // We have an algorithm for SSE2->double, and we turn this into a 177 // 64-bit FILD followed by conditional FADD for other targets. 178 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); 179 // We have an algorithm for SSE2, and we turn this into a 64-bit 180 // FILD or VCVTUSI2SS/SD for other targets. 181 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); 182 } 183 184 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 185 // this operation. 186 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 187 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 188 189 if (!Subtarget.useSoftFloat()) { 190 // SSE has no i16 to fp conversion, only i32 191 if (X86ScalarSSEf32) { 192 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 193 // f32 and f64 cases are Legal, f80 case is not 194 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 195 } else { 196 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 197 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 198 } 199 } else { 200 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 201 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); 202 } 203 204 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 205 // this operation. 206 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 207 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 208 209 if (!Subtarget.useSoftFloat()) { 210 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 211 // are Legal, f80 is custom lowered. 212 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 213 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 214 215 if (X86ScalarSSEf32) { 216 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 217 // f32 and f64 cases are Legal, f80 case is not 218 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 219 } else { 220 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 221 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 222 } 223 } else { 224 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 225 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Expand); 226 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Expand); 227 } 228 229 // Handle FP_TO_UINT by promoting the destination to a larger signed 230 // conversion. 231 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 232 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 233 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 234 235 if (Subtarget.is64Bit()) { 236 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) { 237 // FP_TO_UINT-i32/i64 is legal for f32/f64, but custom for f80. 238 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); 239 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); 240 } else { 241 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 242 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 243 } 244 } else if (!Subtarget.useSoftFloat()) { 245 // Since AVX is a superset of SSE3, only check for SSE here. 246 if (Subtarget.hasSSE1() && !Subtarget.hasSSE3()) 247 // Expand FP_TO_UINT into a select. 248 // FIXME: We would like to use a Custom expander here eventually to do 249 // the optimal thing for SSE vs. the default expansion in the legalizer. 250 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 251 else 252 // With AVX512 we can use vcvts[ds]2usi for f32/f64->i32, f80 is custom. 253 // With SSE3 we can use fisttpll to convert to a signed i64; without 254 // SSE, we're stuck with a fistpll. 255 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); 256 257 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); 258 } 259 260 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 261 if (!X86ScalarSSEf64) { 262 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); 263 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); 264 if (Subtarget.is64Bit()) { 265 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); 266 // Without SSE, i64->f64 goes through memory. 267 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); 268 } 269 } else if (!Subtarget.is64Bit()) 270 setOperationAction(ISD::BITCAST , MVT::i64 , Custom); 271 272 // Scalar integer divide and remainder are lowered to use operations that 273 // produce two results, to match the available instructions. This exposes 274 // the two-result form to trivial CSE, which is able to combine x/y and x%y 275 // into a single instruction. 276 // 277 // Scalar integer multiply-high is also lowered to use two-result 278 // operations, to match the available instructions. However, plain multiply 279 // (low) operations are left as Legal, as there are single-result 280 // instructions for this in x86. Using the two-result multiply instructions 281 // when both high and low results are needed must be arranged by dagcombine. 282 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { 283 setOperationAction(ISD::MULHS, VT, Expand); 284 setOperationAction(ISD::MULHU, VT, Expand); 285 setOperationAction(ISD::SDIV, VT, Expand); 286 setOperationAction(ISD::UDIV, VT, Expand); 287 setOperationAction(ISD::SREM, VT, Expand); 288 setOperationAction(ISD::UREM, VT, Expand); 289 290 // Add/Sub overflow ops with MVT::Glues are lowered to EFLAGS dependences. 291 setOperationAction(ISD::ADDC, VT, Custom); 292 setOperationAction(ISD::ADDE, VT, Custom); 293 setOperationAction(ISD::SUBC, VT, Custom); 294 setOperationAction(ISD::SUBE, VT, Custom); 295 } 296 297 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 298 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 299 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128, 300 MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { 301 setOperationAction(ISD::BR_CC, VT, Expand); 302 setOperationAction(ISD::SELECT_CC, VT, Expand); 303 } 304 if (Subtarget.is64Bit()) 305 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 306 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 307 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 308 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 309 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 310 311 setOperationAction(ISD::FREM , MVT::f32 , Expand); 312 setOperationAction(ISD::FREM , MVT::f64 , Expand); 313 setOperationAction(ISD::FREM , MVT::f80 , Expand); 314 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 315 316 // Promote the i8 variants and force them on up to i32 which has a shorter 317 // encoding. 318 setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32); 319 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32); 320 if (!Subtarget.hasBMI()) { 321 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 322 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 323 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Legal); 324 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal); 325 if (Subtarget.is64Bit()) { 326 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 327 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal); 328 } 329 } 330 331 if (Subtarget.hasLZCNT()) { 332 // When promoting the i8 variants, force them to i32 for a shorter 333 // encoding. 334 setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32); 335 setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); 336 } else { 337 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 338 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 339 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 340 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); 341 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); 342 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); 343 if (Subtarget.is64Bit()) { 344 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 345 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); 346 } 347 } 348 349 // Special handling for half-precision floating point conversions. 350 // If we don't have F16C support, then lower half float conversions 351 // into library calls. 352 if (Subtarget.useSoftFloat() || !Subtarget.hasF16C()) { 353 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); 354 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); 355 } 356 357 // There's never any support for operations beyond MVT::f32. 358 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); 359 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand); 360 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); 361 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand); 362 363 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); 364 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); 365 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand); 366 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 367 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 368 setTruncStoreAction(MVT::f80, MVT::f16, Expand); 369 370 if (Subtarget.hasPOPCNT()) { 371 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); 372 } else { 373 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 374 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 375 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 376 if (Subtarget.is64Bit()) 377 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 378 } 379 380 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 381 382 if (!Subtarget.hasMOVBE()) 383 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 384 385 // These should be promoted to a larger select which is supported. 386 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 387 // X86 wants to expand cmov itself. 388 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) { 389 setOperationAction(ISD::SELECT, VT, Custom); 390 setOperationAction(ISD::SETCC, VT, Custom); 391 } 392 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { 393 if (VT == MVT::i64 && !Subtarget.is64Bit()) 394 continue; 395 setOperationAction(ISD::SELECT, VT, Custom); 396 setOperationAction(ISD::SETCC, VT, Custom); 397 setOperationAction(ISD::SETCCE, VT, Custom); 398 } 399 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 400 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support 401 // SjLj exception handling but a light-weight setjmp/longjmp replacement to 402 // support continuation, user-level threading, and etc.. As a result, no 403 // other SjLj exception interfaces are implemented and please don't build 404 // your own exception handling based on them. 405 // LLVM/Clang supports zero-cost DWARF exception handling. 406 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 407 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 408 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom); 409 if (TM.Options.ExceptionModel == ExceptionHandling::SjLj) 410 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume"); 411 412 // Darwin ABI issue. 413 for (auto VT : { MVT::i32, MVT::i64 }) { 414 if (VT == MVT::i64 && !Subtarget.is64Bit()) 415 continue; 416 setOperationAction(ISD::ConstantPool , VT, Custom); 417 setOperationAction(ISD::JumpTable , VT, Custom); 418 setOperationAction(ISD::GlobalAddress , VT, Custom); 419 setOperationAction(ISD::GlobalTLSAddress, VT, Custom); 420 setOperationAction(ISD::ExternalSymbol , VT, Custom); 421 setOperationAction(ISD::BlockAddress , VT, Custom); 422 } 423 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 424 for (auto VT : { MVT::i32, MVT::i64 }) { 425 if (VT == MVT::i64 && !Subtarget.is64Bit()) 426 continue; 427 setOperationAction(ISD::SHL_PARTS, VT, Custom); 428 setOperationAction(ISD::SRA_PARTS, VT, Custom); 429 setOperationAction(ISD::SRL_PARTS, VT, Custom); 430 } 431 432 if (Subtarget.hasSSE1()) 433 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 434 435 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); 436 437 // Expand certain atomics 438 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { 439 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom); 440 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 441 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom); 442 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom); 443 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom); 444 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom); 445 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); 446 } 447 448 if (Subtarget.hasCmpxchg16b()) { 449 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); 450 } 451 452 // FIXME - use subtarget debug flags 453 if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() && 454 !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() && 455 TM.Options.ExceptionModel != ExceptionHandling::SjLj) { 456 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 457 } 458 459 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 460 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); 461 462 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 463 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 464 465 setOperationAction(ISD::TRAP, MVT::Other, Legal); 466 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); 467 468 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 469 setOperationAction(ISD::VASTART , MVT::Other, Custom); 470 setOperationAction(ISD::VAEND , MVT::Other, Expand); 471 bool Is64Bit = Subtarget.is64Bit(); 472 setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand); 473 setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand); 474 475 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 476 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 477 478 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 479 480 // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering. 481 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom); 482 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom); 483 484 if (!Subtarget.useSoftFloat() && X86ScalarSSEf64) { 485 // f32 and f64 use SSE. 486 // Set up the FP register classes. 487 addRegisterClass(MVT::f32, &X86::FR32RegClass); 488 addRegisterClass(MVT::f64, &X86::FR64RegClass); 489 490 for (auto VT : { MVT::f32, MVT::f64 }) { 491 // Use ANDPD to simulate FABS. 492 setOperationAction(ISD::FABS, VT, Custom); 493 494 // Use XORP to simulate FNEG. 495 setOperationAction(ISD::FNEG, VT, Custom); 496 497 // Use ANDPD and ORPD to simulate FCOPYSIGN. 498 setOperationAction(ISD::FCOPYSIGN, VT, Custom); 499 500 // We don't support sin/cos/fmod 501 setOperationAction(ISD::FSIN , VT, Expand); 502 setOperationAction(ISD::FCOS , VT, Expand); 503 setOperationAction(ISD::FSINCOS, VT, Expand); 504 } 505 506 // Lower this to MOVMSK plus an AND. 507 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); 508 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); 509 510 // Expand FP immediates into loads from the stack, except for the special 511 // cases we handle. 512 addLegalFPImmediate(APFloat(+0.0)); // xorpd 513 addLegalFPImmediate(APFloat(+0.0f)); // xorps 514 } else if (UseX87 && X86ScalarSSEf32) { 515 // Use SSE for f32, x87 for f64. 516 // Set up the FP register classes. 517 addRegisterClass(MVT::f32, &X86::FR32RegClass); 518 addRegisterClass(MVT::f64, &X86::RFP64RegClass); 519 520 // Use ANDPS to simulate FABS. 521 setOperationAction(ISD::FABS , MVT::f32, Custom); 522 523 // Use XORP to simulate FNEG. 524 setOperationAction(ISD::FNEG , MVT::f32, Custom); 525 526 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 527 528 // Use ANDPS and ORPS to simulate FCOPYSIGN. 529 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 530 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 531 532 // We don't support sin/cos/fmod 533 setOperationAction(ISD::FSIN , MVT::f32, Expand); 534 setOperationAction(ISD::FCOS , MVT::f32, Expand); 535 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 536 537 // Special cases we handle for FP constants. 538 addLegalFPImmediate(APFloat(+0.0f)); // xorps 539 addLegalFPImmediate(APFloat(+0.0)); // FLD0 540 addLegalFPImmediate(APFloat(+1.0)); // FLD1 541 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 542 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 543 544 if (!TM.Options.UnsafeFPMath) { 545 setOperationAction(ISD::FSIN , MVT::f64, Expand); 546 setOperationAction(ISD::FCOS , MVT::f64, Expand); 547 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 548 } 549 } else if (UseX87) { 550 // f32 and f64 in x87. 551 // Set up the FP register classes. 552 addRegisterClass(MVT::f64, &X86::RFP64RegClass); 553 addRegisterClass(MVT::f32, &X86::RFP32RegClass); 554 555 for (auto VT : { MVT::f32, MVT::f64 }) { 556 setOperationAction(ISD::UNDEF, VT, Expand); 557 setOperationAction(ISD::FCOPYSIGN, VT, Expand); 558 559 if (!TM.Options.UnsafeFPMath) { 560 setOperationAction(ISD::FSIN , VT, Expand); 561 setOperationAction(ISD::FCOS , VT, Expand); 562 setOperationAction(ISD::FSINCOS, VT, Expand); 563 } 564 } 565 addLegalFPImmediate(APFloat(+0.0)); // FLD0 566 addLegalFPImmediate(APFloat(+1.0)); // FLD1 567 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 568 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 569 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 570 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 571 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 572 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 573 } 574 575 // We don't support FMA. 576 setOperationAction(ISD::FMA, MVT::f64, Expand); 577 setOperationAction(ISD::FMA, MVT::f32, Expand); 578 579 // Long double always uses X87, except f128 in MMX. 580 if (UseX87) { 581 if (Subtarget.is64Bit() && Subtarget.hasMMX()) { 582 addRegisterClass(MVT::f128, &X86::FR128RegClass); 583 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); 584 setOperationAction(ISD::FABS , MVT::f128, Custom); 585 setOperationAction(ISD::FNEG , MVT::f128, Custom); 586 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom); 587 } 588 589 addRegisterClass(MVT::f80, &X86::RFP80RegClass); 590 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 591 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 592 { 593 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended); 594 addLegalFPImmediate(TmpFlt); // FLD0 595 TmpFlt.changeSign(); 596 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 597 598 bool ignored; 599 APFloat TmpFlt2(+1.0); 600 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, 601 &ignored); 602 addLegalFPImmediate(TmpFlt2); // FLD1 603 TmpFlt2.changeSign(); 604 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 605 } 606 607 if (!TM.Options.UnsafeFPMath) { 608 setOperationAction(ISD::FSIN , MVT::f80, Expand); 609 setOperationAction(ISD::FCOS , MVT::f80, Expand); 610 setOperationAction(ISD::FSINCOS, MVT::f80, Expand); 611 } 612 613 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); 614 setOperationAction(ISD::FCEIL, MVT::f80, Expand); 615 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); 616 setOperationAction(ISD::FRINT, MVT::f80, Expand); 617 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); 618 setOperationAction(ISD::FMA, MVT::f80, Expand); 619 } 620 621 // Always use a library call for pow. 622 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 623 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 624 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 625 626 setOperationAction(ISD::FLOG, MVT::f80, Expand); 627 setOperationAction(ISD::FLOG2, MVT::f80, Expand); 628 setOperationAction(ISD::FLOG10, MVT::f80, Expand); 629 setOperationAction(ISD::FEXP, MVT::f80, Expand); 630 setOperationAction(ISD::FEXP2, MVT::f80, Expand); 631 setOperationAction(ISD::FMINNUM, MVT::f80, Expand); 632 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand); 633 634 // Some FP actions are always expanded for vector types. 635 for (auto VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32, 636 MVT::v2f64, MVT::v4f64, MVT::v8f64 }) { 637 setOperationAction(ISD::FSIN, VT, Expand); 638 setOperationAction(ISD::FSINCOS, VT, Expand); 639 setOperationAction(ISD::FCOS, VT, Expand); 640 setOperationAction(ISD::FREM, VT, Expand); 641 setOperationAction(ISD::FPOWI, VT, Expand); 642 setOperationAction(ISD::FCOPYSIGN, VT, Expand); 643 setOperationAction(ISD::FPOW, VT, Expand); 644 setOperationAction(ISD::FLOG, VT, Expand); 645 setOperationAction(ISD::FLOG2, VT, Expand); 646 setOperationAction(ISD::FLOG10, VT, Expand); 647 setOperationAction(ISD::FEXP, VT, Expand); 648 setOperationAction(ISD::FEXP2, VT, Expand); 649 } 650 651 // First set operation action for all vector types to either promote 652 // (for widening) or expand (for scalarization). Then we will selectively 653 // turn on ones that can be effectively codegen'd. 654 for (MVT VT : MVT::vector_valuetypes()) { 655 setOperationAction(ISD::SDIV, VT, Expand); 656 setOperationAction(ISD::UDIV, VT, Expand); 657 setOperationAction(ISD::SREM, VT, Expand); 658 setOperationAction(ISD::UREM, VT, Expand); 659 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand); 660 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); 661 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand); 662 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand); 663 setOperationAction(ISD::FMA, VT, Expand); 664 setOperationAction(ISD::FFLOOR, VT, Expand); 665 setOperationAction(ISD::FCEIL, VT, Expand); 666 setOperationAction(ISD::FTRUNC, VT, Expand); 667 setOperationAction(ISD::FRINT, VT, Expand); 668 setOperationAction(ISD::FNEARBYINT, VT, Expand); 669 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 670 setOperationAction(ISD::MULHS, VT, Expand); 671 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 672 setOperationAction(ISD::MULHU, VT, Expand); 673 setOperationAction(ISD::SDIVREM, VT, Expand); 674 setOperationAction(ISD::UDIVREM, VT, Expand); 675 setOperationAction(ISD::CTPOP, VT, Expand); 676 setOperationAction(ISD::CTTZ, VT, Expand); 677 setOperationAction(ISD::CTLZ, VT, Expand); 678 setOperationAction(ISD::ROTL, VT, Expand); 679 setOperationAction(ISD::ROTR, VT, Expand); 680 setOperationAction(ISD::BSWAP, VT, Expand); 681 setOperationAction(ISD::SETCC, VT, Expand); 682 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 683 setOperationAction(ISD::FP_TO_SINT, VT, Expand); 684 setOperationAction(ISD::UINT_TO_FP, VT, Expand); 685 setOperationAction(ISD::SINT_TO_FP, VT, Expand); 686 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand); 687 setOperationAction(ISD::TRUNCATE, VT, Expand); 688 setOperationAction(ISD::SIGN_EXTEND, VT, Expand); 689 setOperationAction(ISD::ZERO_EXTEND, VT, Expand); 690 setOperationAction(ISD::ANY_EXTEND, VT, Expand); 691 setOperationAction(ISD::SELECT_CC, VT, Expand); 692 for (MVT InnerVT : MVT::vector_valuetypes()) { 693 setTruncStoreAction(InnerVT, VT, Expand); 694 695 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand); 696 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand); 697 698 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like 699 // types, we have to deal with them whether we ask for Expansion or not. 700 // Setting Expand causes its own optimisation problems though, so leave 701 // them legal. 702 if (VT.getVectorElementType() == MVT::i1) 703 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); 704 705 // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are 706 // split/scalarized right now. 707 if (VT.getVectorElementType() == MVT::f16) 708 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); 709 } 710 } 711 712 // FIXME: In order to prevent SSE instructions being expanded to MMX ones 713 // with -msoft-float, disable use of MMX as well. 714 if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) { 715 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass); 716 // No operations on x86mmx supported, everything uses intrinsics. 717 } 718 719 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) { 720 addRegisterClass(MVT::v4f32, &X86::VR128RegClass); 721 722 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 723 setOperationAction(ISD::FABS, MVT::v4f32, Custom); 724 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 725 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 726 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom); 727 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 728 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 729 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); 730 } 731 732 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) { 733 addRegisterClass(MVT::v2f64, &X86::VR128RegClass); 734 735 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM 736 // registers cannot be used even for integer operations. 737 addRegisterClass(MVT::v16i8, &X86::VR128RegClass); 738 addRegisterClass(MVT::v8i16, &X86::VR128RegClass); 739 addRegisterClass(MVT::v4i32, &X86::VR128RegClass); 740 addRegisterClass(MVT::v2i64, &X86::VR128RegClass); 741 742 setOperationAction(ISD::MUL, MVT::v16i8, Custom); 743 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 744 setOperationAction(ISD::MUL, MVT::v2i64, Custom); 745 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom); 746 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom); 747 setOperationAction(ISD::MULHU, MVT::v16i8, Custom); 748 setOperationAction(ISD::MULHS, MVT::v16i8, Custom); 749 setOperationAction(ISD::MULHU, MVT::v8i16, Legal); 750 setOperationAction(ISD::MULHS, MVT::v8i16, Legal); 751 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 752 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 753 setOperationAction(ISD::FABS, MVT::v2f64, Custom); 754 755 setOperationAction(ISD::SMAX, MVT::v8i16, Legal); 756 setOperationAction(ISD::UMAX, MVT::v16i8, Legal); 757 setOperationAction(ISD::SMIN, MVT::v8i16, Legal); 758 setOperationAction(ISD::UMIN, MVT::v16i8, Legal); 759 760 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 761 setOperationAction(ISD::SETCC, MVT::v16i8, Custom); 762 setOperationAction(ISD::SETCC, MVT::v8i16, Custom); 763 setOperationAction(ISD::SETCC, MVT::v4i32, Custom); 764 765 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 766 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 767 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 768 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 769 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 770 771 setOperationAction(ISD::CTPOP, MVT::v16i8, Custom); 772 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom); 773 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); 774 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom); 775 776 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom); 777 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom); 778 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom); 779 // ISD::CTTZ v2i64 - scalarization is faster. 780 781 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 782 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { 783 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 784 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 785 setOperationAction(ISD::VSELECT, VT, Custom); 786 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 787 } 788 789 // We support custom legalizing of sext and anyext loads for specific 790 // memory vector types which we can load as a scalar (or sequence of 791 // scalars) and extend in-register to a legal 128-bit vector type. For sext 792 // loads these must work with a single scalar load. 793 for (MVT VT : MVT::integer_vector_valuetypes()) { 794 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom); 795 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom); 796 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom); 797 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom); 798 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom); 799 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom); 800 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom); 801 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom); 802 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom); 803 } 804 805 for (auto VT : { MVT::v2f64, MVT::v2i64 }) { 806 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 807 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 808 setOperationAction(ISD::VSELECT, VT, Custom); 809 810 if (VT == MVT::v2i64 && !Subtarget.is64Bit()) 811 continue; 812 813 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); 814 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 815 } 816 817 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 818 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { 819 setOperationPromotedToType(ISD::AND, VT, MVT::v2i64); 820 setOperationPromotedToType(ISD::OR, VT, MVT::v2i64); 821 setOperationPromotedToType(ISD::XOR, VT, MVT::v2i64); 822 setOperationPromotedToType(ISD::LOAD, VT, MVT::v2i64); 823 setOperationPromotedToType(ISD::SELECT, VT, MVT::v2i64); 824 } 825 826 // Custom lower v2i64 and v2f64 selects. 827 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 828 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 829 830 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 831 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 832 833 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); 834 835 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); 836 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); 837 // As there is no 64-bit GPR available, we need build a special custom 838 // sequence to convert from v2i32 to v2f32. 839 if (!Subtarget.is64Bit()) 840 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom); 841 842 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); 843 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom); 844 845 for (MVT VT : MVT::fp_vector_valuetypes()) 846 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal); 847 848 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom); 849 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom); 850 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom); 851 852 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom); 853 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom); 854 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom); 855 856 for (auto VT : { MVT::v8i16, MVT::v16i8 }) { 857 setOperationAction(ISD::SRL, VT, Custom); 858 setOperationAction(ISD::SHL, VT, Custom); 859 setOperationAction(ISD::SRA, VT, Custom); 860 } 861 862 // In the customized shift lowering, the legal cases in AVX2 will be 863 // recognized. 864 for (auto VT : { MVT::v4i32, MVT::v2i64 }) { 865 setOperationAction(ISD::SRL, VT, Custom); 866 setOperationAction(ISD::SHL, VT, Custom); 867 setOperationAction(ISD::SRA, VT, Custom); 868 } 869 } 870 871 if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) { 872 setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom); 873 setOperationAction(ISD::CTLZ, MVT::v16i8, Custom); 874 setOperationAction(ISD::CTLZ, MVT::v8i16, Custom); 875 // ISD::CTLZ v4i32 - scalarization is faster. 876 // ISD::CTLZ v2i64 - scalarization is faster. 877 } 878 879 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) { 880 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { 881 setOperationAction(ISD::FFLOOR, RoundedTy, Legal); 882 setOperationAction(ISD::FCEIL, RoundedTy, Legal); 883 setOperationAction(ISD::FTRUNC, RoundedTy, Legal); 884 setOperationAction(ISD::FRINT, RoundedTy, Legal); 885 setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal); 886 } 887 888 setOperationAction(ISD::SMAX, MVT::v16i8, Legal); 889 setOperationAction(ISD::SMAX, MVT::v4i32, Legal); 890 setOperationAction(ISD::UMAX, MVT::v8i16, Legal); 891 setOperationAction(ISD::UMAX, MVT::v4i32, Legal); 892 setOperationAction(ISD::SMIN, MVT::v16i8, Legal); 893 setOperationAction(ISD::SMIN, MVT::v4i32, Legal); 894 setOperationAction(ISD::UMIN, MVT::v8i16, Legal); 895 setOperationAction(ISD::UMIN, MVT::v4i32, Legal); 896 897 // FIXME: Do we need to handle scalar-to-vector here? 898 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 899 900 // We directly match byte blends in the backend as they match the VSELECT 901 // condition form. 902 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); 903 904 // SSE41 brings specific instructions for doing vector sign extend even in 905 // cases where we don't have SRA. 906 for (MVT VT : MVT::integer_vector_valuetypes()) { 907 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom); 908 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom); 909 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom); 910 } 911 912 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X 913 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); 914 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); 915 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); 916 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); 917 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); 918 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); 919 920 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); 921 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); 922 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); 923 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); 924 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); 925 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); 926 927 // i8 vectors are custom because the source register and source 928 // source memory operand types are not the same width. 929 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 930 } 931 932 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) { 933 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, 934 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) 935 setOperationAction(ISD::ROTL, VT, Custom); 936 937 // XOP can efficiently perform BITREVERSE with VPPERM. 938 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) 939 setOperationAction(ISD::BITREVERSE, VT, Custom); 940 941 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, 942 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) 943 setOperationAction(ISD::BITREVERSE, VT, Custom); 944 } 945 946 if (!Subtarget.useSoftFloat() && Subtarget.hasFp256()) { 947 bool HasInt256 = Subtarget.hasInt256(); 948 949 addRegisterClass(MVT::v32i8, &X86::VR256RegClass); 950 addRegisterClass(MVT::v16i16, &X86::VR256RegClass); 951 addRegisterClass(MVT::v8i32, &X86::VR256RegClass); 952 addRegisterClass(MVT::v8f32, &X86::VR256RegClass); 953 addRegisterClass(MVT::v4i64, &X86::VR256RegClass); 954 addRegisterClass(MVT::v4f64, &X86::VR256RegClass); 955 956 for (auto VT : { MVT::v8f32, MVT::v4f64 }) { 957 setOperationAction(ISD::FFLOOR, VT, Legal); 958 setOperationAction(ISD::FCEIL, VT, Legal); 959 setOperationAction(ISD::FTRUNC, VT, Legal); 960 setOperationAction(ISD::FRINT, VT, Legal); 961 setOperationAction(ISD::FNEARBYINT, VT, Legal); 962 setOperationAction(ISD::FNEG, VT, Custom); 963 setOperationAction(ISD::FABS, VT, Custom); 964 } 965 966 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted 967 // even though v8i16 is a legal type. 968 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote); 969 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote); 970 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); 971 972 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); 973 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); 974 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); 975 976 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); 977 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); 978 979 for (MVT VT : MVT::fp_vector_valuetypes()) 980 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal); 981 982 for (auto VT : { MVT::v32i8, MVT::v16i16 }) { 983 setOperationAction(ISD::SRL, VT, Custom); 984 setOperationAction(ISD::SHL, VT, Custom); 985 setOperationAction(ISD::SRA, VT, Custom); 986 } 987 988 setOperationAction(ISD::SETCC, MVT::v32i8, Custom); 989 setOperationAction(ISD::SETCC, MVT::v16i16, Custom); 990 setOperationAction(ISD::SETCC, MVT::v8i32, Custom); 991 setOperationAction(ISD::SETCC, MVT::v4i64, Custom); 992 993 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); 994 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); 995 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); 996 997 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom); 998 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); 999 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); 1000 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom); 1001 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); 1002 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); 1003 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom); 1004 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom); 1005 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom); 1006 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); 1007 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom); 1008 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom); 1009 setOperationAction(ISD::BITREVERSE, MVT::v32i8, Custom); 1010 1011 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { 1012 setOperationAction(ISD::CTPOP, VT, Custom); 1013 setOperationAction(ISD::CTTZ, VT, Custom); 1014 } 1015 1016 // ISD::CTLZ v8i32/v4i64 - scalarization is faster without AVX2 1017 // as we end up splitting the 256-bit vectors. 1018 for (auto VT : { MVT::v32i8, MVT::v16i16 }) 1019 setOperationAction(ISD::CTLZ, VT, Custom); 1020 1021 if (HasInt256) 1022 for (auto VT : { MVT::v8i32, MVT::v4i64 }) 1023 setOperationAction(ISD::CTLZ, VT, Custom); 1024 1025 if (Subtarget.hasAnyFMA()) { 1026 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, 1027 MVT::v2f64, MVT::v4f64 }) 1028 setOperationAction(ISD::FMA, VT, Legal); 1029 } 1030 1031 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { 1032 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom); 1033 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom); 1034 } 1035 1036 setOperationAction(ISD::MUL, MVT::v4i64, Custom); 1037 setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom); 1038 setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom); 1039 setOperationAction(ISD::MUL, MVT::v32i8, Custom); 1040 1041 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom); 1042 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom); 1043 1044 setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom); 1045 setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom); 1046 setOperationAction(ISD::MULHU, MVT::v32i8, Custom); 1047 setOperationAction(ISD::MULHS, MVT::v32i8, Custom); 1048 1049 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { 1050 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom); 1051 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom); 1052 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom); 1053 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom); 1054 } 1055 1056 if (HasInt256) { 1057 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i64, Custom); 1058 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i32, Custom); 1059 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v16i16, Custom); 1060 1061 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting 1062 // when we have a 256bit-wide blend with immediate. 1063 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom); 1064 1065 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X 1066 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); 1067 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); 1068 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); 1069 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); 1070 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); 1071 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); 1072 1073 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); 1074 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); 1075 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); 1076 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); 1077 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); 1078 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); 1079 } 1080 1081 // In the customized shift lowering, the legal cases in AVX2 will be 1082 // recognized. 1083 for (auto VT : { MVT::v8i32, MVT::v4i64 }) { 1084 setOperationAction(ISD::SRL, VT, Custom); 1085 setOperationAction(ISD::SHL, VT, Custom); 1086 setOperationAction(ISD::SRA, VT, Custom); 1087 } 1088 1089 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, 1090 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) { 1091 setOperationAction(ISD::MLOAD, VT, Legal); 1092 setOperationAction(ISD::MSTORE, VT, Legal); 1093 } 1094 1095 // Extract subvector is special because the value type 1096 // (result) is 128-bit but the source is 256-bit wide. 1097 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, 1098 MVT::v4f32, MVT::v2f64 }) { 1099 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); 1100 } 1101 1102 // Custom lower several nodes for 256-bit types. 1103 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, 1104 MVT::v8f32, MVT::v4f64 }) { 1105 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 1106 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 1107 setOperationAction(ISD::VSELECT, VT, Custom); 1108 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); 1109 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 1110 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); 1111 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); 1112 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); 1113 } 1114 1115 if (HasInt256) 1116 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); 1117 1118 // Promote v32i8, v16i16, v8i32 select, and, or, xor to v4i64. 1119 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { 1120 setOperationPromotedToType(ISD::AND, VT, MVT::v4i64); 1121 setOperationPromotedToType(ISD::OR, VT, MVT::v4i64); 1122 setOperationPromotedToType(ISD::XOR, VT, MVT::v4i64); 1123 setOperationPromotedToType(ISD::LOAD, VT, MVT::v4i64); 1124 setOperationPromotedToType(ISD::SELECT, VT, MVT::v4i64); 1125 } 1126 } 1127 1128 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) { 1129 addRegisterClass(MVT::v16i32, &X86::VR512RegClass); 1130 addRegisterClass(MVT::v16f32, &X86::VR512RegClass); 1131 addRegisterClass(MVT::v8i64, &X86::VR512RegClass); 1132 addRegisterClass(MVT::v8f64, &X86::VR512RegClass); 1133 1134 addRegisterClass(MVT::i1, &X86::VK1RegClass); 1135 addRegisterClass(MVT::v8i1, &X86::VK8RegClass); 1136 addRegisterClass(MVT::v16i1, &X86::VK16RegClass); 1137 1138 for (MVT VT : MVT::fp_vector_valuetypes()) 1139 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal); 1140 1141 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD, ISD::EXTLOAD}) { 1142 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal); 1143 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal); 1144 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal); 1145 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal); 1146 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal); 1147 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal); 1148 } 1149 setOperationAction(ISD::BR_CC, MVT::i1, Expand); 1150 setOperationAction(ISD::SETCC, MVT::i1, Custom); 1151 setOperationAction(ISD::SETCCE, MVT::i1, Custom); 1152 setOperationAction(ISD::SELECT_CC, MVT::i1, Expand); 1153 setOperationAction(ISD::XOR, MVT::i1, Legal); 1154 setOperationAction(ISD::OR, MVT::i1, Legal); 1155 setOperationAction(ISD::AND, MVT::i1, Legal); 1156 setOperationAction(ISD::SUB, MVT::i1, Custom); 1157 setOperationAction(ISD::ADD, MVT::i1, Custom); 1158 setOperationAction(ISD::MUL, MVT::i1, Custom); 1159 1160 for (MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i32, MVT::v4i64, MVT::v8i16, 1161 MVT::v16i8, MVT::v16i16, MVT::v32i8, MVT::v16i32, 1162 MVT::v8i64, MVT::v32i16, MVT::v64i8}) { 1163 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); 1164 setLoadExtAction(ISD::SEXTLOAD, VT, MaskVT, Custom); 1165 setLoadExtAction(ISD::ZEXTLOAD, VT, MaskVT, Custom); 1166 setLoadExtAction(ISD::EXTLOAD, VT, MaskVT, Custom); 1167 setTruncStoreAction(VT, MaskVT, Custom); 1168 } 1169 1170 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) { 1171 setOperationAction(ISD::FNEG, VT, Custom); 1172 setOperationAction(ISD::FABS, VT, Custom); 1173 setOperationAction(ISD::FMA, VT, Legal); 1174 } 1175 1176 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal); 1177 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal); 1178 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); 1179 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 1180 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal); 1181 setOperationAction(ISD::SINT_TO_FP, MVT::v8i1, Custom); 1182 setOperationAction(ISD::SINT_TO_FP, MVT::v16i1, Custom); 1183 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Promote); 1184 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Promote); 1185 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal); 1186 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); 1187 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 1188 setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom); 1189 setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Custom); 1190 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal); 1191 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal); 1192 1193 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal); 1194 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal); 1195 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal); 1196 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal); 1197 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal); 1198 if (Subtarget.hasVLX()){ 1199 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal); 1200 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal); 1201 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal); 1202 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal); 1203 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal); 1204 1205 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal); 1206 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal); 1207 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal); 1208 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal); 1209 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal); 1210 } else { 1211 setOperationAction(ISD::MLOAD, MVT::v8i32, Custom); 1212 setOperationAction(ISD::MLOAD, MVT::v8f32, Custom); 1213 setOperationAction(ISD::MSTORE, MVT::v8i32, Custom); 1214 setOperationAction(ISD::MSTORE, MVT::v8f32, Custom); 1215 } 1216 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); 1217 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); 1218 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); 1219 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i1, Custom); 1220 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i1, Custom); 1221 setOperationAction(ISD::VSELECT, MVT::v8i1, Expand); 1222 setOperationAction(ISD::VSELECT, MVT::v16i1, Expand); 1223 if (Subtarget.hasDQI()) { 1224 setOperationAction(ISD::SINT_TO_FP, MVT::v8i64, Legal); 1225 setOperationAction(ISD::UINT_TO_FP, MVT::v8i64, Legal); 1226 setOperationAction(ISD::FP_TO_SINT, MVT::v8i64, Legal); 1227 setOperationAction(ISD::FP_TO_UINT, MVT::v8i64, Legal); 1228 if (Subtarget.hasVLX()) { 1229 setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Legal); 1230 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 1231 setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Legal); 1232 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 1233 setOperationAction(ISD::FP_TO_SINT, MVT::v4i64, Legal); 1234 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 1235 setOperationAction(ISD::FP_TO_UINT, MVT::v4i64, Legal); 1236 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 1237 } 1238 } 1239 if (Subtarget.hasVLX()) { 1240 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); 1241 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); 1242 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); 1243 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); 1244 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 1245 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 1246 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 1247 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 1248 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i32, Custom); 1249 setOperationAction(ISD::ZERO_EXTEND, MVT::v2i64, Custom); 1250 1251 // FIXME. This commands are available on SSE/AVX2, add relevant patterns. 1252 setLoadExtAction(ISD::EXTLOAD, MVT::v8i32, MVT::v8i8, Legal); 1253 setLoadExtAction(ISD::EXTLOAD, MVT::v8i32, MVT::v8i16, Legal); 1254 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i8, Legal); 1255 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i16, Legal); 1256 setLoadExtAction(ISD::EXTLOAD, MVT::v4i64, MVT::v4i8, Legal); 1257 setLoadExtAction(ISD::EXTLOAD, MVT::v4i64, MVT::v4i16, Legal); 1258 setLoadExtAction(ISD::EXTLOAD, MVT::v4i64, MVT::v4i32, Legal); 1259 setLoadExtAction(ISD::EXTLOAD, MVT::v2i64, MVT::v2i8, Legal); 1260 setLoadExtAction(ISD::EXTLOAD, MVT::v2i64, MVT::v2i16, Legal); 1261 setLoadExtAction(ISD::EXTLOAD, MVT::v2i64, MVT::v2i32, Legal); 1262 } 1263 1264 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom); 1265 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom); 1266 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); 1267 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); 1268 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); 1269 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom); 1270 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom); 1271 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); 1272 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); 1273 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom); 1274 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom); 1275 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); 1276 if (Subtarget.hasDQI()) { 1277 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Custom); 1278 setOperationAction(ISD::SIGN_EXTEND, MVT::v2i64, Custom); 1279 } 1280 for (auto VT : { MVT::v16f32, MVT::v8f64 }) { 1281 setOperationAction(ISD::FFLOOR, VT, Legal); 1282 setOperationAction(ISD::FCEIL, VT, Legal); 1283 setOperationAction(ISD::FTRUNC, VT, Legal); 1284 setOperationAction(ISD::FRINT, VT, Legal); 1285 setOperationAction(ISD::FNEARBYINT, VT, Legal); 1286 } 1287 1288 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom); 1289 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom); 1290 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); 1291 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); 1292 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Custom); 1293 1294 setOperationAction(ISD::SETCC, MVT::v16i1, Custom); 1295 setOperationAction(ISD::SETCC, MVT::v8i1, Custom); 1296 1297 setOperationAction(ISD::MUL, MVT::v8i64, Custom); 1298 1299 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom); 1300 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom); 1301 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16i1, Custom); 1302 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom); 1303 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom); 1304 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom); 1305 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom); 1306 setOperationAction(ISD::SELECT, MVT::v8f64, Custom); 1307 setOperationAction(ISD::SELECT, MVT::v8i64, Custom); 1308 setOperationAction(ISD::SELECT, MVT::v16f32, Custom); 1309 setOperationAction(ISD::SELECT, MVT::v16i1, Custom); 1310 setOperationAction(ISD::SELECT, MVT::v8i1, Custom); 1311 1312 setOperationAction(ISD::SMAX, MVT::v16i32, Legal); 1313 setOperationAction(ISD::SMAX, MVT::v8i64, Legal); 1314 setOperationAction(ISD::UMAX, MVT::v16i32, Legal); 1315 setOperationAction(ISD::UMAX, MVT::v8i64, Legal); 1316 setOperationAction(ISD::SMIN, MVT::v16i32, Legal); 1317 setOperationAction(ISD::SMIN, MVT::v8i64, Legal); 1318 setOperationAction(ISD::UMIN, MVT::v16i32, Legal); 1319 setOperationAction(ISD::UMIN, MVT::v8i64, Legal); 1320 1321 setOperationAction(ISD::ADD, MVT::v8i1, Expand); 1322 setOperationAction(ISD::ADD, MVT::v16i1, Expand); 1323 setOperationAction(ISD::SUB, MVT::v8i1, Expand); 1324 setOperationAction(ISD::SUB, MVT::v16i1, Expand); 1325 setOperationAction(ISD::MUL, MVT::v8i1, Expand); 1326 setOperationAction(ISD::MUL, MVT::v16i1, Expand); 1327 1328 setOperationAction(ISD::MUL, MVT::v16i32, Legal); 1329 1330 for (auto VT : { MVT::v16i32, MVT::v8i64 }) { 1331 setOperationAction(ISD::SRL, VT, Custom); 1332 setOperationAction(ISD::SHL, VT, Custom); 1333 setOperationAction(ISD::SRA, VT, Custom); 1334 setOperationAction(ISD::AND, VT, Legal); 1335 setOperationAction(ISD::OR, VT, Legal); 1336 setOperationAction(ISD::XOR, VT, Legal); 1337 setOperationAction(ISD::CTPOP, VT, Custom); 1338 setOperationAction(ISD::CTTZ, VT, Custom); 1339 } 1340 1341 if (Subtarget.hasCDI()) { 1342 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal); 1343 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal); 1344 1345 setOperationAction(ISD::CTLZ, MVT::v8i16, Custom); 1346 setOperationAction(ISD::CTLZ, MVT::v16i8, Custom); 1347 setOperationAction(ISD::CTLZ, MVT::v16i16, Custom); 1348 setOperationAction(ISD::CTLZ, MVT::v32i8, Custom); 1349 1350 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i64, Custom); 1351 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i32, Custom); 1352 1353 if (Subtarget.hasVLX()) { 1354 setOperationAction(ISD::CTLZ, MVT::v4i64, Legal); 1355 setOperationAction(ISD::CTLZ, MVT::v8i32, Legal); 1356 setOperationAction(ISD::CTLZ, MVT::v2i64, Legal); 1357 setOperationAction(ISD::CTLZ, MVT::v4i32, Legal); 1358 } else { 1359 setOperationAction(ISD::CTLZ, MVT::v4i64, Custom); 1360 setOperationAction(ISD::CTLZ, MVT::v8i32, Custom); 1361 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom); 1362 setOperationAction(ISD::CTLZ, MVT::v4i32, Custom); 1363 } 1364 1365 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i64, Custom); 1366 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i32, Custom); 1367 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom); 1368 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); 1369 } // Subtarget.hasCDI() 1370 1371 if (Subtarget.hasDQI()) { 1372 if (Subtarget.hasVLX()) { 1373 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 1374 setOperationAction(ISD::MUL, MVT::v4i64, Legal); 1375 } 1376 setOperationAction(ISD::MUL, MVT::v8i64, Legal); 1377 } 1378 // Custom lower several nodes. 1379 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, 1380 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) { 1381 setOperationAction(ISD::MGATHER, VT, Custom); 1382 setOperationAction(ISD::MSCATTER, VT, Custom); 1383 } 1384 // Extract subvector is special because the value type 1385 // (result) is 256-bit but the source is 512-bit wide. 1386 // 128-bit was made Custom under AVX1. 1387 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, 1388 MVT::v8f32, MVT::v4f64 }) 1389 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); 1390 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, 1391 MVT::v16i1, MVT::v32i1, MVT::v64i1 }) 1392 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); 1393 1394 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) { 1395 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 1396 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); 1397 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 1398 setOperationAction(ISD::VSELECT, VT, Legal); 1399 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 1400 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); 1401 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); 1402 setOperationAction(ISD::MLOAD, VT, Legal); 1403 setOperationAction(ISD::MSTORE, VT, Legal); 1404 setOperationAction(ISD::MGATHER, VT, Legal); 1405 setOperationAction(ISD::MSCATTER, VT, Custom); 1406 } 1407 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32 }) { 1408 setOperationPromotedToType(ISD::SELECT, VT, MVT::v8i64); 1409 } 1410 }// has AVX-512 1411 1412 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) { 1413 addRegisterClass(MVT::v32i16, &X86::VR512RegClass); 1414 addRegisterClass(MVT::v64i8, &X86::VR512RegClass); 1415 1416 addRegisterClass(MVT::v32i1, &X86::VK32RegClass); 1417 addRegisterClass(MVT::v64i1, &X86::VK64RegClass); 1418 1419 setOperationAction(ISD::ADD, MVT::v32i1, Expand); 1420 setOperationAction(ISD::ADD, MVT::v64i1, Expand); 1421 setOperationAction(ISD::SUB, MVT::v32i1, Expand); 1422 setOperationAction(ISD::SUB, MVT::v64i1, Expand); 1423 setOperationAction(ISD::MUL, MVT::v32i1, Expand); 1424 setOperationAction(ISD::MUL, MVT::v64i1, Expand); 1425 1426 setOperationAction(ISD::SETCC, MVT::v32i1, Custom); 1427 setOperationAction(ISD::SETCC, MVT::v64i1, Custom); 1428 setOperationAction(ISD::MUL, MVT::v32i16, Legal); 1429 setOperationAction(ISD::MUL, MVT::v64i8, Custom); 1430 setOperationAction(ISD::MULHS, MVT::v32i16, Legal); 1431 setOperationAction(ISD::MULHU, MVT::v32i16, Legal); 1432 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom); 1433 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom); 1434 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i16, Custom); 1435 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i8, Custom); 1436 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom); 1437 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom); 1438 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i16, Custom); 1439 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i8, Custom); 1440 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v32i16, Custom); 1441 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v64i8, Custom); 1442 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32i16, Custom); 1443 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v64i8, Custom); 1444 setOperationAction(ISD::SELECT, MVT::v32i1, Custom); 1445 setOperationAction(ISD::SELECT, MVT::v64i1, Custom); 1446 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom); 1447 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom); 1448 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom); 1449 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom); 1450 setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom); 1451 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i16, Custom); 1452 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i8, Custom); 1453 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom); 1454 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom); 1455 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i1, Custom); 1456 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i1, Custom); 1457 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i16, Custom); 1458 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i8, Custom); 1459 setOperationAction(ISD::VSELECT, MVT::v32i16, Legal); 1460 setOperationAction(ISD::VSELECT, MVT::v64i8, Legal); 1461 setOperationAction(ISD::TRUNCATE, MVT::v32i1, Custom); 1462 setOperationAction(ISD::TRUNCATE, MVT::v64i1, Custom); 1463 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom); 1464 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i1, Custom); 1465 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i1, Custom); 1466 setOperationAction(ISD::BUILD_VECTOR, MVT::v32i1, Custom); 1467 setOperationAction(ISD::BUILD_VECTOR, MVT::v64i1, Custom); 1468 setOperationAction(ISD::VSELECT, MVT::v32i1, Expand); 1469 setOperationAction(ISD::VSELECT, MVT::v64i1, Expand); 1470 setOperationAction(ISD::BITREVERSE, MVT::v64i8, Custom); 1471 1472 setOperationAction(ISD::SMAX, MVT::v64i8, Legal); 1473 setOperationAction(ISD::SMAX, MVT::v32i16, Legal); 1474 setOperationAction(ISD::UMAX, MVT::v64i8, Legal); 1475 setOperationAction(ISD::UMAX, MVT::v32i16, Legal); 1476 setOperationAction(ISD::SMIN, MVT::v64i8, Legal); 1477 setOperationAction(ISD::SMIN, MVT::v32i16, Legal); 1478 setOperationAction(ISD::UMIN, MVT::v64i8, Legal); 1479 setOperationAction(ISD::UMIN, MVT::v32i16, Legal); 1480 1481 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal); 1482 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal); 1483 if (Subtarget.hasVLX()) 1484 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal); 1485 1486 LegalizeAction Action = Subtarget.hasVLX() ? Legal : Custom; 1487 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) { 1488 setOperationAction(ISD::MLOAD, VT, Action); 1489 setOperationAction(ISD::MSTORE, VT, Action); 1490 } 1491 1492 if (Subtarget.hasCDI()) { 1493 setOperationAction(ISD::CTLZ, MVT::v32i16, Custom); 1494 setOperationAction(ISD::CTLZ, MVT::v64i8, Custom); 1495 } 1496 1497 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { 1498 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 1499 setOperationAction(ISD::VSELECT, VT, Legal); 1500 setOperationAction(ISD::SRL, VT, Custom); 1501 setOperationAction(ISD::SHL, VT, Custom); 1502 setOperationAction(ISD::SRA, VT, Custom); 1503 setOperationAction(ISD::MLOAD, VT, Legal); 1504 setOperationAction(ISD::MSTORE, VT, Legal); 1505 setOperationAction(ISD::CTPOP, VT, Custom); 1506 setOperationAction(ISD::CTTZ, VT, Custom); 1507 1508 setOperationPromotedToType(ISD::AND, VT, MVT::v8i64); 1509 setOperationPromotedToType(ISD::OR, VT, MVT::v8i64); 1510 setOperationPromotedToType(ISD::XOR, VT, MVT::v8i64); 1511 } 1512 1513 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD, ISD::EXTLOAD}) { 1514 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal); 1515 if (Subtarget.hasVLX()) { 1516 // FIXME. This commands are available on SSE/AVX2, add relevant patterns. 1517 setLoadExtAction(ExtType, MVT::v16i16, MVT::v16i8, Legal); 1518 setLoadExtAction(ExtType, MVT::v8i16, MVT::v8i8, Legal); 1519 } 1520 } 1521 } 1522 1523 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) { 1524 addRegisterClass(MVT::v4i1, &X86::VK4RegClass); 1525 addRegisterClass(MVT::v2i1, &X86::VK2RegClass); 1526 1527 setOperationAction(ISD::ADD, MVT::v2i1, Expand); 1528 setOperationAction(ISD::ADD, MVT::v4i1, Expand); 1529 setOperationAction(ISD::SUB, MVT::v2i1, Expand); 1530 setOperationAction(ISD::SUB, MVT::v4i1, Expand); 1531 setOperationAction(ISD::MUL, MVT::v2i1, Expand); 1532 setOperationAction(ISD::MUL, MVT::v4i1, Expand); 1533 1534 setOperationAction(ISD::TRUNCATE, MVT::v2i1, Custom); 1535 setOperationAction(ISD::TRUNCATE, MVT::v4i1, Custom); 1536 setOperationAction(ISD::SETCC, MVT::v4i1, Custom); 1537 setOperationAction(ISD::SETCC, MVT::v2i1, Custom); 1538 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i1, Custom); 1539 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); 1540 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Custom); 1541 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i1, Custom); 1542 setOperationAction(ISD::SELECT, MVT::v4i1, Custom); 1543 setOperationAction(ISD::SELECT, MVT::v2i1, Custom); 1544 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); 1545 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i1, Custom); 1546 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i1, Custom); 1547 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i1, Custom); 1548 setOperationAction(ISD::VSELECT, MVT::v2i1, Expand); 1549 setOperationAction(ISD::VSELECT, MVT::v4i1, Expand); 1550 1551 for (auto VT : { MVT::v4i32, MVT::v8i32 }) { 1552 setOperationAction(ISD::AND, VT, Legal); 1553 setOperationAction(ISD::OR, VT, Legal); 1554 setOperationAction(ISD::XOR, VT, Legal); 1555 } 1556 1557 for (auto VT : { MVT::v2i64, MVT::v4i64 }) { 1558 setOperationAction(ISD::SMAX, VT, Legal); 1559 setOperationAction(ISD::UMAX, VT, Legal); 1560 setOperationAction(ISD::SMIN, VT, Legal); 1561 setOperationAction(ISD::UMIN, VT, Legal); 1562 } 1563 } 1564 1565 // We want to custom lower some of our intrinsics. 1566 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 1567 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 1568 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 1569 if (!Subtarget.is64Bit()) { 1570 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); 1571 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); 1572 } 1573 1574 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't 1575 // handle type legalization for these operations here. 1576 // 1577 // FIXME: We really should do custom legalization for addition and 1578 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better 1579 // than generic legalization for 64-bit multiplication-with-overflow, though. 1580 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { 1581 if (VT == MVT::i64 && !Subtarget.is64Bit()) 1582 continue; 1583 // Add/Sub/Mul with overflow operations are custom lowered. 1584 setOperationAction(ISD::SADDO, VT, Custom); 1585 setOperationAction(ISD::UADDO, VT, Custom); 1586 setOperationAction(ISD::SSUBO, VT, Custom); 1587 setOperationAction(ISD::USUBO, VT, Custom); 1588 setOperationAction(ISD::SMULO, VT, Custom); 1589 setOperationAction(ISD::UMULO, VT, Custom); 1590 } 1591 1592 if (!Subtarget.is64Bit()) { 1593 // These libcalls are not available in 32-bit. 1594 setLibcallName(RTLIB::SHL_I128, nullptr); 1595 setLibcallName(RTLIB::SRL_I128, nullptr); 1596 setLibcallName(RTLIB::SRA_I128, nullptr); 1597 } 1598 1599 // Combine sin / cos into one node or libcall if possible. 1600 if (Subtarget.hasSinCos()) { 1601 setLibcallName(RTLIB::SINCOS_F32, "sincosf"); 1602 setLibcallName(RTLIB::SINCOS_F64, "sincos"); 1603 if (Subtarget.isTargetDarwin()) { 1604 // For MacOSX, we don't want the normal expansion of a libcall to sincos. 1605 // We want to issue a libcall to __sincos_stret to avoid memory traffic. 1606 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); 1607 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); 1608 } 1609 } 1610 1611 if (Subtarget.isTargetWin64()) { 1612 setOperationAction(ISD::SDIV, MVT::i128, Custom); 1613 setOperationAction(ISD::UDIV, MVT::i128, Custom); 1614 setOperationAction(ISD::SREM, MVT::i128, Custom); 1615 setOperationAction(ISD::UREM, MVT::i128, Custom); 1616 setOperationAction(ISD::SDIVREM, MVT::i128, Custom); 1617 setOperationAction(ISD::UDIVREM, MVT::i128, Custom); 1618 } 1619 1620 // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)` 1621 // is. We should promote the value to 64-bits to solve this. 1622 // This is what the CRT headers do - `fmodf` is an inline header 1623 // function casting to f64 and calling `fmod`. 1624 if (Subtarget.is32Bit() && Subtarget.isTargetKnownWindowsMSVC()) 1625 for (ISD::NodeType Op : 1626 {ISD::FCEIL, ISD::FCOS, ISD::FEXP, ISD::FFLOOR, ISD::FREM, ISD::FLOG, 1627 ISD::FLOG10, ISD::FPOW, ISD::FSIN}) 1628 if (isOperationExpand(Op, MVT::f32)) 1629 setOperationAction(Op, MVT::f32, Promote); 1630 1631 // We have target-specific dag combine patterns for the following nodes: 1632 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 1633 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 1634 setTargetDAGCombine(ISD::BITCAST); 1635 setTargetDAGCombine(ISD::VSELECT); 1636 setTargetDAGCombine(ISD::SELECT); 1637 setTargetDAGCombine(ISD::SHL); 1638 setTargetDAGCombine(ISD::SRA); 1639 setTargetDAGCombine(ISD::SRL); 1640 setTargetDAGCombine(ISD::OR); 1641 setTargetDAGCombine(ISD::AND); 1642 setTargetDAGCombine(ISD::ADD); 1643 setTargetDAGCombine(ISD::FADD); 1644 setTargetDAGCombine(ISD::FSUB); 1645 setTargetDAGCombine(ISD::FNEG); 1646 setTargetDAGCombine(ISD::FMA); 1647 setTargetDAGCombine(ISD::FMINNUM); 1648 setTargetDAGCombine(ISD::FMAXNUM); 1649 setTargetDAGCombine(ISD::SUB); 1650 setTargetDAGCombine(ISD::LOAD); 1651 setTargetDAGCombine(ISD::MLOAD); 1652 setTargetDAGCombine(ISD::STORE); 1653 setTargetDAGCombine(ISD::MSTORE); 1654 setTargetDAGCombine(ISD::TRUNCATE); 1655 setTargetDAGCombine(ISD::ZERO_EXTEND); 1656 setTargetDAGCombine(ISD::ANY_EXTEND); 1657 setTargetDAGCombine(ISD::SIGN_EXTEND); 1658 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); 1659 setTargetDAGCombine(ISD::SINT_TO_FP); 1660 setTargetDAGCombine(ISD::UINT_TO_FP); 1661 setTargetDAGCombine(ISD::SETCC); 1662 setTargetDAGCombine(ISD::MUL); 1663 setTargetDAGCombine(ISD::XOR); 1664 setTargetDAGCombine(ISD::MSCATTER); 1665 setTargetDAGCombine(ISD::MGATHER); 1666 1667 computeRegisterProperties(Subtarget.getRegisterInfo()); 1668 1669 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 1670 MaxStoresPerMemsetOptSize = 8; 1671 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores 1672 MaxStoresPerMemcpyOptSize = 4; 1673 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores 1674 MaxStoresPerMemmoveOptSize = 4; 1675 setPrefLoopAlignment(4); // 2^4 bytes. 1676 1677 // An out-of-order CPU can speculatively execute past a predictable branch, 1678 // but a conditional move could be stalled by an expensive earlier operation. 1679 PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder(); 1680 EnableExtLdPromotion = true; 1681 setPrefFunctionAlignment(4); // 2^4 bytes. 1682 1683 verifyIntrinsicTables(); 1684 } 1685 1686 // This has so far only been implemented for 64-bit MachO. 1687 bool X86TargetLowering::useLoadStackGuardNode() const { 1688 return Subtarget.isTargetMachO() && Subtarget.is64Bit(); 1689 } 1690 1691 TargetLoweringBase::LegalizeTypeAction 1692 X86TargetLowering::getPreferredVectorAction(EVT VT) const { 1693 if (ExperimentalVectorWideningLegalization && 1694 VT.getVectorNumElements() != 1 && 1695 VT.getVectorElementType().getSimpleVT() != MVT::i1) 1696 return TypeWidenVector; 1697 1698 return TargetLoweringBase::getPreferredVectorAction(VT); 1699 } 1700 1701 EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL, 1702 LLVMContext& Context, 1703 EVT VT) const { 1704 if (!VT.isVector()) 1705 return Subtarget.hasAVX512() ? MVT::i1: MVT::i8; 1706 1707 if (VT.isSimple()) { 1708 MVT VVT = VT.getSimpleVT(); 1709 const unsigned NumElts = VVT.getVectorNumElements(); 1710 MVT EltVT = VVT.getVectorElementType(); 1711 if (VVT.is512BitVector()) { 1712 if (Subtarget.hasAVX512()) 1713 if (EltVT == MVT::i32 || EltVT == MVT::i64 || 1714 EltVT == MVT::f32 || EltVT == MVT::f64) 1715 switch(NumElts) { 1716 case 8: return MVT::v8i1; 1717 case 16: return MVT::v16i1; 1718 } 1719 if (Subtarget.hasBWI()) 1720 if (EltVT == MVT::i8 || EltVT == MVT::i16) 1721 switch(NumElts) { 1722 case 32: return MVT::v32i1; 1723 case 64: return MVT::v64i1; 1724 } 1725 } 1726 1727 if (Subtarget.hasBWI() && Subtarget.hasVLX()) 1728 return MVT::getVectorVT(MVT::i1, NumElts); 1729 1730 if (!isTypeLegal(VT) && getTypeAction(Context, VT) == TypePromoteInteger) { 1731 EVT LegalVT = getTypeToTransformTo(Context, VT); 1732 EltVT = LegalVT.getVectorElementType().getSimpleVT(); 1733 } 1734 1735 if (Subtarget.hasVLX() && EltVT.getSizeInBits() >= 32) 1736 switch(NumElts) { 1737 case 2: return MVT::v2i1; 1738 case 4: return MVT::v4i1; 1739 case 8: return MVT::v8i1; 1740 } 1741 } 1742 1743 return VT.changeVectorElementTypeToInteger(); 1744 } 1745 1746 /// Helper for getByValTypeAlignment to determine 1747 /// the desired ByVal argument alignment. 1748 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) { 1749 if (MaxAlign == 16) 1750 return; 1751 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 1752 if (VTy->getBitWidth() == 128) 1753 MaxAlign = 16; 1754 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 1755 unsigned EltAlign = 0; 1756 getMaxByValAlign(ATy->getElementType(), EltAlign); 1757 if (EltAlign > MaxAlign) 1758 MaxAlign = EltAlign; 1759 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 1760 for (auto *EltTy : STy->elements()) { 1761 unsigned EltAlign = 0; 1762 getMaxByValAlign(EltTy, EltAlign); 1763 if (EltAlign > MaxAlign) 1764 MaxAlign = EltAlign; 1765 if (MaxAlign == 16) 1766 break; 1767 } 1768 } 1769 } 1770 1771 /// Return the desired alignment for ByVal aggregate 1772 /// function arguments in the caller parameter area. For X86, aggregates 1773 /// that contain SSE vectors are placed at 16-byte boundaries while the rest 1774 /// are at 4-byte boundaries. 1775 unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty, 1776 const DataLayout &DL) const { 1777 if (Subtarget.is64Bit()) { 1778 // Max of 8 and alignment of type. 1779 unsigned TyAlign = DL.getABITypeAlignment(Ty); 1780 if (TyAlign > 8) 1781 return TyAlign; 1782 return 8; 1783 } 1784 1785 unsigned Align = 4; 1786 if (Subtarget.hasSSE1()) 1787 getMaxByValAlign(Ty, Align); 1788 return Align; 1789 } 1790 1791 /// Returns the target specific optimal type for load 1792 /// and store operations as a result of memset, memcpy, and memmove 1793 /// lowering. If DstAlign is zero that means it's safe to destination 1794 /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 1795 /// means there isn't a need to check it against alignment requirement, 1796 /// probably because the source does not need to be loaded. If 'IsMemset' is 1797 /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that 1798 /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy 1799 /// source is constant so it does not need to be loaded. 1800 /// It returns EVT::Other if the type should be determined using generic 1801 /// target-independent logic. 1802 EVT 1803 X86TargetLowering::getOptimalMemOpType(uint64_t Size, 1804 unsigned DstAlign, unsigned SrcAlign, 1805 bool IsMemset, bool ZeroMemset, 1806 bool MemcpyStrSrc, 1807 MachineFunction &MF) const { 1808 const Function *F = MF.getFunction(); 1809 if (!F->hasFnAttribute(Attribute::NoImplicitFloat)) { 1810 if (Size >= 16 && 1811 (!Subtarget.isUnalignedMem16Slow() || 1812 ((DstAlign == 0 || DstAlign >= 16) && 1813 (SrcAlign == 0 || SrcAlign >= 16)))) { 1814 // FIXME: Check if unaligned 32-byte accesses are slow. 1815 if (Size >= 32 && Subtarget.hasAVX()) { 1816 // Although this isn't a well-supported type for AVX1, we'll let 1817 // legalization and shuffle lowering produce the optimal codegen. If we 1818 // choose an optimal type with a vector element larger than a byte, 1819 // getMemsetStores() may create an intermediate splat (using an integer 1820 // multiply) before we splat as a vector. 1821 return MVT::v32i8; 1822 } 1823 if (Subtarget.hasSSE2()) 1824 return MVT::v16i8; 1825 // TODO: Can SSE1 handle a byte vector? 1826 if (Subtarget.hasSSE1()) 1827 return MVT::v4f32; 1828 } else if ((!IsMemset || ZeroMemset) && !MemcpyStrSrc && Size >= 8 && 1829 !Subtarget.is64Bit() && Subtarget.hasSSE2()) { 1830 // Do not use f64 to lower memcpy if source is string constant. It's 1831 // better to use i32 to avoid the loads. 1832 // Also, do not use f64 to lower memset unless this is a memset of zeros. 1833 // The gymnastics of splatting a byte value into an XMM register and then 1834 // only using 8-byte stores (because this is a CPU with slow unaligned 1835 // 16-byte accesses) makes that a loser. 1836 return MVT::f64; 1837 } 1838 } 1839 // This is a compromise. If we reach here, unaligned accesses may be slow on 1840 // this target. However, creating smaller, aligned accesses could be even 1841 // slower and would certainly be a lot more code. 1842 if (Subtarget.is64Bit() && Size >= 8) 1843 return MVT::i64; 1844 return MVT::i32; 1845 } 1846 1847 bool X86TargetLowering::isSafeMemOpType(MVT VT) const { 1848 if (VT == MVT::f32) 1849 return X86ScalarSSEf32; 1850 else if (VT == MVT::f64) 1851 return X86ScalarSSEf64; 1852 return true; 1853 } 1854 1855 bool 1856 X86TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 1857 unsigned, 1858 unsigned, 1859 bool *Fast) const { 1860 if (Fast) { 1861 switch (VT.getSizeInBits()) { 1862 default: 1863 // 8-byte and under are always assumed to be fast. 1864 *Fast = true; 1865 break; 1866 case 128: 1867 *Fast = !Subtarget.isUnalignedMem16Slow(); 1868 break; 1869 case 256: 1870 *Fast = !Subtarget.isUnalignedMem32Slow(); 1871 break; 1872 // TODO: What about AVX-512 (512-bit) accesses? 1873 } 1874 } 1875 // Misaligned accesses of any size are always allowed. 1876 return true; 1877 } 1878 1879 /// Return the entry encoding for a jump table in the 1880 /// current function. The returned value is a member of the 1881 /// MachineJumpTableInfo::JTEntryKind enum. 1882 unsigned X86TargetLowering::getJumpTableEncoding() const { 1883 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF 1884 // symbol. 1885 if (isPositionIndependent() && Subtarget.isPICStyleGOT()) 1886 return MachineJumpTableInfo::EK_Custom32; 1887 1888 // Otherwise, use the normal jump table encoding heuristics. 1889 return TargetLowering::getJumpTableEncoding(); 1890 } 1891 1892 bool X86TargetLowering::useSoftFloat() const { 1893 return Subtarget.useSoftFloat(); 1894 } 1895 1896 const MCExpr * 1897 X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, 1898 const MachineBasicBlock *MBB, 1899 unsigned uid,MCContext &Ctx) const{ 1900 assert(isPositionIndependent() && Subtarget.isPICStyleGOT()); 1901 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF 1902 // entries. 1903 return MCSymbolRefExpr::create(MBB->getSymbol(), 1904 MCSymbolRefExpr::VK_GOTOFF, Ctx); 1905 } 1906 1907 /// Returns relocation base for the given PIC jumptable. 1908 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 1909 SelectionDAG &DAG) const { 1910 if (!Subtarget.is64Bit()) 1911 // This doesn't have SDLoc associated with it, but is not really the 1912 // same as a Register. 1913 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), 1914 getPointerTy(DAG.getDataLayout())); 1915 return Table; 1916 } 1917 1918 /// This returns the relocation base for the given PIC jumptable, 1919 /// the same as getPICJumpTableRelocBase, but as an MCExpr. 1920 const MCExpr *X86TargetLowering:: 1921 getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, 1922 MCContext &Ctx) const { 1923 // X86-64 uses RIP relative addressing based on the jump table label. 1924 if (Subtarget.isPICStyleRIPRel()) 1925 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 1926 1927 // Otherwise, the reference is relative to the PIC base. 1928 return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); 1929 } 1930 1931 std::pair<const TargetRegisterClass *, uint8_t> 1932 X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, 1933 MVT VT) const { 1934 const TargetRegisterClass *RRC = nullptr; 1935 uint8_t Cost = 1; 1936 switch (VT.SimpleTy) { 1937 default: 1938 return TargetLowering::findRepresentativeClass(TRI, VT); 1939 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: 1940 RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass; 1941 break; 1942 case MVT::x86mmx: 1943 RRC = &X86::VR64RegClass; 1944 break; 1945 case MVT::f32: case MVT::f64: 1946 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: 1947 case MVT::v4f32: case MVT::v2f64: 1948 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32: 1949 case MVT::v4f64: 1950 RRC = &X86::VR128RegClass; 1951 break; 1952 } 1953 return std::make_pair(RRC, Cost); 1954 } 1955 1956 unsigned X86TargetLowering::getAddressSpace() const { 1957 if (Subtarget.is64Bit()) 1958 return (getTargetMachine().getCodeModel() == CodeModel::Kernel) ? 256 : 257; 1959 return 256; 1960 } 1961 1962 Value *X86TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const { 1963 // glibc has a special slot for the stack guard in tcbhead_t, use it instead 1964 // of the usual global variable (see sysdeps/{i386,x86_64}/nptl/tls.h) 1965 if (!Subtarget.isTargetGlibc()) 1966 return TargetLowering::getIRStackGuard(IRB); 1967 1968 // %fs:0x28, unless we're using a Kernel code model, in which case it's %gs: 1969 // %gs:0x14 on i386 1970 unsigned Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14; 1971 unsigned AddressSpace = getAddressSpace(); 1972 return ConstantExpr::getIntToPtr( 1973 ConstantInt::get(Type::getInt32Ty(IRB.getContext()), Offset), 1974 Type::getInt8PtrTy(IRB.getContext())->getPointerTo(AddressSpace)); 1975 } 1976 1977 void X86TargetLowering::insertSSPDeclarations(Module &M) const { 1978 // MSVC CRT provides functionalities for stack protection. 1979 if (Subtarget.getTargetTriple().isOSMSVCRT()) { 1980 // MSVC CRT has a global variable holding security cookie. 1981 M.getOrInsertGlobal("__security_cookie", 1982 Type::getInt8PtrTy(M.getContext())); 1983 1984 // MSVC CRT has a function to validate security cookie. 1985 auto *SecurityCheckCookie = cast<Function>( 1986 M.getOrInsertFunction("__security_check_cookie", 1987 Type::getVoidTy(M.getContext()), 1988 Type::getInt8PtrTy(M.getContext()), nullptr)); 1989 SecurityCheckCookie->setCallingConv(CallingConv::X86_FastCall); 1990 SecurityCheckCookie->addAttribute(1, Attribute::AttrKind::InReg); 1991 return; 1992 } 1993 // glibc has a special slot for the stack guard. 1994 if (Subtarget.isTargetGlibc()) 1995 return; 1996 TargetLowering::insertSSPDeclarations(M); 1997 } 1998 1999 Value *X86TargetLowering::getSDagStackGuard(const Module &M) const { 2000 // MSVC CRT has a global variable holding security cookie. 2001 if (Subtarget.getTargetTriple().isOSMSVCRT()) 2002 return M.getGlobalVariable("__security_cookie"); 2003 return TargetLowering::getSDagStackGuard(M); 2004 } 2005 2006 Value *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const { 2007 // MSVC CRT has a function to validate security cookie. 2008 if (Subtarget.getTargetTriple().isOSMSVCRT()) 2009 return M.getFunction("__security_check_cookie"); 2010 return TargetLowering::getSSPStackGuardCheck(M); 2011 } 2012 2013 Value *X86TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const { 2014 if (!Subtarget.isTargetAndroid()) 2015 return TargetLowering::getSafeStackPointerLocation(IRB); 2016 2017 // Android provides a fixed TLS slot for the SafeStack pointer. See the 2018 // definition of TLS_SLOT_SAFESTACK in 2019 // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h 2020 unsigned AddressSpace, Offset; 2021 2022 // %fs:0x48, unless we're using a Kernel code model, in which case it's %gs: 2023 // %gs:0x24 on i386 2024 Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24; 2025 AddressSpace = getAddressSpace(); 2026 return ConstantExpr::getIntToPtr( 2027 ConstantInt::get(Type::getInt32Ty(IRB.getContext()), Offset), 2028 Type::getInt8PtrTy(IRB.getContext())->getPointerTo(AddressSpace)); 2029 } 2030 2031 bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS, 2032 unsigned DestAS) const { 2033 assert(SrcAS != DestAS && "Expected different address spaces!"); 2034 2035 return SrcAS < 256 && DestAS < 256; 2036 } 2037 2038 //===----------------------------------------------------------------------===// 2039 // Return Value Calling Convention Implementation 2040 //===----------------------------------------------------------------------===// 2041 2042 #include "X86GenCallingConv.inc" 2043 2044 bool X86TargetLowering::CanLowerReturn( 2045 CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, 2046 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { 2047 SmallVector<CCValAssign, 16> RVLocs; 2048 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 2049 return CCInfo.CheckReturn(Outs, RetCC_X86); 2050 } 2051 2052 const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const { 2053 static const MCPhysReg ScratchRegs[] = { X86::R11, 0 }; 2054 return ScratchRegs; 2055 } 2056 2057 SDValue 2058 X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 2059 bool isVarArg, 2060 const SmallVectorImpl<ISD::OutputArg> &Outs, 2061 const SmallVectorImpl<SDValue> &OutVals, 2062 const SDLoc &dl, SelectionDAG &DAG) const { 2063 MachineFunction &MF = DAG.getMachineFunction(); 2064 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 2065 2066 if (CallConv == CallingConv::X86_INTR && !Outs.empty()) 2067 report_fatal_error("X86 interrupts may not return any value"); 2068 2069 SmallVector<CCValAssign, 16> RVLocs; 2070 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext()); 2071 CCInfo.AnalyzeReturn(Outs, RetCC_X86); 2072 2073 SDValue Flag; 2074 SmallVector<SDValue, 6> RetOps; 2075 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 2076 // Operand #1 = Bytes To Pop 2077 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), dl, 2078 MVT::i32)); 2079 2080 // Copy the result values into the output registers. 2081 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 2082 CCValAssign &VA = RVLocs[i]; 2083 assert(VA.isRegLoc() && "Can only return in registers!"); 2084 SDValue ValToCopy = OutVals[i]; 2085 EVT ValVT = ValToCopy.getValueType(); 2086 2087 // Promote values to the appropriate types. 2088 if (VA.getLocInfo() == CCValAssign::SExt) 2089 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy); 2090 else if (VA.getLocInfo() == CCValAssign::ZExt) 2091 ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy); 2092 else if (VA.getLocInfo() == CCValAssign::AExt) { 2093 if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) 2094 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy); 2095 else 2096 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy); 2097 } 2098 else if (VA.getLocInfo() == CCValAssign::BCvt) 2099 ValToCopy = DAG.getBitcast(VA.getLocVT(), ValToCopy); 2100 2101 assert(VA.getLocInfo() != CCValAssign::FPExt && 2102 "Unexpected FP-extend for return value."); 2103 2104 // If this is x86-64, and we disabled SSE, we can't return FP values, 2105 // or SSE or MMX vectors. 2106 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || 2107 VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) && 2108 (Subtarget.is64Bit() && !Subtarget.hasSSE1())) { 2109 report_fatal_error("SSE register return with SSE disabled"); 2110 } 2111 // Likewise we can't return F64 values with SSE1 only. gcc does so, but 2112 // llvm-gcc has never done it right and no one has noticed, so this 2113 // should be OK for now. 2114 if (ValVT == MVT::f64 && 2115 (Subtarget.is64Bit() && !Subtarget.hasSSE2())) 2116 report_fatal_error("SSE2 register return with SSE2 disabled"); 2117 2118 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 2119 // the RET instruction and handled by the FP Stackifier. 2120 if (VA.getLocReg() == X86::FP0 || 2121 VA.getLocReg() == X86::FP1) { 2122 // If this is a copy from an xmm register to ST(0), use an FPExtend to 2123 // change the value to the FP stack register class. 2124 if (isScalarFPTypeInSSEReg(VA.getValVT())) 2125 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); 2126 RetOps.push_back(ValToCopy); 2127 // Don't emit a copytoreg. 2128 continue; 2129 } 2130 2131 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64 2132 // which is returned in RAX / RDX. 2133 if (Subtarget.is64Bit()) { 2134 if (ValVT == MVT::x86mmx) { 2135 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) { 2136 ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy); 2137 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, 2138 ValToCopy); 2139 // If we don't have SSE2 available, convert to v4f32 so the generated 2140 // register is legal. 2141 if (!Subtarget.hasSSE2()) 2142 ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy); 2143 } 2144 } 2145 } 2146 2147 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag); 2148 Flag = Chain.getValue(1); 2149 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2150 } 2151 2152 // Swift calling convention does not require we copy the sret argument 2153 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift. 2154 2155 // All x86 ABIs require that for returning structs by value we copy 2156 // the sret argument into %rax/%eax (depending on ABI) for the return. 2157 // We saved the argument into a virtual register in the entry block, 2158 // so now we copy the value out and into %rax/%eax. 2159 // 2160 // Checking Function.hasStructRetAttr() here is insufficient because the IR 2161 // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is 2162 // false, then an sret argument may be implicitly inserted in the SelDAG. In 2163 // either case FuncInfo->setSRetReturnReg() will have been called. 2164 if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) { 2165 // When we have both sret and another return value, we should use the 2166 // original Chain stored in RetOps[0], instead of the current Chain updated 2167 // in the above loop. If we only have sret, RetOps[0] equals to Chain. 2168 2169 // For the case of sret and another return value, we have 2170 // Chain_0 at the function entry 2171 // Chain_1 = getCopyToReg(Chain_0) in the above loop 2172 // If we use Chain_1 in getCopyFromReg, we will have 2173 // Val = getCopyFromReg(Chain_1) 2174 // Chain_2 = getCopyToReg(Chain_1, Val) from below 2175 2176 // getCopyToReg(Chain_0) will be glued together with 2177 // getCopyToReg(Chain_1, Val) into Unit A, getCopyFromReg(Chain_1) will be 2178 // in Unit B, and we will have cyclic dependency between Unit A and Unit B: 2179 // Data dependency from Unit B to Unit A due to usage of Val in 2180 // getCopyToReg(Chain_1, Val) 2181 // Chain dependency from Unit A to Unit B 2182 2183 // So here, we use RetOps[0] (i.e Chain_0) for getCopyFromReg. 2184 SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg, 2185 getPointerTy(MF.getDataLayout())); 2186 2187 unsigned RetValReg 2188 = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ? 2189 X86::RAX : X86::EAX; 2190 Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag); 2191 Flag = Chain.getValue(1); 2192 2193 // RAX/EAX now acts like a return value. 2194 RetOps.push_back( 2195 DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout()))); 2196 } 2197 2198 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); 2199 const MCPhysReg *I = 2200 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 2201 if (I) { 2202 for (; *I; ++I) { 2203 if (X86::GR64RegClass.contains(*I)) 2204 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); 2205 else 2206 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 2207 } 2208 } 2209 2210 RetOps[0] = Chain; // Update chain. 2211 2212 // Add the flag if we have it. 2213 if (Flag.getNode()) 2214 RetOps.push_back(Flag); 2215 2216 X86ISD::NodeType opcode = X86ISD::RET_FLAG; 2217 if (CallConv == CallingConv::X86_INTR) 2218 opcode = X86ISD::IRET; 2219 return DAG.getNode(opcode, dl, MVT::Other, RetOps); 2220 } 2221 2222 bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { 2223 if (N->getNumValues() != 1 || !N->hasNUsesOfValue(1, 0)) 2224 return false; 2225 2226 SDValue TCChain = Chain; 2227 SDNode *Copy = *N->use_begin(); 2228 if (Copy->getOpcode() == ISD::CopyToReg) { 2229 // If the copy has a glue operand, we conservatively assume it isn't safe to 2230 // perform a tail call. 2231 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) 2232 return false; 2233 TCChain = Copy->getOperand(0); 2234 } else if (Copy->getOpcode() != ISD::FP_EXTEND) 2235 return false; 2236 2237 bool HasRet = false; 2238 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end(); 2239 UI != UE; ++UI) { 2240 if (UI->getOpcode() != X86ISD::RET_FLAG) 2241 return false; 2242 // If we are returning more than one value, we can definitely 2243 // not make a tail call see PR19530 2244 if (UI->getNumOperands() > 4) 2245 return false; 2246 if (UI->getNumOperands() == 4 && 2247 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue) 2248 return false; 2249 HasRet = true; 2250 } 2251 2252 if (!HasRet) 2253 return false; 2254 2255 Chain = TCChain; 2256 return true; 2257 } 2258 2259 EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT, 2260 ISD::NodeType ExtendKind) const { 2261 MVT ReturnMVT = MVT::i32; 2262 2263 bool Darwin = Subtarget.getTargetTriple().isOSDarwin(); 2264 if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) { 2265 // The ABI does not require i1, i8 or i16 to be extended. 2266 // 2267 // On Darwin, there is code in the wild relying on Clang's old behaviour of 2268 // always extending i8/i16 return values, so keep doing that for now. 2269 // (PR26665). 2270 ReturnMVT = MVT::i8; 2271 } 2272 2273 EVT MinVT = getRegisterType(Context, ReturnMVT); 2274 return VT.bitsLT(MinVT) ? MinVT : VT; 2275 } 2276 2277 /// Lower the result values of a call into the 2278 /// appropriate copies out of appropriate physical registers. 2279 /// 2280 SDValue X86TargetLowering::LowerCallResult( 2281 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 2282 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 2283 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 2284 2285 // Assign locations to each value returned by this call. 2286 SmallVector<CCValAssign, 16> RVLocs; 2287 bool Is64Bit = Subtarget.is64Bit(); 2288 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 2289 *DAG.getContext()); 2290 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 2291 2292 // Copy all of the result registers out of their specified physreg. 2293 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 2294 CCValAssign &VA = RVLocs[i]; 2295 EVT CopyVT = VA.getLocVT(); 2296 2297 // If this is x86-64, and we disabled SSE, we can't return FP values 2298 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64 || CopyVT == MVT::f128) && 2299 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget.hasSSE1())) { 2300 report_fatal_error("SSE register return with SSE disabled"); 2301 } 2302 2303 // If we prefer to use the value in xmm registers, copy it out as f80 and 2304 // use a truncate to move it from fp stack reg to xmm reg. 2305 bool RoundAfterCopy = false; 2306 if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) && 2307 isScalarFPTypeInSSEReg(VA.getValVT())) { 2308 if (!Subtarget.hasX87()) 2309 report_fatal_error("X87 register return with X87 disabled"); 2310 CopyVT = MVT::f80; 2311 RoundAfterCopy = (CopyVT != VA.getLocVT()); 2312 } 2313 2314 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), 2315 CopyVT, InFlag).getValue(1); 2316 SDValue Val = Chain.getValue(0); 2317 2318 if (RoundAfterCopy) 2319 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, 2320 // This truncation won't change the value. 2321 DAG.getIntPtrConstant(1, dl)); 2322 2323 if (VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1) 2324 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 2325 2326 InFlag = Chain.getValue(2); 2327 InVals.push_back(Val); 2328 } 2329 2330 return Chain; 2331 } 2332 2333 //===----------------------------------------------------------------------===// 2334 // C & StdCall & Fast Calling Convention implementation 2335 //===----------------------------------------------------------------------===// 2336 // StdCall calling convention seems to be standard for many Windows' API 2337 // routines and around. It differs from C calling convention just a little: 2338 // callee should clean up the stack, not caller. Symbols should be also 2339 // decorated in some fancy way :) It doesn't support any vector arguments. 2340 // For info on fast calling convention see Fast Calling Convention (tail call) 2341 // implementation LowerX86_32FastCCCallTo. 2342 2343 /// CallIsStructReturn - Determines whether a call uses struct return 2344 /// semantics. 2345 enum StructReturnType { 2346 NotStructReturn, 2347 RegStructReturn, 2348 StackStructReturn 2349 }; 2350 static StructReturnType 2351 callIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsMCU) { 2352 if (Outs.empty()) 2353 return NotStructReturn; 2354 2355 const ISD::ArgFlagsTy &Flags = Outs[0].Flags; 2356 if (!Flags.isSRet()) 2357 return NotStructReturn; 2358 if (Flags.isInReg() || IsMCU) 2359 return RegStructReturn; 2360 return StackStructReturn; 2361 } 2362 2363 /// Determines whether a function uses struct return semantics. 2364 static StructReturnType 2365 argsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins, bool IsMCU) { 2366 if (Ins.empty()) 2367 return NotStructReturn; 2368 2369 const ISD::ArgFlagsTy &Flags = Ins[0].Flags; 2370 if (!Flags.isSRet()) 2371 return NotStructReturn; 2372 if (Flags.isInReg() || IsMCU) 2373 return RegStructReturn; 2374 return StackStructReturn; 2375 } 2376 2377 /// Make a copy of an aggregate at address specified by "Src" to address 2378 /// "Dst" with size and alignment information specified by the specific 2379 /// parameter attribute. The copy will be passed as a byval function parameter. 2380 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, 2381 SDValue Chain, ISD::ArgFlagsTy Flags, 2382 SelectionDAG &DAG, const SDLoc &dl) { 2383 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); 2384 2385 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 2386 /*isVolatile*/false, /*AlwaysInline=*/true, 2387 /*isTailCall*/false, 2388 MachinePointerInfo(), MachinePointerInfo()); 2389 } 2390 2391 /// Return true if the calling convention is one that we can guarantee TCO for. 2392 static bool canGuaranteeTCO(CallingConv::ID CC) { 2393 return (CC == CallingConv::Fast || CC == CallingConv::GHC || 2394 CC == CallingConv::HiPE || CC == CallingConv::HHVM); 2395 } 2396 2397 /// Return true if we might ever do TCO for calls with this calling convention. 2398 static bool mayTailCallThisCC(CallingConv::ID CC) { 2399 switch (CC) { 2400 // C calling conventions: 2401 case CallingConv::C: 2402 case CallingConv::X86_64_Win64: 2403 case CallingConv::X86_64_SysV: 2404 // Callee pop conventions: 2405 case CallingConv::X86_ThisCall: 2406 case CallingConv::X86_StdCall: 2407 case CallingConv::X86_VectorCall: 2408 case CallingConv::X86_FastCall: 2409 return true; 2410 default: 2411 return canGuaranteeTCO(CC); 2412 } 2413 } 2414 2415 /// Return true if the function is being made into a tailcall target by 2416 /// changing its ABI. 2417 static bool shouldGuaranteeTCO(CallingConv::ID CC, bool GuaranteedTailCallOpt) { 2418 return GuaranteedTailCallOpt && canGuaranteeTCO(CC); 2419 } 2420 2421 bool X86TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 2422 auto Attr = 2423 CI->getParent()->getParent()->getFnAttribute("disable-tail-calls"); 2424 if (!CI->isTailCall() || Attr.getValueAsString() == "true") 2425 return false; 2426 2427 CallSite CS(CI); 2428 CallingConv::ID CalleeCC = CS.getCallingConv(); 2429 if (!mayTailCallThisCC(CalleeCC)) 2430 return false; 2431 2432 return true; 2433 } 2434 2435 SDValue 2436 X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv, 2437 const SmallVectorImpl<ISD::InputArg> &Ins, 2438 const SDLoc &dl, SelectionDAG &DAG, 2439 const CCValAssign &VA, 2440 MachineFrameInfo *MFI, unsigned i) const { 2441 // Create the nodes corresponding to a load from this parameter slot. 2442 ISD::ArgFlagsTy Flags = Ins[i].Flags; 2443 bool AlwaysUseMutable = shouldGuaranteeTCO( 2444 CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt); 2445 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 2446 EVT ValVT; 2447 2448 // If value is passed by pointer we have address passed instead of the value 2449 // itself. 2450 bool ExtendedInMem = VA.isExtInLoc() && 2451 VA.getValVT().getScalarType() == MVT::i1; 2452 2453 if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem) 2454 ValVT = VA.getLocVT(); 2455 else 2456 ValVT = VA.getValVT(); 2457 2458 // Calculate SP offset of interrupt parameter, re-arrange the slot normally 2459 // taken by a return address. 2460 int Offset = 0; 2461 if (CallConv == CallingConv::X86_INTR) { 2462 const X86Subtarget& Subtarget = 2463 static_cast<const X86Subtarget&>(DAG.getSubtarget()); 2464 // X86 interrupts may take one or two arguments. 2465 // On the stack there will be no return address as in regular call. 2466 // Offset of last argument need to be set to -4/-8 bytes. 2467 // Where offset of the first argument out of two, should be set to 0 bytes. 2468 Offset = (Subtarget.is64Bit() ? 8 : 4) * ((i + 1) % Ins.size() - 1); 2469 } 2470 2471 // FIXME: For now, all byval parameter objects are marked mutable. This can be 2472 // changed with more analysis. 2473 // In case of tail call optimization mark all arguments mutable. Since they 2474 // could be overwritten by lowering of arguments in case of a tail call. 2475 if (Flags.isByVal()) { 2476 unsigned Bytes = Flags.getByValSize(); 2477 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects. 2478 int FI = MFI->CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable); 2479 // Adjust SP offset of interrupt parameter. 2480 if (CallConv == CallingConv::X86_INTR) { 2481 MFI->setObjectOffset(FI, Offset); 2482 } 2483 return DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 2484 } else { 2485 int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8, 2486 VA.getLocMemOffset(), isImmutable); 2487 2488 // Set SExt or ZExt flag. 2489 if (VA.getLocInfo() == CCValAssign::ZExt) { 2490 MFI->setObjectZExt(FI, true); 2491 } else if (VA.getLocInfo() == CCValAssign::SExt) { 2492 MFI->setObjectSExt(FI, true); 2493 } 2494 2495 // Adjust SP offset of interrupt parameter. 2496 if (CallConv == CallingConv::X86_INTR) { 2497 MFI->setObjectOffset(FI, Offset); 2498 } 2499 2500 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 2501 SDValue Val = DAG.getLoad( 2502 ValVT, dl, Chain, FIN, 2503 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), false, 2504 false, false, 0); 2505 return ExtendedInMem ? 2506 DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val) : Val; 2507 } 2508 } 2509 2510 // FIXME: Get this from tablegen. 2511 static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv, 2512 const X86Subtarget &Subtarget) { 2513 assert(Subtarget.is64Bit()); 2514 2515 if (Subtarget.isCallingConvWin64(CallConv)) { 2516 static const MCPhysReg GPR64ArgRegsWin64[] = { 2517 X86::RCX, X86::RDX, X86::R8, X86::R9 2518 }; 2519 return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64)); 2520 } 2521 2522 static const MCPhysReg GPR64ArgRegs64Bit[] = { 2523 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 2524 }; 2525 return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit)); 2526 } 2527 2528 // FIXME: Get this from tablegen. 2529 static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF, 2530 CallingConv::ID CallConv, 2531 const X86Subtarget &Subtarget) { 2532 assert(Subtarget.is64Bit()); 2533 if (Subtarget.isCallingConvWin64(CallConv)) { 2534 // The XMM registers which might contain var arg parameters are shadowed 2535 // in their paired GPR. So we only need to save the GPR to their home 2536 // slots. 2537 // TODO: __vectorcall will change this. 2538 return None; 2539 } 2540 2541 const Function *Fn = MF.getFunction(); 2542 bool NoImplicitFloatOps = Fn->hasFnAttribute(Attribute::NoImplicitFloat); 2543 bool isSoftFloat = Subtarget.useSoftFloat(); 2544 assert(!(isSoftFloat && NoImplicitFloatOps) && 2545 "SSE register cannot be used when SSE is disabled!"); 2546 if (isSoftFloat || NoImplicitFloatOps || !Subtarget.hasSSE1()) 2547 // Kernel mode asks for SSE to be disabled, so there are no XMM argument 2548 // registers. 2549 return None; 2550 2551 static const MCPhysReg XMMArgRegs64Bit[] = { 2552 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 2553 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 2554 }; 2555 return makeArrayRef(std::begin(XMMArgRegs64Bit), std::end(XMMArgRegs64Bit)); 2556 } 2557 2558 SDValue X86TargetLowering::LowerFormalArguments( 2559 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 2560 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 2561 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 2562 MachineFunction &MF = DAG.getMachineFunction(); 2563 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 2564 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering(); 2565 2566 const Function *Fn = MF.getFunction(); 2567 if (Fn->hasExternalLinkage() && 2568 Subtarget.isTargetCygMing() && 2569 Fn->getName() == "main") 2570 FuncInfo->setForceFramePointer(true); 2571 2572 MachineFrameInfo *MFI = MF.getFrameInfo(); 2573 bool Is64Bit = Subtarget.is64Bit(); 2574 bool IsWin64 = Subtarget.isCallingConvWin64(CallConv); 2575 2576 assert(!(isVarArg && canGuaranteeTCO(CallConv)) && 2577 "Var args not supported with calling convention fastcc, ghc or hipe"); 2578 2579 if (CallConv == CallingConv::X86_INTR) { 2580 bool isLegal = Ins.size() == 1 || 2581 (Ins.size() == 2 && ((Is64Bit && Ins[1].VT == MVT::i64) || 2582 (!Is64Bit && Ins[1].VT == MVT::i32))); 2583 if (!isLegal) 2584 report_fatal_error("X86 interrupts may take one or two arguments"); 2585 } 2586 2587 // Assign locations to all of the incoming arguments. 2588 SmallVector<CCValAssign, 16> ArgLocs; 2589 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 2590 2591 // Allocate shadow area for Win64 2592 if (IsWin64) 2593 CCInfo.AllocateStack(32, 8); 2594 2595 CCInfo.AnalyzeFormalArguments(Ins, CC_X86); 2596 2597 unsigned LastVal = ~0U; 2598 SDValue ArgValue; 2599 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2600 CCValAssign &VA = ArgLocs[i]; 2601 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 2602 // places. 2603 assert(VA.getValNo() != LastVal && 2604 "Don't support value assigned to multiple locs yet"); 2605 (void)LastVal; 2606 LastVal = VA.getValNo(); 2607 2608 if (VA.isRegLoc()) { 2609 EVT RegVT = VA.getLocVT(); 2610 const TargetRegisterClass *RC; 2611 if (RegVT == MVT::i32) 2612 RC = &X86::GR32RegClass; 2613 else if (Is64Bit && RegVT == MVT::i64) 2614 RC = &X86::GR64RegClass; 2615 else if (RegVT == MVT::f32) 2616 RC = &X86::FR32RegClass; 2617 else if (RegVT == MVT::f64) 2618 RC = &X86::FR64RegClass; 2619 else if (RegVT == MVT::f128) 2620 RC = &X86::FR128RegClass; 2621 else if (RegVT.is512BitVector()) 2622 RC = &X86::VR512RegClass; 2623 else if (RegVT.is256BitVector()) 2624 RC = &X86::VR256RegClass; 2625 else if (RegVT.is128BitVector()) 2626 RC = &X86::VR128RegClass; 2627 else if (RegVT == MVT::x86mmx) 2628 RC = &X86::VR64RegClass; 2629 else if (RegVT == MVT::i1) 2630 RC = &X86::VK1RegClass; 2631 else if (RegVT == MVT::v8i1) 2632 RC = &X86::VK8RegClass; 2633 else if (RegVT == MVT::v16i1) 2634 RC = &X86::VK16RegClass; 2635 else if (RegVT == MVT::v32i1) 2636 RC = &X86::VK32RegClass; 2637 else if (RegVT == MVT::v64i1) 2638 RC = &X86::VK64RegClass; 2639 else 2640 llvm_unreachable("Unknown argument type!"); 2641 2642 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 2643 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); 2644 2645 // If this is an 8 or 16-bit value, it is really passed promoted to 32 2646 // bits. Insert an assert[sz]ext to capture this, then truncate to the 2647 // right size. 2648 if (VA.getLocInfo() == CCValAssign::SExt) 2649 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, 2650 DAG.getValueType(VA.getValVT())); 2651 else if (VA.getLocInfo() == CCValAssign::ZExt) 2652 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, 2653 DAG.getValueType(VA.getValVT())); 2654 else if (VA.getLocInfo() == CCValAssign::BCvt) 2655 ArgValue = DAG.getBitcast(VA.getValVT(), ArgValue); 2656 2657 if (VA.isExtInLoc()) { 2658 // Handle MMX values passed in XMM regs. 2659 if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1) 2660 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue); 2661 else 2662 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 2663 } 2664 } else { 2665 assert(VA.isMemLoc()); 2666 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i); 2667 } 2668 2669 // If value is passed via pointer - do a load. 2670 if (VA.getLocInfo() == CCValAssign::Indirect) 2671 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, 2672 MachinePointerInfo(), false, false, false, 0); 2673 2674 InVals.push_back(ArgValue); 2675 } 2676 2677 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2678 // Swift calling convention does not require we copy the sret argument 2679 // into %rax/%eax for the return. We don't set SRetReturnReg for Swift. 2680 if (CallConv == CallingConv::Swift) 2681 continue; 2682 2683 // All x86 ABIs require that for returning structs by value we copy the 2684 // sret argument into %rax/%eax (depending on ABI) for the return. Save 2685 // the argument into a virtual register so that we can access it from the 2686 // return points. 2687 if (Ins[i].Flags.isSRet()) { 2688 unsigned Reg = FuncInfo->getSRetReturnReg(); 2689 if (!Reg) { 2690 MVT PtrTy = getPointerTy(DAG.getDataLayout()); 2691 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy)); 2692 FuncInfo->setSRetReturnReg(Reg); 2693 } 2694 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[i]); 2695 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); 2696 break; 2697 } 2698 } 2699 2700 unsigned StackSize = CCInfo.getNextStackOffset(); 2701 // Align stack specially for tail calls. 2702 if (shouldGuaranteeTCO(CallConv, 2703 MF.getTarget().Options.GuaranteedTailCallOpt)) 2704 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 2705 2706 // If the function takes variable number of arguments, make a frame index for 2707 // the start of the first vararg value... for expansion of llvm.va_start. We 2708 // can skip this if there are no va_start calls. 2709 if (MFI->hasVAStart() && 2710 (Is64Bit || (CallConv != CallingConv::X86_FastCall && 2711 CallConv != CallingConv::X86_ThisCall))) { 2712 FuncInfo->setVarArgsFrameIndex( 2713 MFI->CreateFixedObject(1, StackSize, true)); 2714 } 2715 2716 // Figure out if XMM registers are in use. 2717 assert(!(Subtarget.useSoftFloat() && 2718 Fn->hasFnAttribute(Attribute::NoImplicitFloat)) && 2719 "SSE register cannot be used when SSE is disabled!"); 2720 2721 // 64-bit calling conventions support varargs and register parameters, so we 2722 // have to do extra work to spill them in the prologue. 2723 if (Is64Bit && isVarArg && MFI->hasVAStart()) { 2724 // Find the first unallocated argument registers. 2725 ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget); 2726 ArrayRef<MCPhysReg> ArgXMMs = get64BitArgumentXMMs(MF, CallConv, Subtarget); 2727 unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs); 2728 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs); 2729 assert(!(NumXMMRegs && !Subtarget.hasSSE1()) && 2730 "SSE register cannot be used when SSE is disabled!"); 2731 2732 // Gather all the live in physical registers. 2733 SmallVector<SDValue, 6> LiveGPRs; 2734 SmallVector<SDValue, 8> LiveXMMRegs; 2735 SDValue ALVal; 2736 for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) { 2737 unsigned GPR = MF.addLiveIn(Reg, &X86::GR64RegClass); 2738 LiveGPRs.push_back( 2739 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64)); 2740 } 2741 if (!ArgXMMs.empty()) { 2742 unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass); 2743 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8); 2744 for (MCPhysReg Reg : ArgXMMs.slice(NumXMMRegs)) { 2745 unsigned XMMReg = MF.addLiveIn(Reg, &X86::VR128RegClass); 2746 LiveXMMRegs.push_back( 2747 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32)); 2748 } 2749 } 2750 2751 if (IsWin64) { 2752 // Get to the caller-allocated home save location. Add 8 to account 2753 // for the return address. 2754 int HomeOffset = TFI.getOffsetOfLocalArea() + 8; 2755 FuncInfo->setRegSaveFrameIndex( 2756 MFI->CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false)); 2757 // Fixup to set vararg frame on shadow area (4 x i64). 2758 if (NumIntRegs < 4) 2759 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex()); 2760 } else { 2761 // For X86-64, if there are vararg parameters that are passed via 2762 // registers, then we must store them to their spots on the stack so 2763 // they may be loaded by dereferencing the result of va_next. 2764 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8); 2765 FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16); 2766 FuncInfo->setRegSaveFrameIndex(MFI->CreateStackObject( 2767 ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false)); 2768 } 2769 2770 // Store the integer parameter registers. 2771 SmallVector<SDValue, 8> MemOps; 2772 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), 2773 getPointerTy(DAG.getDataLayout())); 2774 unsigned Offset = FuncInfo->getVarArgsGPOffset(); 2775 for (SDValue Val : LiveGPRs) { 2776 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), 2777 RSFIN, DAG.getIntPtrConstant(Offset, dl)); 2778 SDValue Store = 2779 DAG.getStore(Val.getValue(1), dl, Val, FIN, 2780 MachinePointerInfo::getFixedStack( 2781 DAG.getMachineFunction(), 2782 FuncInfo->getRegSaveFrameIndex(), Offset), 2783 false, false, 0); 2784 MemOps.push_back(Store); 2785 Offset += 8; 2786 } 2787 2788 if (!ArgXMMs.empty() && NumXMMRegs != ArgXMMs.size()) { 2789 // Now store the XMM (fp + vector) parameter registers. 2790 SmallVector<SDValue, 12> SaveXMMOps; 2791 SaveXMMOps.push_back(Chain); 2792 SaveXMMOps.push_back(ALVal); 2793 SaveXMMOps.push_back(DAG.getIntPtrConstant( 2794 FuncInfo->getRegSaveFrameIndex(), dl)); 2795 SaveXMMOps.push_back(DAG.getIntPtrConstant( 2796 FuncInfo->getVarArgsFPOffset(), dl)); 2797 SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(), 2798 LiveXMMRegs.end()); 2799 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl, 2800 MVT::Other, SaveXMMOps)); 2801 } 2802 2803 if (!MemOps.empty()) 2804 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 2805 } 2806 2807 if (isVarArg && MFI->hasMustTailInVarArgFunc()) { 2808 // Find the largest legal vector type. 2809 MVT VecVT = MVT::Other; 2810 // FIXME: Only some x86_32 calling conventions support AVX512. 2811 if (Subtarget.hasAVX512() && 2812 (Is64Bit || (CallConv == CallingConv::X86_VectorCall || 2813 CallConv == CallingConv::Intel_OCL_BI))) 2814 VecVT = MVT::v16f32; 2815 else if (Subtarget.hasAVX()) 2816 VecVT = MVT::v8f32; 2817 else if (Subtarget.hasSSE2()) 2818 VecVT = MVT::v4f32; 2819 2820 // We forward some GPRs and some vector types. 2821 SmallVector<MVT, 2> RegParmTypes; 2822 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32; 2823 RegParmTypes.push_back(IntVT); 2824 if (VecVT != MVT::Other) 2825 RegParmTypes.push_back(VecVT); 2826 2827 // Compute the set of forwarded registers. The rest are scratch. 2828 SmallVectorImpl<ForwardedRegister> &Forwards = 2829 FuncInfo->getForwardedMustTailRegParms(); 2830 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86); 2831 2832 // Conservatively forward AL on x86_64, since it might be used for varargs. 2833 if (Is64Bit && !CCInfo.isAllocated(X86::AL)) { 2834 unsigned ALVReg = MF.addLiveIn(X86::AL, &X86::GR8RegClass); 2835 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8)); 2836 } 2837 2838 // Copy all forwards from physical to virtual registers. 2839 for (ForwardedRegister &F : Forwards) { 2840 // FIXME: Can we use a less constrained schedule? 2841 SDValue RegVal = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT); 2842 F.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(F.VT)); 2843 Chain = DAG.getCopyToReg(Chain, dl, F.VReg, RegVal); 2844 } 2845 } 2846 2847 // Some CCs need callee pop. 2848 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, 2849 MF.getTarget().Options.GuaranteedTailCallOpt)) { 2850 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything. 2851 } else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) { 2852 // X86 interrupts must pop the error code if present 2853 FuncInfo->setBytesToPopOnReturn(Is64Bit ? 8 : 4); 2854 } else { 2855 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing. 2856 // If this is an sret function, the return should pop the hidden pointer. 2857 if (!Is64Bit && !canGuaranteeTCO(CallConv) && 2858 !Subtarget.getTargetTriple().isOSMSVCRT() && 2859 argsAreStructReturn(Ins, Subtarget.isTargetMCU()) == StackStructReturn) 2860 FuncInfo->setBytesToPopOnReturn(4); 2861 } 2862 2863 if (!Is64Bit) { 2864 // RegSaveFrameIndex is X86-64 only. 2865 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA); 2866 if (CallConv == CallingConv::X86_FastCall || 2867 CallConv == CallingConv::X86_ThisCall) 2868 // fastcc functions can't have varargs. 2869 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA); 2870 } 2871 2872 FuncInfo->setArgumentStackSize(StackSize); 2873 2874 if (WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo()) { 2875 EHPersonality Personality = classifyEHPersonality(Fn->getPersonalityFn()); 2876 if (Personality == EHPersonality::CoreCLR) { 2877 assert(Is64Bit); 2878 // TODO: Add a mechanism to frame lowering that will allow us to indicate 2879 // that we'd prefer this slot be allocated towards the bottom of the frame 2880 // (i.e. near the stack pointer after allocating the frame). Every 2881 // funclet needs a copy of this slot in its (mostly empty) frame, and the 2882 // offset from the bottom of this and each funclet's frame must be the 2883 // same, so the size of funclets' (mostly empty) frames is dictated by 2884 // how far this slot is from the bottom (since they allocate just enough 2885 // space to accommodate holding this slot at the correct offset). 2886 int PSPSymFI = MFI->CreateStackObject(8, 8, /*isSS=*/false); 2887 EHInfo->PSPSymFrameIdx = PSPSymFI; 2888 } 2889 } 2890 2891 return Chain; 2892 } 2893 2894 SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, 2895 SDValue Arg, const SDLoc &dl, 2896 SelectionDAG &DAG, 2897 const CCValAssign &VA, 2898 ISD::ArgFlagsTy Flags) const { 2899 unsigned LocMemOffset = VA.getLocMemOffset(); 2900 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 2901 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), 2902 StackPtr, PtrOff); 2903 if (Flags.isByVal()) 2904 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl); 2905 2906 return DAG.getStore( 2907 Chain, dl, Arg, PtrOff, 2908 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset), 2909 false, false, 0); 2910 } 2911 2912 /// Emit a load of return address if tail call 2913 /// optimization is performed and it is required. 2914 SDValue X86TargetLowering::EmitTailCallLoadRetAddr( 2915 SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall, 2916 bool Is64Bit, int FPDiff, const SDLoc &dl) const { 2917 // Adjust the Return address stack slot. 2918 EVT VT = getPointerTy(DAG.getDataLayout()); 2919 OutRetAddr = getReturnAddressFrameIndex(DAG); 2920 2921 // Load the "old" Return address. 2922 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo(), 2923 false, false, false, 0); 2924 return SDValue(OutRetAddr.getNode(), 1); 2925 } 2926 2927 /// Emit a store of the return address if tail call 2928 /// optimization is performed and it is required (FPDiff!=0). 2929 static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF, 2930 SDValue Chain, SDValue RetAddrFrIdx, 2931 EVT PtrVT, unsigned SlotSize, 2932 int FPDiff, const SDLoc &dl) { 2933 // Store the return address to the appropriate stack slot. 2934 if (!FPDiff) return Chain; 2935 // Calculate the new stack slot for the return address. 2936 int NewReturnAddrFI = 2937 MF.getFrameInfo()->CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize, 2938 false); 2939 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT); 2940 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, 2941 MachinePointerInfo::getFixedStack( 2942 DAG.getMachineFunction(), NewReturnAddrFI), 2943 false, false, 0); 2944 return Chain; 2945 } 2946 2947 /// Returns a vector_shuffle mask for an movs{s|d}, movd 2948 /// operation of specified width. 2949 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, 2950 SDValue V2) { 2951 unsigned NumElems = VT.getVectorNumElements(); 2952 SmallVector<int, 8> Mask; 2953 Mask.push_back(NumElems); 2954 for (unsigned i = 1; i != NumElems; ++i) 2955 Mask.push_back(i); 2956 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); 2957 } 2958 2959 SDValue 2960 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 2961 SmallVectorImpl<SDValue> &InVals) const { 2962 SelectionDAG &DAG = CLI.DAG; 2963 SDLoc &dl = CLI.DL; 2964 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 2965 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 2966 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 2967 SDValue Chain = CLI.Chain; 2968 SDValue Callee = CLI.Callee; 2969 CallingConv::ID CallConv = CLI.CallConv; 2970 bool &isTailCall = CLI.IsTailCall; 2971 bool isVarArg = CLI.IsVarArg; 2972 2973 MachineFunction &MF = DAG.getMachineFunction(); 2974 bool Is64Bit = Subtarget.is64Bit(); 2975 bool IsWin64 = Subtarget.isCallingConvWin64(CallConv); 2976 StructReturnType SR = callIsStructReturn(Outs, Subtarget.isTargetMCU()); 2977 bool IsSibcall = false; 2978 X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>(); 2979 auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls"); 2980 2981 if (CallConv == CallingConv::X86_INTR) 2982 report_fatal_error("X86 interrupts may not be called directly"); 2983 2984 if (Attr.getValueAsString() == "true") 2985 isTailCall = false; 2986 2987 if (Subtarget.isPICStyleGOT() && 2988 !MF.getTarget().Options.GuaranteedTailCallOpt) { 2989 // If we are using a GOT, disable tail calls to external symbols with 2990 // default visibility. Tail calling such a symbol requires using a GOT 2991 // relocation, which forces early binding of the symbol. This breaks code 2992 // that require lazy function symbol resolution. Using musttail or 2993 // GuaranteedTailCallOpt will override this. 2994 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 2995 if (!G || (!G->getGlobal()->hasLocalLinkage() && 2996 G->getGlobal()->hasDefaultVisibility())) 2997 isTailCall = false; 2998 } 2999 3000 bool IsMustTail = CLI.CS && CLI.CS->isMustTailCall(); 3001 if (IsMustTail) { 3002 // Force this to be a tail call. The verifier rules are enough to ensure 3003 // that we can lower this successfully without moving the return address 3004 // around. 3005 isTailCall = true; 3006 } else if (isTailCall) { 3007 // Check if it's really possible to do a tail call. 3008 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, 3009 isVarArg, SR != NotStructReturn, 3010 MF.getFunction()->hasStructRetAttr(), CLI.RetTy, 3011 Outs, OutVals, Ins, DAG); 3012 3013 // Sibcalls are automatically detected tailcalls which do not require 3014 // ABI changes. 3015 if (!MF.getTarget().Options.GuaranteedTailCallOpt && isTailCall) 3016 IsSibcall = true; 3017 3018 if (isTailCall) 3019 ++NumTailCalls; 3020 } 3021 3022 assert(!(isVarArg && canGuaranteeTCO(CallConv)) && 3023 "Var args not supported with calling convention fastcc, ghc or hipe"); 3024 3025 // Analyze operands of the call, assigning locations to each operand. 3026 SmallVector<CCValAssign, 16> ArgLocs; 3027 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 3028 3029 // Allocate shadow area for Win64 3030 if (IsWin64) 3031 CCInfo.AllocateStack(32, 8); 3032 3033 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 3034 3035 // Get a count of how many bytes are to be pushed on the stack. 3036 unsigned NumBytes = CCInfo.getAlignedCallFrameSize(); 3037 if (IsSibcall) 3038 // This is a sibcall. The memory operands are available in caller's 3039 // own caller's stack. 3040 NumBytes = 0; 3041 else if (MF.getTarget().Options.GuaranteedTailCallOpt && 3042 canGuaranteeTCO(CallConv)) 3043 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 3044 3045 int FPDiff = 0; 3046 if (isTailCall && !IsSibcall && !IsMustTail) { 3047 // Lower arguments at fp - stackoffset + fpdiff. 3048 unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn(); 3049 3050 FPDiff = NumBytesCallerPushed - NumBytes; 3051 3052 // Set the delta of movement of the returnaddr stackslot. 3053 // But only set if delta is greater than previous delta. 3054 if (FPDiff < X86Info->getTCReturnAddrDelta()) 3055 X86Info->setTCReturnAddrDelta(FPDiff); 3056 } 3057 3058 unsigned NumBytesToPush = NumBytes; 3059 unsigned NumBytesToPop = NumBytes; 3060 3061 // If we have an inalloca argument, all stack space has already been allocated 3062 // for us and be right at the top of the stack. We don't support multiple 3063 // arguments passed in memory when using inalloca. 3064 if (!Outs.empty() && Outs.back().Flags.isInAlloca()) { 3065 NumBytesToPush = 0; 3066 if (!ArgLocs.back().isMemLoc()) 3067 report_fatal_error("cannot use inalloca attribute on a register " 3068 "parameter"); 3069 if (ArgLocs.back().getLocMemOffset() != 0) 3070 report_fatal_error("any parameter with the inalloca attribute must be " 3071 "the only memory argument"); 3072 } 3073 3074 if (!IsSibcall) 3075 Chain = DAG.getCALLSEQ_START( 3076 Chain, DAG.getIntPtrConstant(NumBytesToPush, dl, true), dl); 3077 3078 SDValue RetAddrFrIdx; 3079 // Load return address for tail calls. 3080 if (isTailCall && FPDiff) 3081 Chain = EmitTailCallLoadRetAddr