1 //===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the PPCISelLowering class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "PPCISelLowering.h" 15 #include "MCTargetDesc/PPCPredicates.h" 16 #include "PPCCallingConv.h" 17 #include "PPCMachineFunctionInfo.h" 18 #include "PPCPerfectShuffle.h" 19 #include "PPCTargetMachine.h" 20 #include "PPCTargetObjectFile.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/ADT/StringSwitch.h" 23 #include "llvm/ADT/Triple.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/MachineFrameInfo.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineInstrBuilder.h" 28 #include "llvm/CodeGen/MachineLoopInfo.h" 29 #include "llvm/CodeGen/MachineRegisterInfo.h" 30 #include "llvm/CodeGen/SelectionDAG.h" 31 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 32 #include "llvm/IR/CallingConv.h" 33 #include "llvm/IR/Constants.h" 34 #include "llvm/IR/DerivedTypes.h" 35 #include "llvm/IR/Function.h" 36 #include "llvm/IR/Intrinsics.h" 37 #include "llvm/Support/CommandLine.h" 38 #include "llvm/Support/ErrorHandling.h" 39 #include "llvm/Support/MathExtras.h" 40 #include "llvm/Support/raw_ostream.h" 41 #include "llvm/Target/TargetOptions.h" 42 43 using namespace llvm; 44 45 static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc", 46 cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); 47 48 static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref", 49 cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); 50 51 static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned", 52 cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); 53 54 // FIXME: Remove this once the bug has been fixed! 55 extern cl::opt<bool> ANDIGlueBug; 56 57 PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, 58 const PPCSubtarget &STI) 59 : TargetLowering(TM), Subtarget(STI) { 60 // Use _setjmp/_longjmp instead of setjmp/longjmp. 61 setUseUnderscoreSetJmp(true); 62 setUseUnderscoreLongJmp(true); 63 64 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all 65 // arguments are at least 4/8 bytes aligned. 66 bool isPPC64 = Subtarget.isPPC64(); 67 setMinStackArgumentAlignment(isPPC64 ? 8:4); 68 69 // Set up the register classes. 70 addRegisterClass(MVT::i32, &PPC::GPRCRegClass); 71 if (!Subtarget.useSoftFloat()) { 72 addRegisterClass(MVT::f32, &PPC::F4RCRegClass); 73 addRegisterClass(MVT::f64, &PPC::F8RCRegClass); 74 } 75 76 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD 77 for (MVT VT : MVT::integer_valuetypes()) { 78 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 79 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); 80 } 81 82 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 83 84 // PowerPC has pre-inc load and store's. 85 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); 86 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); 87 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); 88 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); 89 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); 90 setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal); 91 setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal); 92 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); 93 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); 94 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); 95 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); 96 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); 97 setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal); 98 setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal); 99 100 if (Subtarget.useCRBits()) { 101 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 102 103 if (isPPC64 || Subtarget.hasFPCVT()) { 104 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); 105 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, 106 isPPC64 ? MVT::i64 : MVT::i32); 107 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); 108 AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, 109 isPPC64 ? MVT::i64 : MVT::i32); 110 } else { 111 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); 112 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); 113 } 114 115 // PowerPC does not support direct load / store of condition registers 116 setOperationAction(ISD::LOAD, MVT::i1, Custom); 117 setOperationAction(ISD::STORE, MVT::i1, Custom); 118 119 // FIXME: Remove this once the ANDI glue bug is fixed: 120 if (ANDIGlueBug) 121 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); 122 123 for (MVT VT : MVT::integer_valuetypes()) { 124 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 125 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 126 setTruncStoreAction(VT, MVT::i1, Expand); 127 } 128 129 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); 130 } 131 132 // This is used in the ppcf128->int sequence. Note it has different semantics 133 // from FP_ROUND: that rounds to nearest, this rounds to zero. 134 setOperationAction(ISD::FP_ROUND_INREG, MVT::ppcf128, Custom); 135 136 // We do not currently implement these libm ops for PowerPC. 137 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); 138 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); 139 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); 140 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); 141 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); 142 setOperationAction(ISD::FREM, MVT::ppcf128, Expand); 143 144 // PowerPC has no SREM/UREM instructions 145 setOperationAction(ISD::SREM, MVT::i32, Expand); 146 setOperationAction(ISD::UREM, MVT::i32, Expand); 147 setOperationAction(ISD::SREM, MVT::i64, Expand); 148 setOperationAction(ISD::UREM, MVT::i64, Expand); 149 150 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. 151 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 152 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 153 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 154 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 155 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 156 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 157 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); 158 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); 159 160 // We don't support sin/cos/sqrt/fmod/pow 161 setOperationAction(ISD::FSIN , MVT::f64, Expand); 162 setOperationAction(ISD::FCOS , MVT::f64, Expand); 163 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 164 setOperationAction(ISD::FREM , MVT::f64, Expand); 165 setOperationAction(ISD::FPOW , MVT::f64, Expand); 166 setOperationAction(ISD::FMA , MVT::f64, Legal); 167 setOperationAction(ISD::FSIN , MVT::f32, Expand); 168 setOperationAction(ISD::FCOS , MVT::f32, Expand); 169 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 170 setOperationAction(ISD::FREM , MVT::f32, Expand); 171 setOperationAction(ISD::FPOW , MVT::f32, Expand); 172 setOperationAction(ISD::FMA , MVT::f32, Legal); 173 174 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 175 176 // If we're enabling GP optimizations, use hardware square root 177 if (!Subtarget.hasFSQRT() && 178 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() && 179 Subtarget.hasFRE())) 180 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 181 182 if (!Subtarget.hasFSQRT() && 183 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() && 184 Subtarget.hasFRES())) 185 setOperationAction(ISD::FSQRT, MVT::f32, Expand); 186 187 if (Subtarget.hasFCPSGN()) { 188 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); 189 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); 190 } else { 191 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 192 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 193 } 194 195 if (Subtarget.hasFPRND()) { 196 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 197 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 198 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 199 setOperationAction(ISD::FROUND, MVT::f64, Legal); 200 201 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 202 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 203 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 204 setOperationAction(ISD::FROUND, MVT::f32, Legal); 205 } 206 207 // PowerPC does not have BSWAP, CTPOP or CTTZ 208 setOperationAction(ISD::BSWAP, MVT::i32 , Expand); 209 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 210 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); 211 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); 212 setOperationAction(ISD::BSWAP, MVT::i64 , Expand); 213 setOperationAction(ISD::CTTZ , MVT::i64 , Expand); 214 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); 215 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); 216 217 if (Subtarget.hasPOPCNTD()) { 218 setOperationAction(ISD::CTPOP, MVT::i32 , Legal); 219 setOperationAction(ISD::CTPOP, MVT::i64 , Legal); 220 } else { 221 setOperationAction(ISD::CTPOP, MVT::i32 , Expand); 222 setOperationAction(ISD::CTPOP, MVT::i64 , Expand); 223 } 224 225 // PowerPC does not have ROTR 226 setOperationAction(ISD::ROTR, MVT::i32 , Expand); 227 setOperationAction(ISD::ROTR, MVT::i64 , Expand); 228 229 if (!Subtarget.useCRBits()) { 230 // PowerPC does not have Select 231 setOperationAction(ISD::SELECT, MVT::i32, Expand); 232 setOperationAction(ISD::SELECT, MVT::i64, Expand); 233 setOperationAction(ISD::SELECT, MVT::f32, Expand); 234 setOperationAction(ISD::SELECT, MVT::f64, Expand); 235 } 236 237 // PowerPC wants to turn select_cc of FP into fsel when possible. 238 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 239 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 240 241 // PowerPC wants to optimize integer setcc a bit 242 if (!Subtarget.useCRBits()) 243 setOperationAction(ISD::SETCC, MVT::i32, Custom); 244 245 // PowerPC does not have BRCOND which requires SetCC 246 if (!Subtarget.useCRBits()) 247 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 248 249 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 250 251 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. 252 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 253 254 // PowerPC does not have [U|S]INT_TO_FP 255 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); 256 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); 257 258 if (Subtarget.hasDirectMove()) { 259 setOperationAction(ISD::BITCAST, MVT::f32, Legal); 260 setOperationAction(ISD::BITCAST, MVT::i32, Legal); 261 setOperationAction(ISD::BITCAST, MVT::i64, Legal); 262 setOperationAction(ISD::BITCAST, MVT::f64, Legal); 263 } else { 264 setOperationAction(ISD::BITCAST, MVT::f32, Expand); 265 setOperationAction(ISD::BITCAST, MVT::i32, Expand); 266 setOperationAction(ISD::BITCAST, MVT::i64, Expand); 267 setOperationAction(ISD::BITCAST, MVT::f64, Expand); 268 } 269 270 // We cannot sextinreg(i1). Expand to shifts. 271 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 272 273 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support 274 // SjLj exception handling but a light-weight setjmp/longjmp replacement to 275 // support continuation, user-level threading, and etc.. As a result, no 276 // other SjLj exception interfaces are implemented and please don't build 277 // your own exception handling based on them. 278 // LLVM/Clang supports zero-cost DWARF exception handling. 279 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 280 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 281 282 // We want to legalize GlobalAddress and ConstantPool nodes into the 283 // appropriate instructions to materialize the address. 284 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 285 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 286 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 287 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 288 setOperationAction(ISD::JumpTable, MVT::i32, Custom); 289 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 290 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 291 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); 292 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); 293 setOperationAction(ISD::JumpTable, MVT::i64, Custom); 294 295 // TRAP is legal. 296 setOperationAction(ISD::TRAP, MVT::Other, Legal); 297 298 // TRAMPOLINE is custom lowered. 299 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 300 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 301 302 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 303 setOperationAction(ISD::VASTART , MVT::Other, Custom); 304 305 if (Subtarget.isSVR4ABI()) { 306 if (isPPC64) { 307 // VAARG always uses double-word chunks, so promote anything smaller. 308 setOperationAction(ISD::VAARG, MVT::i1, Promote); 309 AddPromotedToType (ISD::VAARG, MVT::i1, MVT::i64); 310 setOperationAction(ISD::VAARG, MVT::i8, Promote); 311 AddPromotedToType (ISD::VAARG, MVT::i8, MVT::i64); 312 setOperationAction(ISD::VAARG, MVT::i16, Promote); 313 AddPromotedToType (ISD::VAARG, MVT::i16, MVT::i64); 314 setOperationAction(ISD::VAARG, MVT::i32, Promote); 315 AddPromotedToType (ISD::VAARG, MVT::i32, MVT::i64); 316 setOperationAction(ISD::VAARG, MVT::Other, Expand); 317 } else { 318 // VAARG is custom lowered with the 32-bit SVR4 ABI. 319 setOperationAction(ISD::VAARG, MVT::Other, Custom); 320 setOperationAction(ISD::VAARG, MVT::i64, Custom); 321 } 322 } else 323 setOperationAction(ISD::VAARG, MVT::Other, Expand); 324 325 if (Subtarget.isSVR4ABI() && !isPPC64) 326 // VACOPY is custom lowered with the 32-bit SVR4 ABI. 327 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 328 else 329 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 330 331 // Use the default implementation. 332 setOperationAction(ISD::VAEND , MVT::Other, Expand); 333 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); 334 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); 335 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); 336 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); 337 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom); 338 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom); 339 340 // We want to custom lower some of our intrinsics. 341 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 342 343 // To handle counter-based loop conditions. 344 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); 345 346 // Comparisons that require checking two conditions. 347 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 348 setCondCodeAction(ISD::SETULT, MVT::f64, Expand); 349 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 350 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); 351 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 352 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); 353 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); 354 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); 355 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 356 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); 357 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 358 setCondCodeAction(ISD::SETONE, MVT::f64, Expand); 359 360 if (Subtarget.has64BitSupport()) { 361 // They also have instructions for converting between i64 and fp. 362 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 363 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); 364 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 365 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 366 // This is just the low 32 bits of a (signed) fp->i64 conversion. 367 // We cannot do this with Promote because i64 is not a legal type. 368 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 369 370 if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) 371 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 372 } else { 373 // PowerPC does not have FP_TO_UINT on 32-bit implementations. 374 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); 375 } 376 377 // With the instructions enabled under FPCVT, we can do everything. 378 if (Subtarget.hasFPCVT()) { 379 if (Subtarget.has64BitSupport()) { 380 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 381 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 382 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 383 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 384 } 385 386 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 387 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 388 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 389 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 390 } 391 392 if (Subtarget.use64BitRegs()) { 393 // 64-bit PowerPC implementations can support i64 types directly 394 addRegisterClass(MVT::i64, &PPC::G8RCRegClass); 395 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or 396 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 397 // 64-bit PowerPC wants to expand i128 shifts itself. 398 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); 399 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); 400 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); 401 } else { 402 // 32-bit PowerPC wants to expand i64 shifts itself. 403 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 404 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 405 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 406 } 407 408 if (Subtarget.hasAltivec()) { 409 // First set operation action for all vector types to expand. Then we 410 // will selectively turn on ones that can be effectively codegen'd. 411 for (MVT VT : MVT::vector_valuetypes()) { 412 // add/sub are legal for all supported vector VT's. 413 setOperationAction(ISD::ADD, VT, Legal); 414 setOperationAction(ISD::SUB, VT, Legal); 415 416 // Vector instructions introduced in P8 417 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) { 418 setOperationAction(ISD::CTPOP, VT, Legal); 419 setOperationAction(ISD::CTLZ, VT, Legal); 420 } 421 else { 422 setOperationAction(ISD::CTPOP, VT, Expand); 423 setOperationAction(ISD::CTLZ, VT, Expand); 424 } 425 426 // We promote all shuffles to v16i8. 427 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); 428 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); 429 430 // We promote all non-typed operations to v4i32. 431 setOperationAction(ISD::AND , VT, Promote); 432 AddPromotedToType (ISD::AND , VT, MVT::v4i32); 433 setOperationAction(ISD::OR , VT, Promote); 434 AddPromotedToType (ISD::OR , VT, MVT::v4i32); 435 setOperationAction(ISD::XOR , VT, Promote); 436 AddPromotedToType (ISD::XOR , VT, MVT::v4i32); 437 setOperationAction(ISD::LOAD , VT, Promote); 438 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); 439 setOperationAction(ISD::SELECT, VT, Promote); 440 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); 441 setOperationAction(ISD::SELECT_CC, VT, Promote); 442 AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32); 443 setOperationAction(ISD::STORE, VT, Promote); 444 AddPromotedToType (ISD::STORE, VT, MVT::v4i32); 445 446 // No other operations are legal. 447 setOperationAction(ISD::MUL , VT, Expand); 448 setOperationAction(ISD::SDIV, VT, Expand); 449 setOperationAction(ISD::SREM, VT, Expand); 450 setOperationAction(ISD::UDIV, VT, Expand); 451 setOperationAction(ISD::UREM, VT, Expand); 452 setOperationAction(ISD::FDIV, VT, Expand); 453 setOperationAction(ISD::FREM, VT, Expand); 454 setOperationAction(ISD::FNEG, VT, Expand); 455 setOperationAction(ISD::FSQRT, VT, Expand); 456 setOperationAction(ISD::FLOG, VT, Expand); 457 setOperationAction(ISD::FLOG10, VT, Expand); 458 setOperationAction(ISD::FLOG2, VT, Expand); 459 setOperationAction(ISD::FEXP, VT, Expand); 460 setOperationAction(ISD::FEXP2, VT, Expand); 461 setOperationAction(ISD::FSIN, VT, Expand); 462 setOperationAction(ISD::FCOS, VT, Expand); 463 setOperationAction(ISD::FABS, VT, Expand); 464 setOperationAction(ISD::FPOWI, VT, Expand); 465 setOperationAction(ISD::FFLOOR, VT, Expand); 466 setOperationAction(ISD::FCEIL, VT, Expand); 467 setOperationAction(ISD::FTRUNC, VT, Expand); 468 setOperationAction(ISD::FRINT, VT, Expand); 469 setOperationAction(ISD::FNEARBYINT, VT, Expand); 470 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); 471 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); 472 setOperationAction(ISD::BUILD_VECTOR, VT, Expand); 473 setOperationAction(ISD::MULHU, VT, Expand); 474 setOperationAction(ISD::MULHS, VT, Expand); 475 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 476 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 477 setOperationAction(ISD::UDIVREM, VT, Expand); 478 setOperationAction(ISD::SDIVREM, VT, Expand); 479 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); 480 setOperationAction(ISD::FPOW, VT, Expand); 481 setOperationAction(ISD::BSWAP, VT, Expand); 482 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 483 setOperationAction(ISD::CTTZ, VT, Expand); 484 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 485 setOperationAction(ISD::VSELECT, VT, Expand); 486 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 487 setOperationAction(ISD::ROTL, VT, Expand); 488 setOperationAction(ISD::ROTR, VT, Expand); 489 490 for (MVT InnerVT : MVT::vector_valuetypes()) { 491 setTruncStoreAction(VT, InnerVT, Expand); 492 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 493 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 494 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 495 } 496 } 497 498 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle 499 // with merges, splats, etc. 500 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); 501 502 setOperationAction(ISD::AND , MVT::v4i32, Legal); 503 setOperationAction(ISD::OR , MVT::v4i32, Legal); 504 setOperationAction(ISD::XOR , MVT::v4i32, Legal); 505 setOperationAction(ISD::LOAD , MVT::v4i32, Legal); 506 setOperationAction(ISD::SELECT, MVT::v4i32, 507 Subtarget.useCRBits() ? Legal : Expand); 508 setOperationAction(ISD::STORE , MVT::v4i32, Legal); 509 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 510 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 511 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 512 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 513 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 514 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 515 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 516 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 517 518 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); 519 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); 520 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); 521 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); 522 523 setOperationAction(ISD::MUL, MVT::v4f32, Legal); 524 setOperationAction(ISD::FMA, MVT::v4f32, Legal); 525 526 if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) { 527 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 528 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 529 } 530 531 if (Subtarget.hasP8Altivec()) 532 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 533 else 534 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 535 536 setOperationAction(ISD::MUL, MVT::v8i16, Custom); 537 setOperationAction(ISD::MUL, MVT::v16i8, Custom); 538 539 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); 540 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); 541 542 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); 543 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); 544 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); 545 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 546 547 // Altivec does not contain unordered floating-point compare instructions 548 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); 549 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); 550 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); 551 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); 552 553 if (Subtarget.hasVSX()) { 554 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 555 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 556 if (Subtarget.hasP8Vector()) { 557 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 558 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal); 559 } 560 if (Subtarget.hasDirectMove()) { 561 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal); 562 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal); 563 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal); 564 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal); 565 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal); 566 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal); 567 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 568 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 569 } 570 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 571 572 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 573 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 574 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 575 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 576 setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 577 578 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 579 580 setOperationAction(ISD::MUL, MVT::v2f64, Legal); 581 setOperationAction(ISD::FMA, MVT::v2f64, Legal); 582 583 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 584 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 585 586 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); 587 setOperationAction(ISD::VSELECT, MVT::v8i16, Legal); 588 setOperationAction(ISD::VSELECT, MVT::v4i32, Legal); 589 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); 590 setOperationAction(ISD::VSELECT, MVT::v2f64, Legal); 591 592 // Share the Altivec comparison restrictions. 593 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); 594 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); 595 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); 596 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); 597 598 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 599 setOperationAction(ISD::STORE, MVT::v2f64, Legal); 600 601 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); 602 603 if (Subtarget.hasP8Vector()) 604 addRegisterClass(MVT::f32, &PPC::VSSRCRegClass); 605 606 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); 607 608 addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass); 609 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); 610 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); 611 612 if (Subtarget.hasP8Altivec()) { 613 setOperationAction(ISD::SHL, MVT::v2i64, Legal); 614 setOperationAction(ISD::SRA, MVT::v2i64, Legal); 615 setOperationAction(ISD::SRL, MVT::v2i64, Legal); 616 617 setOperationAction(ISD::SETCC, MVT::v2i64, Legal); 618 } 619 else { 620 setOperationAction(ISD::SHL, MVT::v2i64, Expand); 621 setOperationAction(ISD::SRA, MVT::v2i64, Expand); 622 setOperationAction(ISD::SRL, MVT::v2i64, Expand); 623 624 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 625 626 // VSX v2i64 only supports non-arithmetic operations. 627 setOperationAction(ISD::ADD, MVT::v2i64, Expand); 628 setOperationAction(ISD::SUB, MVT::v2i64, Expand); 629 } 630 631 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); 632 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); 633 setOperationAction(ISD::STORE, MVT::v2i64, Promote); 634 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); 635 636 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); 637 638 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 639 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 640 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 641 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 642 643 // Vector operation legalization checks the result type of 644 // SIGN_EXTEND_INREG, overall legalization checks the inner type. 645 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal); 646 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); 647 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom); 648 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom); 649 650 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); 651 } 652 653 if (Subtarget.hasP8Altivec()) { 654 addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass); 655 addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass); 656 } 657 } 658 659 if (Subtarget.hasQPX()) { 660 setOperationAction(ISD::FADD, MVT::v4f64, Legal); 661 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); 662 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); 663 setOperationAction(ISD::FREM, MVT::v4f64, Expand); 664 665 setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal); 666 setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand); 667 668 setOperationAction(ISD::LOAD , MVT::v4f64, Custom); 669 setOperationAction(ISD::STORE , MVT::v4f64, Custom); 670 671 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom); 672 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom); 673 674 if (!Subtarget.useCRBits()) 675 setOperationAction(ISD::SELECT, MVT::v4f64, Expand); 676 setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); 677 678 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal); 679 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand); 680 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand); 681 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand); 682 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom); 683 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal); 684 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); 685 686 setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal); 687 setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand); 688 689 setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal); 690 setOperationAction(ISD::FP_ROUND_INREG , MVT::v4f32, Expand); 691 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal); 692 693 setOperationAction(ISD::FNEG , MVT::v4f64, Legal); 694 setOperationAction(ISD::FABS , MVT::v4f64, Legal); 695 setOperationAction(ISD::FSIN , MVT::v4f64, Expand); 696 setOperationAction(ISD::FCOS , MVT::v4f64, Expand); 697 setOperationAction(ISD::FPOWI , MVT::v4f64, Expand); 698 setOperationAction(ISD::FPOW , MVT::v4f64, Expand); 699 setOperationAction(ISD::FLOG , MVT::v4f64, Expand); 700 setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand); 701 setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand); 702 setOperationAction(ISD::FEXP , MVT::v4f64, Expand); 703 setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand); 704 705 setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal); 706 setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal); 707 708 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal); 709 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal); 710 711 addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass); 712 713 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 714 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 715 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 716 setOperationAction(ISD::FREM, MVT::v4f32, Expand); 717 718 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); 719 setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand); 720 721 setOperationAction(ISD::LOAD , MVT::v4f32, Custom); 722 setOperationAction(ISD::STORE , MVT::v4f32, Custom); 723 724 if (!Subtarget.useCRBits()) 725 setOperationAction(ISD::SELECT, MVT::v4f32, Expand); 726 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); 727 728 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal); 729 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand); 730 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand); 731 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand); 732 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom); 733 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 734 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 735 736 setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal); 737 setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand); 738 739 setOperationAction(ISD::FNEG , MVT::v4f32, Legal); 740 setOperationAction(ISD::FABS , MVT::v4f32, Legal); 741 setOperationAction(ISD::FSIN , MVT::v4f32, Expand); 742 setOperationAction(ISD::FCOS , MVT::v4f32, Expand); 743 setOperationAction(ISD::FPOWI , MVT::v4f32, Expand); 744 setOperationAction(ISD::FPOW , MVT::v4f32, Expand); 745 setOperationAction(ISD::FLOG , MVT::v4f32, Expand); 746 setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand); 747 setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand); 748 setOperationAction(ISD::FEXP , MVT::v4f32, Expand); 749 setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand); 750 751 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); 752 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); 753 754 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal); 755 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal); 756 757 addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass); 758 759 setOperationAction(ISD::AND , MVT::v4i1, Legal); 760 setOperationAction(ISD::OR , MVT::v4i1, Legal); 761 setOperationAction(ISD::XOR , MVT::v4i1, Legal); 762 763 if (!Subtarget.useCRBits()) 764 setOperationAction(ISD::SELECT, MVT::v4i1, Expand); 765 setOperationAction(ISD::VSELECT, MVT::v4i1, Legal); 766 767 setOperationAction(ISD::LOAD , MVT::v4i1, Custom); 768 setOperationAction(ISD::STORE , MVT::v4i1, Custom); 769 770 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom); 771 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand); 772 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand); 773 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand); 774 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom); 775 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand); 776 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); 777 778 setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom); 779 setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom); 780 781 addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass); 782 783 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); 784 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); 785 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); 786 setOperationAction(ISD::FROUND, MVT::v4f64, Legal); 787 788 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 789 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 790 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 791 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 792 793 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand); 794 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); 795 796 // These need to set FE_INEXACT, and so cannot be vectorized here. 797 setOperationAction(ISD::FRINT, MVT::v4f64, Expand); 798 setOperationAction(ISD::FRINT, MVT::v4f32, Expand); 799 800 if (TM.Options.UnsafeFPMath) { 801 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); 802 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); 803 804 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 805 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 806 } else { 807 setOperationAction(ISD::FDIV, MVT::v4f64, Expand); 808 setOperationAction(ISD::FSQRT, MVT::v4f64, Expand); 809 810 setOperationAction(ISD::FDIV, MVT::v4f32, Expand); 811 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); 812 } 813 } 814 815 if (Subtarget.has64BitSupport()) 816 setOperationAction(ISD::PREFETCH, MVT::Other, Legal); 817 818 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); 819 820 if (!isPPC64) { 821 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); 822 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); 823 } 824 825 setBooleanContents(ZeroOrOneBooleanContent); 826 827 if (Subtarget.hasAltivec()) { 828 // Altivec instructions set fields to all zeros or all ones. 829 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 830 } 831 832 if (!isPPC64) { 833 // These libcalls are not available in 32-bit. 834 setLibcallName(RTLIB::SHL_I128, nullptr); 835 setLibcallName(RTLIB::SRL_I128, nullptr); 836 setLibcallName(RTLIB::SRA_I128, nullptr); 837 } 838 839 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); 840 841 // We have target-specific dag combine patterns for the following nodes: 842 setTargetDAGCombine(ISD::SINT_TO_FP); 843 if (Subtarget.hasFPCVT()) 844 setTargetDAGCombine(ISD::UINT_TO_FP); 845 setTargetDAGCombine(ISD::LOAD); 846 setTargetDAGCombine(ISD::STORE); 847 setTargetDAGCombine(ISD::BR_CC); 848 if (Subtarget.useCRBits()) 849 setTargetDAGCombine(ISD::BRCOND); 850 setTargetDAGCombine(ISD::BSWAP); 851 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 852 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 853 setTargetDAGCombine(ISD::INTRINSIC_VOID); 854 855 setTargetDAGCombine(ISD::SIGN_EXTEND); 856 setTargetDAGCombine(ISD::ZERO_EXTEND); 857 setTargetDAGCombine(ISD::ANY_EXTEND); 858 859 if (Subtarget.useCRBits()) { 860 setTargetDAGCombine(ISD::TRUNCATE); 861 setTargetDAGCombine(ISD::SETCC); 862 setTargetDAGCombine(ISD::SELECT_CC); 863 } 864 865 // Use reciprocal estimates. 866 if (TM.Options.UnsafeFPMath) { 867 setTargetDAGCombine(ISD::FDIV); 868 setTargetDAGCombine(ISD::FSQRT); 869 } 870 871 // Darwin long double math library functions have $LDBL128 appended. 872 if (Subtarget.isDarwin()) { 873 setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128"); 874 setLibcallName(RTLIB::POW_PPCF128, "powl$LDBL128"); 875 setLibcallName(RTLIB::REM_PPCF128, "fmodl$LDBL128"); 876 setLibcallName(RTLIB::SIN_PPCF128, "sinl$LDBL128"); 877 setLibcallName(RTLIB::SQRT_PPCF128, "sqrtl$LDBL128"); 878 setLibcallName(RTLIB::LOG_PPCF128, "logl$LDBL128"); 879 setLibcallName(RTLIB::LOG2_PPCF128, "log2l$LDBL128"); 880 setLibcallName(RTLIB::LOG10_PPCF128, "log10l$LDBL128"); 881 setLibcallName(RTLIB::EXP_PPCF128, "expl$LDBL128"); 882 setLibcallName(RTLIB::EXP2_PPCF128, "exp2l$LDBL128"); 883 } 884 885 // With 32 condition bits, we don't need to sink (and duplicate) compares 886 // aggressively in CodeGenPrep. 887 if (Subtarget.useCRBits()) { 888 setHasMultipleConditionRegisters(); 889 setJumpIsExpensive(); 890 } 891 892 setMinFunctionAlignment(2); 893 if (Subtarget.isDarwin()) 894 setPrefFunctionAlignment(4); 895 896 switch (Subtarget.getDarwinDirective()) { 897 default: break; 898 case PPC::DIR_970: 899 case PPC::DIR_A2: 900 case PPC::DIR_E500mc: 901 case PPC::DIR_E5500: 902 case PPC::DIR_PWR4: 903 case PPC::DIR_PWR5: 904 case PPC::DIR_PWR5X: 905 case PPC::DIR_PWR6: 906 case PPC::DIR_PWR6X: 907 case PPC::DIR_PWR7: 908 case PPC::DIR_PWR8: 909 setPrefFunctionAlignment(4); 910 setPrefLoopAlignment(4); 911 break; 912 } 913 914 setInsertFencesForAtomic(true); 915 916 if (Subtarget.enableMachineScheduler()) 917 setSchedulingPreference(Sched::Source); 918 else 919 setSchedulingPreference(Sched::Hybrid); 920 921 computeRegisterProperties(STI.getRegisterInfo()); 922 923 // The Freescale cores do better with aggressive inlining of memcpy and 924 // friends. GCC uses same threshold of 128 bytes (= 32 word stores). 925 if (Subtarget.getDarwinDirective() == PPC::DIR_E500mc || 926 Subtarget.getDarwinDirective() == PPC::DIR_E5500) { 927 MaxStoresPerMemset = 32; 928 MaxStoresPerMemsetOptSize = 16; 929 MaxStoresPerMemcpy = 32; 930 MaxStoresPerMemcpyOptSize = 8; 931 MaxStoresPerMemmove = 32; 932 MaxStoresPerMemmoveOptSize = 8; 933 } else if (Subtarget.getDarwinDirective() == PPC::DIR_A2) { 934 // The A2 also benefits from (very) aggressive inlining of memcpy and 935 // friends. The overhead of a the function call, even when warm, can be 936 // over one hundred cycles. 937 MaxStoresPerMemset = 128; 938 MaxStoresPerMemcpy = 128; 939 MaxStoresPerMemmove = 128; 940 } 941 } 942 943 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine 944 /// the desired ByVal argument alignment. 945 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign, 946 unsigned MaxMaxAlign) { 947 if (MaxAlign == MaxMaxAlign) 948 return; 949 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 950 if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256) 951 MaxAlign = 32; 952 else if (VTy->getBitWidth() >= 128 && MaxAlign < 16) 953 MaxAlign = 16; 954 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 955 unsigned EltAlign = 0; 956 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); 957 if (EltAlign > MaxAlign) 958 MaxAlign = EltAlign; 959 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 960 for (auto *EltTy : STy->elements()) { 961 unsigned EltAlign = 0; 962 getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign); 963 if (EltAlign > MaxAlign) 964 MaxAlign = EltAlign; 965 if (MaxAlign == MaxMaxAlign) 966 break; 967 } 968 } 969 } 970 971 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 972 /// function arguments in the caller parameter area. 973 unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty, 974 const DataLayout &DL) const { 975 // Darwin passes everything on 4 byte boundary. 976 if (Subtarget.isDarwin()) 977 return 4; 978 979 // 16byte and wider vectors are passed on 16byte boundary. 980 // The rest is 8 on PPC64 and 4 on PPC32 boundary. 981 unsigned Align = Subtarget.isPPC64() ? 8 : 4; 982 if (Subtarget.hasAltivec() || Subtarget.hasQPX()) 983 getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16); 984 return Align; 985 } 986 987 bool PPCTargetLowering::useSoftFloat() const { 988 return Subtarget.useSoftFloat(); 989 } 990 991 const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { 992 switch ((PPCISD::NodeType)Opcode) { 993 case PPCISD::FIRST_NUMBER: break; 994 case PPCISD::FSEL: return "PPCISD::FSEL"; 995 case PPCISD::FCFID: return "PPCISD::FCFID"; 996 case PPCISD::FCFIDU: return "PPCISD::FCFIDU"; 997 case PPCISD::FCFIDS: return "PPCISD::FCFIDS"; 998 case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS"; 999 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; 1000 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; 1001 case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ"; 1002 case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ"; 1003 case PPCISD::FRE: return "PPCISD::FRE"; 1004 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; 1005 case PPCISD::STFIWX: return "PPCISD::STFIWX"; 1006 case PPCISD::VMADDFP: return "PPCISD::VMADDFP"; 1007 case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP"; 1008 case PPCISD::VPERM: return "PPCISD::VPERM"; 1009 case PPCISD::CMPB: return "PPCISD::CMPB"; 1010 case PPCISD::Hi: return "PPCISD::Hi"; 1011 case PPCISD::Lo: return "PPCISD::Lo"; 1012 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; 1013 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; 1014 case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET"; 1015 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; 1016 case PPCISD::SRL: return "PPCISD::SRL"; 1017 case PPCISD::SRA: return "PPCISD::SRA"; 1018 case PPCISD::SHL: return "PPCISD::SHL"; 1019 case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE"; 1020 case PPCISD::CALL: return "PPCISD::CALL"; 1021 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; 1022 case PPCISD::MTCTR: return "PPCISD::MTCTR"; 1023 case PPCISD::BCTRL: return "PPCISD::BCTRL"; 1024 case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC"; 1025 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; 1026 case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE"; 1027 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; 1028 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; 1029 case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; 1030 case PPCISD::MFVSR: return "PPCISD::MFVSR"; 1031 case PPCISD::MTVSRA: return "PPCISD::MTVSRA"; 1032 case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ"; 1033 case PPCISD::ANDIo_1_EQ_BIT: return "PPCISD::ANDIo_1_EQ_BIT"; 1034 case PPCISD::ANDIo_1_GT_BIT: return "PPCISD::ANDIo_1_GT_BIT"; 1035 case PPCISD::VCMP: return "PPCISD::VCMP"; 1036 case PPCISD::VCMPo: return "PPCISD::VCMPo"; 1037 case PPCISD::LBRX: return "PPCISD::LBRX"; 1038 case PPCISD::STBRX: return "PPCISD::STBRX"; 1039 case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; 1040 case PPCISD::LFIWZX: return "PPCISD::LFIWZX"; 1041 case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; 1042 case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; 1043 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; 1044 case PPCISD::BDNZ: return "PPCISD::BDNZ"; 1045 case PPCISD::BDZ: return "PPCISD::BDZ"; 1046 case PPCISD::MFFS: return "PPCISD::MFFS"; 1047 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; 1048 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; 1049 case PPCISD::CR6SET: return "PPCISD::CR6SET"; 1050 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; 1051 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; 1052 case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT"; 1053 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; 1054 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; 1055 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; 1056 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; 1057 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; 1058 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; 1059 case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; 1060 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; 1061 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; 1062 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; 1063 case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; 1064 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; 1065 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; 1066 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; 1067 case PPCISD::SC: return "PPCISD::SC"; 1068 case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; 1069 case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE"; 1070 case PPCISD::RFEBB: return "PPCISD::RFEBB"; 1071 case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; 1072 case PPCISD::QVFPERM: return "PPCISD::QVFPERM"; 1073 case PPCISD::QVGPCI: return "PPCISD::QVGPCI"; 1074 case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI"; 1075 case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI"; 1076 case PPCISD::QBFLT: return "PPCISD::QBFLT"; 1077 case PPCISD::QVLFSb: return "PPCISD::QVLFSb"; 1078 } 1079 return nullptr; 1080 } 1081 1082 EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, 1083 EVT VT) const { 1084 if (!VT.isVector()) 1085 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; 1086 1087 if (Subtarget.hasQPX()) 1088 return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); 1089 1090 return VT.changeVectorElementTypeToInteger(); 1091 } 1092 1093 bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const { 1094 assert(VT.isFloatingPoint() && "Non-floating-point FMA?"); 1095 return true; 1096 } 1097 1098 //===----------------------------------------------------------------------===// 1099 // Node matching predicates, for use by the tblgen matching code. 1100 //===----------------------------------------------------------------------===// 1101 1102 /// isFloatingPointZero - Return true if this is 0.0 or -0.0. 1103 static bool isFloatingPointZero(SDValue Op) { 1104 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 1105 return CFP->getValueAPF().isZero(); 1106 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 1107 // Maybe this has already been legalized into the constant pool? 1108 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1))) 1109 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 1110 return CFP->getValueAPF().isZero(); 1111 } 1112 return false; 1113 } 1114 1115 /// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return 1116 /// true if Op is undef or if it matches the specified value. 1117 static bool isConstantOrUndef(int Op, int Val) { 1118 return Op < 0 || Op == Val; 1119 } 1120 1121 /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a 1122 /// VPKUHUM instruction. 1123 /// The ShuffleKind distinguishes between big-endian operations with 1124 /// two different inputs (0), either-endian operations with two identical 1125 /// inputs (1), and little-endian operations with two different inputs (2). 1126 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1127 bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1128 SelectionDAG &DAG) { 1129 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1130 if (ShuffleKind == 0) { 1131 if (IsLE) 1132 return false; 1133 for (unsigned i = 0; i != 16; ++i) 1134 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) 1135 return false; 1136 } else if (ShuffleKind == 2) { 1137 if (!IsLE) 1138 return false; 1139 for (unsigned i = 0; i != 16; ++i) 1140 if (!isConstantOrUndef(N->getMaskElt(i), i*2)) 1141 return false; 1142 } else if (ShuffleKind == 1) { 1143 unsigned j = IsLE ? 0 : 1; 1144 for (unsigned i = 0; i != 8; ++i) 1145 if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) || 1146 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j)) 1147 return false; 1148 } 1149 return true; 1150 } 1151 1152 /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a 1153 /// VPKUWUM instruction. 1154 /// The ShuffleKind distinguishes between big-endian operations with 1155 /// two different inputs (0), either-endian operations with two identical 1156 /// inputs (1), and little-endian operations with two different inputs (2). 1157 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1158 bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1159 SelectionDAG &DAG) { 1160 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1161 if (ShuffleKind == 0) { 1162 if (IsLE) 1163 return false; 1164 for (unsigned i = 0; i != 16; i += 2) 1165 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || 1166 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) 1167 return false; 1168 } else if (ShuffleKind == 2) { 1169 if (!IsLE) 1170 return false; 1171 for (unsigned i = 0; i != 16; i += 2) 1172 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1173 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1)) 1174 return false; 1175 } else if (ShuffleKind == 1) { 1176 unsigned j = IsLE ? 0 : 2; 1177 for (unsigned i = 0; i != 8; i += 2) 1178 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1179 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1180 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1181 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1)) 1182 return false; 1183 } 1184 return true; 1185 } 1186 1187 /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a 1188 /// VPKUDUM instruction, AND the VPKUDUM instruction exists for the 1189 /// current subtarget. 1190 /// 1191 /// The ShuffleKind distinguishes between big-endian operations with 1192 /// two different inputs (0), either-endian operations with two identical 1193 /// inputs (1), and little-endian operations with two different inputs (2). 1194 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1195 bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1196 SelectionDAG &DAG) { 1197 const PPCSubtarget& Subtarget = 1198 static_cast<const PPCSubtarget&>(DAG.getSubtarget()); 1199 if (!Subtarget.hasP8Vector()) 1200 return false; 1201 1202 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1203 if (ShuffleKind == 0) { 1204 if (IsLE) 1205 return false; 1206 for (unsigned i = 0; i != 16; i += 4) 1207 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) || 1208 !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) || 1209 !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) || 1210 !isConstantOrUndef(N->getMaskElt(i+3), i*2+7)) 1211 return false; 1212 } else if (ShuffleKind == 2) { 1213 if (!IsLE) 1214 return false; 1215 for (unsigned i = 0; i != 16; i += 4) 1216 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1217 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) || 1218 !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) || 1219 !isConstantOrUndef(N->getMaskElt(i+3), i*2+3)) 1220 return false; 1221 } else if (ShuffleKind == 1) { 1222 unsigned j = IsLE ? 0 : 4; 1223 for (unsigned i = 0; i != 8; i += 4) 1224 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1225 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1226 !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) || 1227 !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) || 1228 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1229 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) || 1230 !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) || 1231 !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3)) 1232 return false; 1233 } 1234 return true; 1235 } 1236 1237 /// isVMerge - Common function, used to match vmrg* shuffles. 1238 /// 1239 static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, 1240 unsigned LHSStart, unsigned RHSStart) { 1241 if (N->getValueType(0) != MVT::v16i8) 1242 return false; 1243 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && 1244 "Unsupported merge size!"); 1245 1246 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units 1247 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit 1248 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), 1249 LHSStart+j+i*UnitSize) || 1250 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), 1251 RHSStart+j+i*UnitSize)) 1252 return false; 1253 } 1254 return true; 1255 } 1256 1257 /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for 1258 /// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes). 1259 /// The ShuffleKind distinguishes between big-endian merges with two 1260 /// different inputs (0), either-endian merges with two identical inputs (1), 1261 /// and little-endian merges with two different inputs (2). For the latter, 1262 /// the input operands are swapped (see PPCInstrAltivec.td). 1263 bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1264 unsigned ShuffleKind, SelectionDAG &DAG) { 1265 if (DAG.getDataLayout().isLittleEndian()) { 1266 if (ShuffleKind == 1) // unary 1267 return isVMerge(N, UnitSize, 0, 0); 1268 else if (ShuffleKind == 2) // swapped 1269 return isVMerge(N, UnitSize, 0, 16); 1270 else 1271 return false; 1272 } else { 1273 if (ShuffleKind == 1) // unary 1274 return isVMerge(N, UnitSize, 8, 8); 1275 else if (ShuffleKind == 0) // normal 1276 return isVMerge(N, UnitSize, 8, 24); 1277 else 1278 return false; 1279 } 1280 } 1281 1282 /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for 1283 /// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes). 1284 /// The ShuffleKind distinguishes between big-endian merges with two 1285 /// different inputs (0), either-endian merges with two identical inputs (1), 1286 /// and little-endian merges with two different inputs (2). For the latter, 1287 /// the input operands are swapped (see PPCInstrAltivec.td). 1288 bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1289 unsigned ShuffleKind, SelectionDAG &DAG) { 1290 if (DAG.getDataLayout().isLittleEndian()) { 1291 if (ShuffleKind == 1) // unary 1292 return isVMerge(N, UnitSize, 8, 8); 1293 else if (ShuffleKind == 2) // swapped 1294 return isVMerge(N, UnitSize, 8, 24); 1295 else 1296 return false; 1297 } else { 1298 if (ShuffleKind == 1) // unary 1299 return isVMerge(N, UnitSize, 0, 0); 1300 else if (ShuffleKind == 0) // normal 1301 return isVMerge(N, UnitSize, 0, 16); 1302 else 1303 return false; 1304 } 1305 } 1306 1307 /** 1308 * \brief Common function used to match vmrgew and vmrgow shuffles 1309 * 1310 * The indexOffset determines whether to look for even or odd words in 1311 * the shuffle mask. This is based on the of the endianness of the target 1312 * machine. 1313 * - Little Endian: 1314 * - Use offset of 0 to check for odd elements 1315 * - Use offset of 4 to check for even elements 1316 * - Big Endian: 1317 * - Use offset of 0 to check for even elements 1318 * - Use offset of 4 to check for odd elements 1319 * A detailed description of the vector element ordering for little endian and 1320 * big endian can be found at 1321 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html 1322 * Targeting your applications - what little endian and big endian IBM XL C/C++ 1323 * compiler differences mean to you 1324 * 1325 * The mask to the shuffle vector instruction specifies the indices of the 1326 * elements from the two input vectors to place in the result. The elements are 1327 * numbered in array-access order, starting with the first vector. These vectors 1328 * are always of type v16i8, thus each vector will contain 16 elements of size 1329 * 8. More info on the shuffle vector can be found in the 1330 * http://llvm.org/docs/LangRef.html#shufflevector-instruction 1331 * Language Reference. 1332 * 1333 * The RHSStartValue indicates whether the same input vectors are used (unary) 1334 * or two different input vectors are used, based on the following: 1335 * - If the instruction uses the same vector for both inputs, the range of the 1336 * indices will be 0 to 15. In this case, the RHSStart value passed should 1337 * be 0. 1338 * - If the instruction has two different vectors then the range of the 1339 * indices will be 0 to 31. In this case, the RHSStart value passed should 1340 * be 16 (indices 0-15 specify elements in the first vector while indices 16 1341 * to 31 specify elements in the second vector). 1342 * 1343 * \param[in] N The shuffle vector SD Node to analyze 1344 * \param[in] IndexOffset Specifies whether to look for even or odd elements 1345 * \param[in] RHSStartValue Specifies the starting index for the righthand input 1346 * vector to the shuffle_vector instruction 1347 * \return true iff this shuffle vector represents an even or odd word merge 1348 */ 1349 static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset, 1350 unsigned RHSStartValue) { 1351 if (N->getValueType(0) != MVT::v16i8) 1352 return false; 1353 1354 for (unsigned i = 0; i < 2; ++i) 1355 for (unsigned j = 0; j < 4; ++j) 1356 if (!isConstantOrUndef(N->getMaskElt(i*4+j), 1357 i*RHSStartValue+j+IndexOffset) || 1358 !isConstantOrUndef(N->getMaskElt(i*4+j+8), 1359 i*RHSStartValue+j+IndexOffset+8)) 1360 return false; 1361 return true; 1362 } 1363 1364 /** 1365 * \brief Determine if the specified shuffle mask is suitable for the vmrgew or 1366 * vmrgow instructions. 1367 * 1368 * \param[in] N The shuffle vector SD Node to analyze 1369 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false) 1370 * \param[in] ShuffleKind Identify the type of merge: 1371 * - 0 = big-endian merge with two different inputs; 1372 * - 1 = either-endian merge with two identical inputs; 1373 * - 2 = little-endian merge with two different inputs (inputs are swapped for 1374 * little-endian merges). 1375 * \param[in] DAG The current SelectionDAG 1376 * \return true iff this shuffle mask 1377 */ 1378 bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, 1379 unsigned ShuffleKind, SelectionDAG &DAG) { 1380 if (DAG.getDataLayout().isLittleEndian()) { 1381 unsigned indexOffset = CheckEven ? 4 : 0; 1382 if (ShuffleKind == 1) // Unary 1383 return isVMerge(N, indexOffset, 0); 1384 else if (ShuffleKind == 2) // swapped 1385 return isVMerge(N, indexOffset, 16); 1386 else 1387 return false; 1388 } 1389 else { 1390 unsigned indexOffset = CheckEven ? 0 : 4; 1391 if (ShuffleKind == 1) // Unary 1392 return isVMerge(N, indexOffset, 0); 1393 else if (ShuffleKind == 0) // Normal 1394 return isVMerge(N, indexOffset, 16); 1395 else 1396 return false; 1397 } 1398 return false; 1399 } 1400 1401 /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift 1402 /// amount, otherwise return -1. 1403 /// The ShuffleKind distinguishes between big-endian operations with two 1404 /// different inputs (0), either-endian operations with two identical inputs 1405 /// (1), and little-endian operations with two different inputs (2). For the 1406 /// latter, the input operands are swapped (see PPCInstrAltivec.td). 1407 int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, 1408 SelectionDAG &DAG) { 1409 if (N->getValueType(0) != MVT::v16i8) 1410 return -1; 1411 1412 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 1413 1414 // Find the first non-undef value in the shuffle mask. 1415 unsigned i; 1416 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) 1417 /*search*/; 1418 1419 if (i == 16) return -1; // all undef. 1420 1421 // Otherwise, check to see if the rest of the elements are consecutively 1422 // numbered from this value. 1423 unsigned ShiftAmt = SVOp->getMaskElt(i); 1424 if (ShiftAmt < i) return -1; 1425 1426 ShiftAmt -= i; 1427 bool isLE = DAG.getDataLayout().isLittleEndian(); 1428 1429 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) { 1430 // Check the rest of the elements to see if they are consecutive. 1431 for (++i; i != 16; ++i) 1432 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 1433 return -1; 1434 } else if (ShuffleKind == 1) { 1435 // Check the rest of the elements to see if they are consecutive. 1436 for (++i; i != 16; ++i) 1437 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) 1438 return -1; 1439 } else 1440 return -1; 1441 1442 if (isLE) 1443 ShiftAmt = 16 - ShiftAmt; 1444 1445 return ShiftAmt; 1446 } 1447 1448 /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand 1449 /// specifies a splat of a single element that is suitable for input to 1450 /// VSPLTB/VSPLTH/VSPLTW. 1451 bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { 1452 assert(N->getValueType(0) == MVT::v16i8 && 1453 (EltSize == 1 || EltSize == 2 || EltSize == 4)); 1454 1455 // The consecutive indices need to specify an element, not part of two 1456 // different elements. So abandon ship early if this isn't the case. 1457 if (N->getMaskElt(0) % EltSize != 0) 1458 return false; 1459 1460 // This is a splat operation if each element of the permute is the same, and 1461 // if the value doesn't reference the second vector. 1462 unsigned ElementBase = N->getMaskElt(0); 1463 1464 // FIXME: Handle UNDEF elements too! 1465 if (ElementBase >= 16) 1466 return false; 1467 1468 // Check that the indices are consecutive, in the case of a multi-byte element 1469 // splatted with a v16i8 mask. 1470 for (unsigned i = 1; i != EltSize; ++i) 1471 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) 1472 return false; 1473 1474 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { 1475 if (N->getMaskElt(i) < 0) continue; 1476 for (unsigned j = 0; j != EltSize; ++j) 1477 if (N->getMaskElt(i+j) != N->getMaskElt(j)) 1478 return false; 1479 } 1480 return true; 1481 } 1482 1483 /// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the 1484 /// specified isSplatShuffleMask VECTOR_SHUFFLE mask. 1485 unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize, 1486 SelectionDAG &DAG) { 1487 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 1488 assert(isSplatShuffleMask(SVOp, EltSize)); 1489 if (DAG.getDataLayout().isLittleEndian()) 1490 return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize); 1491 else 1492 return SVOp->getMaskElt(0) / EltSize; 1493 } 1494 1495 /// get_VSPLTI_elt - If this is a build_vector of constants which can be formed 1496 /// by using a vspltis[bhw] instruction of the specified element size, return 1497 /// the constant being splatted. The ByteSize field indicates the number of 1498 /// bytes of each element [124] -> [bhw]. 1499 SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { 1500 SDValue OpVal(nullptr, 0); 1501 1502 // If ByteSize of the splat is bigger than the element size of the 1503 // build_vector, then we have a case where we are checking for a splat where 1504 // multiple elements of the buildvector are folded together into a single 1505 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). 1506 unsigned EltSize = 16/N->getNumOperands(); 1507 if (EltSize < ByteSize) { 1508 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. 1509 SDValue UniquedVals[4]; 1510 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); 1511 1512 // See if all of the elements in the buildvector agree across. 1513 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 1514 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1515 // If the element isn't a constant, bail fully out. 1516 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue(); 1517 1518 1519 if (!UniquedVals[i&(Multiple-1)].getNode()) 1520 UniquedVals[i&(Multiple-1)] = N->getOperand(i); 1521 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) 1522 return SDValue(); // no match. 1523 } 1524 1525 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains 1526 // either constant or undef values that are identical for each chunk. See 1527 // if these chunks can form into a larger vspltis*. 1528 1529 // Check to see if all of the leading entries are either 0 or -1. If 1530 // neither, then this won't fit into the immediate field. 1531 bool LeadingZero = true; 1532 bool LeadingOnes = true; 1533 for (unsigned i = 0; i != Multiple-1; ++i) { 1534 if (!UniquedVals[i].getNode()) continue; // Must have been undefs. 1535 1536 LeadingZero &= isNullConstant(UniquedVals[i]); 1537 LeadingOnes &= isAllOnesConstant(UniquedVals[i]); 1538 } 1539 // Finally, check the least significant entry. 1540 if (LeadingZero) { 1541 if (!UniquedVals[Multiple-1].getNode()) 1542 return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef 1543 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue(); 1544 if (Val < 16) // 0,0,0,4 -> vspltisw(4) 1545 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 1546 } 1547 if (LeadingOnes) { 1548 if (!UniquedVals[Multiple-1].getNode()) 1549 return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef 1550 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue(); 1551 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) 1552 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 1553 } 1554 1555 return SDValue(); 1556 } 1557 1558 // Check to see if this buildvec has a single non-undef value in its elements. 1559 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 1560 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1561 if (!OpVal.getNode()) 1562 OpVal = N->getOperand(i); 1563 else if (OpVal != N->getOperand(i)) 1564 return SDValue(); 1565 } 1566 1567 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. 1568 1569 unsigned ValSizeInBytes = EltSize; 1570 uint64_t Value = 0; 1571 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) { 1572 Value = CN->getZExtValue(); 1573 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) { 1574 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); 1575 Value = FloatToBits(CN->getValueAPF().convertToFloat()); 1576 } 1577 1578 // If the splat value is larger than the element value, then we can never do 1579 // this splat. The only case that we could fit the replicated bits into our 1580 // immediate field for would be zero, and we prefer to use vxor for it. 1581 if (ValSizeInBytes < ByteSize) return SDValue(); 1582 1583 // If the element value is larger than the splat value, check if it consists 1584 // of a repeated bit pattern of size ByteSize. 1585 if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8)) 1586 return SDValue(); 1587 1588 // Properly sign extend the value. 1589 int MaskVal = SignExtend32(Value, ByteSize * 8); 1590 1591 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. 1592 if (MaskVal == 0) return SDValue(); 1593 1594 // Finally, if this value fits in a 5 bit sext field, return it 1595 if (SignExtend32<5>(MaskVal) == MaskVal) 1596 return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32); 1597 return SDValue(); 1598 } 1599 1600 /// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift 1601 /// amount, otherwise return -1. 1602 int PPC::isQVALIGNIShuffleMask(SDNode *N) { 1603 EVT VT = N->getValueType(0); 1604 if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1) 1605 return -1; 1606 1607 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 1608 1609 // Find the first non-undef value in the shuffle mask. 1610 unsigned i; 1611 for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i) 1612 /*search*/; 1613 1614 if (i == 4) return -1; // all undef. 1615 1616 // Otherwise, check to see if the rest of the elements are consecutively 1617 // numbered from this value. 1618 unsigned ShiftAmt = SVOp->getMaskElt(i); 1619 if (ShiftAmt < i) return -1; 1620 ShiftAmt -= i; 1621 1622 // Check the rest of the elements to see if they are consecutive. 1623 for (++i; i != 4; ++i) 1624 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 1625 return -1; 1626 1627 return ShiftAmt; 1628 } 1629 1630 //===----------------------------------------------------------------------===// 1631 // Addressing Mode Selection 1632 //===----------------------------------------------------------------------===// 1633 1634 /// isIntS16Immediate - This method tests to see if the node is either a 32-bit 1635 /// or 64-bit immediate, and if the value can be accurately represented as a 1636 /// sign extension from a 16-bit value. If so, this returns true and the 1637 /// immediate. 1638 static bool isIntS16Immediate(SDNode *N, short &Imm) { 1639 if (!isa<ConstantSDNode>(N)) 1640 return false; 1641 1642 Imm = (short)cast<ConstantSDNode>(N)->getZExtValue(); 1643 if (N->getValueType(0) == MVT::i32) 1644 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue(); 1645 else 1646 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); 1647 } 1648 static bool isIntS16Immediate(SDValue Op, short &Imm) { 1649 return isIntS16Immediate(Op.getNode(), Imm); 1650 } 1651 1652 /// SelectAddressRegReg - Given the specified addressed, check to see if it 1653 /// can be represented as an indexed [r+r] operation. Returns false if it 1654 /// can be more efficiently represented with [r+imm]. 1655 bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, 1656 SDValue &Index, 1657 SelectionDAG &DAG) const { 1658 short imm = 0; 1659 if (N.getOpcode() == ISD::ADD) { 1660 if (isIntS16Immediate(N.getOperand(1), imm)) 1661 return false; // r+i 1662 if (N.getOperand(1).getOpcode() == PPCISD::Lo) 1663 return false; // r+i 1664 1665 Base = N.getOperand(0); 1666 Index = N.getOperand(1); 1667 return true; 1668 } else if (N.getOpcode() == ISD::OR) { 1669 if (isIntS16Immediate(N.getOperand(1), imm)) 1670 return false; // r+i can fold it if we can. 1671 1672 // If this is an or of disjoint bitfields, we can codegen this as an add 1673 // (for better address arithmetic) if the LHS and RHS of the OR are provably 1674 // disjoint. 1675 APInt LHSKnownZero, LHSKnownOne; 1676 APInt RHSKnownZero, RHSKnownOne; 1677 DAG.computeKnownBits(N.getOperand(0), 1678 LHSKnownZero, LHSKnownOne); 1679 1680 if (LHSKnownZero.getBoolValue()) { 1681 DAG.computeKnownBits(N.getOperand(1), 1682 RHSKnownZero, RHSKnownOne); 1683 // If all of the bits are known zero on the LHS or RHS, the add won't 1684 // carry. 1685 if (~(LHSKnownZero | RHSKnownZero) == 0) { 1686 Base = N.getOperand(0); 1687 Index = N.getOperand(1); 1688 return true; 1689 } 1690 } 1691 } 1692 1693 return false; 1694 } 1695 1696 // If we happen to be doing an i64 load or store into a stack slot that has 1697 // less than a 4-byte alignment, then the frame-index elimination may need to 1698 // use an indexed load or store instruction (because the offset may not be a 1699 // multiple of 4). The extra register needed to hold the offset comes from the 1700 // register scavenger, and it is possible that the scavenger will need to use 1701 // an emergency spill slot. As a result, we need to make sure that a spill slot 1702 // is allocated when doing an i64 load/store into a less-than-4-byte-aligned 1703 // stack slot. 1704 static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { 1705 // FIXME: This does not handle the LWA case. 1706 if (VT != MVT::i64) 1707 return; 1708 1709 // NOTE: We'll exclude negative FIs here, which come from argument 1710 // lowering, because there are no known test cases triggering this problem 1711 // using packed structures (or similar). We can remove this exclusion if 1712 // we find such a test case. The reason why this is so test-case driven is 1713 // because this entire 'fixup' is only to prevent crashes (from the 1714 // register scavenger) on not-really-valid inputs. For example, if we have: 1715 // %a = alloca i1 1716 // %b = bitcast i1* %a to i64* 1717 // store i64* a, i64 b 1718 // then the store should really be marked as 'align 1', but is not. If it 1719 // were marked as 'align 1' then the indexed form would have been 1720 // instruction-selected initially, and the problem this 'fixup' is preventing 1721 // won't happen regardless. 1722 if (FrameIdx < 0) 1723 return; 1724 1725 MachineFunction &MF = DAG.getMachineFunction(); 1726 MachineFrameInfo *MFI = MF.getFrameInfo(); 1727 1728 unsigned Align = MFI->getObjectAlignment(FrameIdx); 1729 if (Align >= 4) 1730 return; 1731 1732 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 1733 FuncInfo->setHasNonRISpills(); 1734 } 1735 1736 /// Returns true if the address N can be represented by a base register plus 1737 /// a signed 16-bit displacement [r+imm], and if it is not better 1738 /// represented as reg+reg. If Aligned is true, only accept displacements 1739 /// suitable for STD and friends, i.e. multiples of 4. 1740 bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, 1741 SDValue &Base, 1742 SelectionDAG &DAG, 1743 bool Aligned) const { 1744 // FIXME dl should come from parent load or store, not from address 1745 SDLoc dl(N); 1746 // If this can be more profitably realized as r+r, fail. 1747 if (SelectAddressRegReg(N, Disp, Base, DAG)) 1748 return false; 1749 1750 if (N.getOpcode() == ISD::ADD) { 1751 short imm = 0; 1752 if (isIntS16Immediate(N.getOperand(1), imm) && 1753 (!Aligned || (imm & 3) == 0)) { 1754 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 1755 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 1756 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 1757 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 1758 } else { 1759 Base = N.getOperand(0); 1760 } 1761 return true; // [r+i] 1762 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { 1763 // Match LOAD (ADD (X, Lo(G))). 1764 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() 1765 && "Cannot handle constant offsets yet!"); 1766 Disp = N.getOperand(1).getOperand(0); // The global address. 1767 assert(Disp.getOpcode() == ISD::TargetGlobalAddress || 1768 Disp.getOpcode() == ISD::TargetGlobalTLSAddress || 1769 Disp.getOpcode() == ISD::TargetConstantPool || 1770 Disp.getOpcode() == ISD::TargetJumpTable); 1771 Base = N.getOperand(0); 1772 return true; // [&g+r] 1773 } 1774 } else if (N.getOpcode() == ISD::OR) { 1775 short imm = 0; 1776 if (isIntS16Immediate(N.getOperand(1), imm) && 1777 (!Aligned || (imm & 3) == 0)) { 1778 // If this is an or of disjoint bitfields, we can codegen this as an add 1779 // (for better address arithmetic) if the LHS and RHS of the OR are 1780 // provably disjoint. 1781 APInt LHSKnownZero, LHSKnownOne; 1782 DAG.computeKnownBits(N.getOperand(0), LHSKnownZero, LHSKnownOne); 1783 1784 if ((LHSKnownZero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { 1785 // If all of the bits are known zero on the LHS or RHS, the add won't 1786 // carry. 1787 if (FrameIndexSDNode *FI = 1788 dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 1789 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 1790 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 1791 } else { 1792 Base = N.getOperand(0); 1793 } 1794 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 1795 return true; 1796 } 1797 } 1798 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 1799 // Loading from a constant address. 1800 1801 // If this address fits entirely in a 16-bit sext immediate field, codegen 1802 // this as "d, 0" 1803 short Imm; 1804 if (isIntS16Immediate(CN, Imm) && (!Aligned || (Imm & 3) == 0)) { 1805 Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); 1806 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 1807 CN->getValueType(0)); 1808 return true; 1809 } 1810 1811 // Handle 32-bit sext immediates with LIS + addr mode. 1812 if ((CN->getValueType(0) == MVT::i32 || 1813 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && 1814 (!Aligned || (CN->getZExtValue() & 3) == 0)) { 1815 int Addr = (int)CN->getZExtValue(); 1816 1817 // Otherwise, break this down into an LIS + disp. 1818 Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32); 1819 1820 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl, 1821 MVT::i32); 1822 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; 1823 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); 1824 return true; 1825 } 1826 } 1827 1828 Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout())); 1829 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { 1830 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 1831 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 1832 } else 1833 Base = N; 1834 return true; // [r+0] 1835 } 1836 1837 /// SelectAddressRegRegOnly - Given the specified addressed, force it to be 1838 /// represented as an indexed [r+r] operation. 1839 bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, 1840 SDValue &Index, 1841 SelectionDAG &DAG) const { 1842 // Check to see if we can easily represent this as an [r+r] address. This 1843 // will fail if it thinks that the address is more profitably represented as 1844 // reg+imm, e.g. where imm = 0. 1845 if (SelectAddressRegReg(N, Base, Index, DAG)) 1846 return true; 1847 1848 // If the operand is an addition, always emit this as [r+r], since this is 1849 // better (for code size, and execution, as the memop does the add for free) 1850 // than emitting an explicit add. 1851 if (N.getOpcode() == ISD::ADD) { 1852 Base = N.getOperand(0); 1853 Index = N.getOperand(1); 1854 return true; 1855 } 1856 1857 // Otherwise, do it the hard way, using R0 as the base register. 1858 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 1859 N.getValueType()); 1860 Index = N; 1861 return true; 1862 } 1863 1864 /// getPreIndexedAddressParts - returns true by value, base pointer and 1865 /// offset pointer and addressing mode by reference if the node's address 1866 /// can be legally represented as pre-indexed load / store address. 1867 bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 1868 SDValue &Offset, 1869 ISD::MemIndexedMode &AM, 1870 SelectionDAG &DAG) const { 1871 if (DisablePPCPreinc) return false; 1872 1873 bool isLoad = true; 1874 SDValue Ptr; 1875 EVT VT; 1876 unsigned Alignment; 1877 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 1878 Ptr = LD->getBasePtr(); 1879 VT = LD->getMemoryVT(); 1880 Alignment = LD->getAlignment(); 1881 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 1882 Ptr = ST->getBasePtr(); 1883 VT = ST->getMemoryVT(); 1884 Alignment = ST->getAlignment(); 1885 isLoad = false; 1886 } else 1887 return false; 1888 1889 // PowerPC doesn't have preinc load/store instructions for vectors (except 1890 // for QPX, which does have preinc r+r forms). 1891 if (VT.isVector()) { 1892 if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) { 1893 return false; 1894 } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) { 1895 AM = ISD::PRE_INC; 1896 return true; 1897 } 1898 } 1899 1900 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { 1901 1902 // Common code will reject creating a pre-inc form if the base pointer 1903 // is a frame index, or if N is a store and the base pointer is either 1904 // the same as or a predecessor of the value being stored. Check for 1905 // those situations here, and try with swapped Base/Offset instead. 1906 bool Swap = false; 1907 1908 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base)) 1909 Swap = true; 1910 else if (!isLoad) { 1911 SDValue Val = cast<StoreSDNode>(N)->getValue(); 1912 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) 1913 Swap = true; 1914 } 1915 1916 if (Swap) 1917 std::swap(Base, Offset); 1918 1919 AM = ISD::PRE_INC; 1920 return true; 1921 } 1922 1923 // LDU/STU can only handle immediates that are a multiple of 4. 1924 if (VT != MVT::i64) { 1925 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, false)) 1926 return false; 1927 } else { 1928 // LDU/STU need an address with at least 4-byte alignment. 1929 if (Alignment < 4) 1930 return false; 1931 1932 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, true)) 1933 return false; 1934 } 1935 1936 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 1937 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of 1938 // sext i32 to i64 when addr mode is r+i. 1939 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && 1940 LD->getExtensionType() == ISD::SEXTLOAD && 1941 isa<ConstantSDNode>(Offset)) 1942 return false; 1943 } 1944 1945 AM = ISD::PRE_INC; 1946 return true; 1947 } 1948 1949 //===----------------------------------------------------------------------===// 1950 // LowerOperation implementation 1951 //===----------------------------------------------------------------------===// 1952 1953 /// GetLabelAccessInfo - Return true if we should reference labels using a 1954 /// PICBase, set the HiOpFlags and LoOpFlags to the target MO flags. 1955 static bool GetLabelAccessInfo(const TargetMachine &TM, 1956 const PPCSubtarget &Subtarget, 1957 unsigned &HiOpFlags, unsigned &LoOpFlags, 1958 const GlobalValue *GV = nullptr) { 1959 HiOpFlags = PPCII::MO_HA; 1960 LoOpFlags = PPCII::MO_LO; 1961 1962 // Don't use the pic base if not in PIC relocation model. 1963 bool isPIC = TM.getRelocationModel() == Reloc::PIC_; 1964 1965 if (isPIC) { 1966 HiOpFlags |= PPCII::MO_PIC_FLAG; 1967 LoOpFlags |= PPCII::MO_PIC_FLAG; 1968 } 1969 1970 // If this is a reference to a global value that requires a non-lazy-ptr, make 1971 // sure that instruction lowering adds it. 1972 if (GV && Subtarget.hasLazyResolverStub(GV)) { 1973 HiOpFlags |= PPCII::MO_NLP_FLAG; 1974 LoOpFlags |= PPCII::MO_NLP_FLAG; 1975 1976 if (GV->hasHiddenVisibility()) { 1977 HiOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 1978 LoOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 1979 } 1980 } 1981 1982 return isPIC; 1983 } 1984 1985 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, 1986 SelectionDAG &DAG) { 1987 SDLoc DL(HiPart); 1988 EVT PtrVT = HiPart.getValueType(); 1989 SDValue Zero = DAG.getConstant(0, DL, PtrVT); 1990 1991 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); 1992 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); 1993 1994 // With PIC, the first instruction is actually "GR+hi(&G)". 1995 if (isPIC) 1996 Hi = DAG.getNode(ISD::ADD, DL, PtrVT, 1997 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); 1998 1999 // Generate non-pic code that has direct accesses to the constant pool. 2000 // The address of the global is just (hi(&g)+lo(&g)). 2001 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); 2002 } 2003 2004 static void setUsesTOCBasePtr(MachineFunction &MF) { 2005 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2006 FuncInfo->setUsesTOCBasePtr(); 2007 } 2008 2009 static void setUsesTOCBasePtr(SelectionDAG &DAG) { 2010 setUsesTOCBasePtr(DAG.getMachineFunction()); 2011 } 2012 2013 static SDValue getTOCEntry(SelectionDAG &DAG, SDLoc dl, bool Is64Bit, 2014 SDValue GA) { 2015 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 2016 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) : 2017 DAG.getNode(PPCISD::GlobalBaseReg, dl, VT); 2018 2019 SDValue Ops[] = { GA, Reg }; 2020 return DAG.getMemIntrinsicNode( 2021 PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, 2022 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0, false, true, 2023 false, 0); 2024 } 2025 2026 SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, 2027 SelectionDAG &DAG) const { 2028 EVT PtrVT = Op.getValueType(); 2029 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 2030 const Constant *C = CP->getConstVal(); 2031 2032 // 64-bit SVR4 ABI code is always position-independent. 2033 // The actual address of the GlobalValue is stored in the TOC. 2034 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2035 setUsesTOCBasePtr(DAG); 2036 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0); 2037 return getTOCEntry(DAG, SDLoc(CP), true, GA); 2038 } 2039 2040 unsigned MOHiFlag, MOLoFlag; 2041 bool isPIC = 2042 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag); 2043 2044 if (isPIC && Subtarget.isSVR4ABI()) { 2045 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 2046 PPCII::MO_PIC_FLAG); 2047 return getTOCEntry(DAG, SDLoc(CP), false, GA); 2048 } 2049 2050 SDValue CPIHi = 2051 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag); 2052 SDValue CPILo = 2053 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag); 2054 return LowerLabelRef(CPIHi, CPILo, isPIC, DAG); 2055 } 2056 2057 SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 2058 EVT PtrVT = Op.getValueType(); 2059 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 2060 2061 // 64-bit SVR4 ABI code is always position-independent. 2062 // The actual address of the GlobalValue is stored in the TOC. 2063 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2064 setUsesTOCBasePtr(DAG); 2065 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 2066 return getTOCEntry(DAG, SDLoc(JT), true, GA); 2067 } 2068 2069 unsigned MOHiFlag, MOLoFlag; 2070 bool isPIC = 2071 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag); 2072 2073 if (isPIC && Subtarget.isSVR4ABI()) { 2074 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, 2075 PPCII::MO_PIC_FLAG); 2076 return getTOCEntry(DAG, SDLoc(GA), false, GA); 2077 } 2078 2079 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); 2080 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); 2081 return LowerLabelRef(JTIHi, JTILo, isPIC, DAG); 2082 } 2083 2084 SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, 2085 SelectionDAG &DAG) const { 2086 EVT PtrVT = Op.getValueType(); 2087 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op); 2088 const BlockAddress *BA = BASDN->getBlockAddress(); 2089 2090 // 64-bit SVR4 ABI code is always position-independent. 2091 // The actual BlockAddress is stored in the TOC. 2092 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2093 setUsesTOCBasePtr(DAG); 2094 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()); 2095 return getTOCEntry(DAG, SDLoc(BASDN), true, GA); 2096 } 2097 2098 unsigned MOHiFlag, MOLoFlag; 2099 bool isPIC = 2100 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag); 2101 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); 2102 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); 2103 return LowerLabelRef(TgtBAHi, TgtBALo, isPIC, DAG); 2104 } 2105 2106 SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, 2107 SelectionDAG &DAG) const { 2108 2109 // FIXME: TLS addresses currently use medium model code sequences, 2110 // which is the most useful form. Eventually support for small and 2111 // large models could be added if users need it, at the cost of 2112 // additional complexity. 2113 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 2114 if (DAG.getTarget().Options.EmulatedTLS) 2115 return LowerToTLSEmulatedModel(GA, DAG); 2116 2117 SDLoc dl(GA); 2118 const GlobalValue *GV = GA->getGlobal(); 2119 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2120 bool is64bit = Subtarget.isPPC64(); 2121 const Module *M = DAG.getMachineFunction().getFunction()->getParent(); 2122 PICLevel::Level picLevel = M->getPICLevel(); 2123 2124 TLSModel::Model Model = getTargetMachine().getTLSModel(GV); 2125 2126 if (Model == TLSModel::LocalExec) { 2127 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2128 PPCII::MO_TPREL_HA); 2129 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2130 PPCII::MO_TPREL_LO); 2131 SDValue TLSReg = DAG.getRegister(is64bit ? PPC::X13 : PPC::R2, 2132 is64bit ? MVT::i64 : MVT::i32); 2133 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); 2134 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); 2135 } 2136 2137 if (Model == TLSModel::InitialExec) { 2138 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2139 SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2140 PPCII::MO_TLS); 2141 SDValue GOTPtr; 2142 if (is64bit) { 2143 setUsesTOCBasePtr(DAG); 2144 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2145 GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, 2146 PtrVT, GOTReg, TGA); 2147 } else 2148 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); 2149 SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, 2150 PtrVT, TGA, GOTPtr); 2151 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); 2152 } 2153 2154 if (Model == TLSModel::GeneralDynamic) { 2155 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2156 SDValue GOTPtr; 2157 if (is64bit) { 2158 setUsesTOCBasePtr(DAG); 2159 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2160 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, 2161 GOTReg, TGA); 2162 } else { 2163 if (picLevel == PICLevel::Small) 2164 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2165 else 2166 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2167 } 2168 return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT, 2169 GOTPtr, TGA, TGA); 2170 } 2171 2172 if (Model == TLSModel::LocalDynamic) { 2173 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2174 SDValue GOTPtr; 2175 if (is64bit) { 2176 setUsesTOCBasePtr(DAG); 2177 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2178 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, 2179 GOTReg, TGA); 2180 } else { 2181 if (picLevel == PICLevel::Small) 2182 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2183 else 2184 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2185 } 2186 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl, 2187 PtrVT, GOTPtr, TGA, TGA); 2188 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, 2189 PtrVT, TLSAddr, TGA); 2190 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); 2191 } 2192 2193 llvm_unreachable("Unknown TLS model!"); 2194 } 2195 2196 SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, 2197 SelectionDAG &DAG) const { 2198 EVT PtrVT = Op.getValueType(); 2199 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op); 2200 SDLoc DL(GSDN); 2201 const GlobalValue *GV = GSDN->getGlobal(); 2202 2203 // 64-bit SVR4 ABI code is always position-independent. 2204 // The actual address of the GlobalValue is stored in the TOC. 2205 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2206 setUsesTOCBasePtr(DAG); 2207 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); 2208 return getTOCEntry(DAG, DL, true, GA); 2209 } 2210 2211 unsigned MOHiFlag, MOLoFlag; 2212 bool isPIC = 2213 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag, GV); 2214 2215 if (isPIC && Subtarget.isSVR4ABI()) { 2216 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 2217 GSDN->getOffset(), 2218 PPCII::MO_PIC_FLAG); 2219 return getTOCEntry(DAG, DL, false, GA); 2220 } 2221 2222 SDValue GAHi = 2223 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); 2224 SDValue GALo = 2225 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); 2226 2227 SDValue Ptr = LowerLabelRef(GAHi, GALo, isPIC, DAG); 2228 2229 // If the global reference is actually to a non-lazy-pointer, we have to do an 2230 // extra load to get the address of the global. 2231 if (MOHiFlag & PPCII::MO_NLP_FLAG) 2232 Ptr = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo(), 2233 false, false, false, 0); 2234 return Ptr; 2235 } 2236 2237 SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 2238 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 2239 SDLoc dl(Op); 2240 2241 if (Op.getValueType() == MVT::v2i64) { 2242 // When the operands themselves are v2i64 values, we need to do something 2243 // special because VSX has no underlying comparison operations for these. 2244 if (Op.getOperand(0).getValueType() == MVT::v2i64) { 2245 // Equality can be handled by casting to the legal type for Altivec 2246 // comparisons, everything else needs to be expanded. 2247 if (CC == ISD::SETEQ || CC == ISD::SETNE) { 2248 return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 2249 DAG.getSetCC(dl, MVT::v4i32, 2250 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), 2251 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), 2252 CC)); 2253 } 2254 2255 return SDValue(); 2256 } 2257 2258 // We handle most of these in the usual way. 2259 return Op; 2260 } 2261 2262 // If we're comparing for equality to zero, expose the fact that this is 2263 // implented as a ctlz/srl pair on ppc, so that the dag combiner can 2264 // fold the new nodes. 2265 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2266 if (C->isNullValue() && CC == ISD::SETEQ) { 2267 EVT VT = Op.getOperand(0).getValueType(); 2268 SDValue Zext = Op.getOperand(0); 2269 if (VT.bitsLT(MVT::i32)) { 2270 VT = MVT::i32; 2271 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); 2272 } 2273 unsigned Log2b = Log2_32(VT.getSizeInBits()); 2274 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); 2275 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, 2276 DAG.getConstant(Log2b, dl, MVT::i32)); 2277 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc); 2278 } 2279 // Leave comparisons against 0 and -1 alone for now, since they're usually 2280 // optimized. FIXME: revisit this when we can custom lower all setcc 2281 // optimizations. 2282 if (C->isAllOnesValue() || C->isNullValue()) 2283 return SDValue(); 2284 } 2285 2286 // If we have an integer seteq/setne, turn it into a compare against zero 2287 // by xor'ing the rhs with the lhs, which is faster than setting a 2288 // condition register, reading it back out, and masking the correct bit. The 2289 // normal approach here uses sub to do this instead of xor. Using xor exposes 2290 // the result to other bit-twiddling opportunities. 2291 EVT LHSVT = Op.getOperand(0).getValueType(); 2292 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { 2293 EVT VT = Op.getValueType(); 2294 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), 2295 Op.getOperand(1)); 2296 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); 2297 } 2298 return SDValue(); 2299 } 2300 2301 SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG, 2302 const PPCSubtarget &Subtarget) const { 2303 SDNode *Node = Op.getNode(); 2304 EVT VT = Node->getValueType(0); 2305 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 2306 SDValue InChain = Node->getOperand(0); 2307 SDValue VAListPtr = Node->getOperand(1); 2308 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 2309 SDLoc dl(Node); 2310 2311 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); 2312 2313 // gpr_index 2314 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 2315 VAListPtr, MachinePointerInfo(SV), MVT::i8, 2316 false, false, false, 0); 2317 InChain = GprIndex.getValue(1); 2318 2319 if (VT == MVT::i64) { 2320 // Check if GprIndex is even 2321 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, 2322 DAG.getConstant(1, dl, MVT::i32)); 2323 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, 2324 DAG.getConstant(0, dl, MVT::i32), ISD::SETNE); 2325 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, 2326 DAG.getConstant(1, dl, MVT::i32)); 2327 // Align GprIndex to be even if it isn't 2328 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, 2329 GprIndex); 2330 } 2331 2332 // fpr index is 1 byte after gpr 2333 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 2334 DAG.getConstant(1, dl, MVT::i32)); 2335 2336 // fpr 2337 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 2338 FprPtr, MachinePointerInfo(SV), MVT::i8, 2339 false, false, false, 0); 2340 InChain = FprIndex.getValue(1); 2341 2342 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 2343 DAG.getConstant(8, dl, MVT::i32)); 2344 2345 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 2346 DAG.getConstant(4, dl, MVT::i32)); 2347 2348 // areas 2349 SDValue OverflowArea = DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, 2350 MachinePointerInfo(), false, false, 2351 false, 0); 2352 InChain = OverflowArea.getValue(1); 2353 2354 SDValue RegSaveArea = DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, 2355 MachinePointerInfo(), false, false, 2356 false, 0); 2357 InChain = RegSaveArea.getValue(1); 2358 2359 // select overflow_area if index > 8 2360 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, 2361 DAG.getConstant(8, dl, MVT::i32), ISD::SETLT); 2362 2363 // adjustment constant gpr_index * 4/8 2364 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, 2365 VT.isInteger() ? GprIndex : FprIndex, 2366 DAG.getConstant(VT.isInteger() ? 4 : 8, dl, 2367 MVT::i32)); 2368 2369 // OurReg = RegSaveArea + RegConstant 2370 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, 2371 RegConstant); 2372 2373 // Floating types are 32 bytes into RegSaveArea 2374 if (VT.isFloatingPoint()) 2375 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, 2376 DAG.getConstant(32, dl, MVT::i32)); 2377 2378 // increase {f,g}pr_index by 1 (or 2 if VT is i64) 2379 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, 2380 VT.isInteger() ? GprIndex : FprIndex, 2381 DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl, 2382 MVT::i32)); 2383 2384 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, 2385 VT.isInteger() ? VAListPtr : FprPtr, 2386 MachinePointerInfo(SV), 2387 MVT::i8, false, false, 0); 2388 2389 // determine if we should load from reg_save_area or overflow_area 2390 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); 2391 2392 // increase overflow_area by 4/8 if gpr/fpr > 8 2393 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, 2394 DAG.getConstant(VT.isInteger() ? 4 : 8, 2395 dl, MVT::i32)); 2396 2397 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, 2398 OverflowAreaPlusN); 2399 2400 InChain = DAG.getTruncStore(InChain, dl, OverflowArea, 2401 OverflowAreaPtr, 2402 MachinePointerInfo(), 2403 MVT::i32, false, false, 0); 2404 2405 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo(), 2406 false, false, false, 0); 2407 } 2408 2409 SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG, 2410 const PPCSubtarget &Subtarget) const { 2411 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); 2412 2413 // We have to copy the entire va_list struct: 2414 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte 2415 return DAG.getMemcpy(Op.getOperand(0), Op, 2416 Op.getOperand(1), Op.getOperand(2), 2417 DAG.getConstant(12, SDLoc(Op), MVT::i32), 8, false, true, 2418 false, MachinePointerInfo(), MachinePointerInfo()); 2419 } 2420 2421 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, 2422 SelectionDAG &DAG) const { 2423 return Op.getOperand(0); 2424 } 2425 2426 SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, 2427 SelectionDAG &DAG) const { 2428 SDValue Chain = Op.getOperand(0); 2429 SDValue Trmp = Op.getOperand(1); // trampoline 2430 SDValue FPtr = Op.getOperand(2); // nested function 2431 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 2432 SDLoc dl(Op); 2433 2434 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 2435 bool isPPC64 = (PtrVT == MVT::i64); 2436 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); 2437 2438 TargetLowering::ArgListTy Args; 2439 TargetLowering::ArgListEntry Entry; 2440 2441 Entry.Ty = IntPtrTy; 2442 Entry.Node = Trmp; Args.push_back(Entry); 2443 2444 // TrampSize == (isPPC64 ? 48 : 40); 2445 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl, 2446 isPPC64 ? MVT::i64 : MVT::i32); 2447 Args.push_back(Entry); 2448 2449 Entry.Node = FPtr; Args.push_back(Entry); 2450 Entry.Node = Nest; Args.push_back(Entry); 2451 2452 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) 2453 TargetLowering::CallLoweringInfo CLI(DAG); 2454 CLI.setDebugLoc(dl).setChain(Chain) 2455 .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 2456 DAG.getExternalSymbol("__trampoline_setup", PtrVT), 2457 std::move(Args), 0); 2458 2459 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 2460 return CallResult.second; 2461 } 2462 2463 SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG, 2464 const PPCSubtarget &Subtarget) const { 2465 MachineFunction &MF = DAG.getMachineFunction(); 2466 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2467 2468 SDLoc dl(Op); 2469 2470 if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) { 2471 // vastart just stores the address of the VarArgsFrameIndex slot into the 2472 // memory location argument. 2473 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 2474 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 2475 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 2476 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 2477 MachinePointerInfo(SV), 2478 false, false, 0); 2479 } 2480 2481 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. 2482 // We suppose the given va_list is already allocated. 2483 // 2484 // typedef struct { 2485 // char gpr; /* index into the array of 8 GPRs 2486 // * stored in the register save area 2487 // * gpr=0 corresponds to r3, 2488 // * gpr=1 to r4, etc. 2489 // */ 2490 // char fpr; /* index into the array of 8 FPRs 2491 // * stored in the register save area 2492 // * fpr=0 corresponds to f1, 2493 // * fpr=1 to f2, etc. 2494 // */ 2495 // char *overflow_arg_area; 2496 // /* location on stack that holds 2497 // * the next overflow argument 2498 // */ 2499 // char *reg_save_area; 2500 // /* where r3:r10 and f1:f8 (if saved) 2501 // * are stored 2502 // */ 2503 // } va_list[1]; 2504 2505 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32); 2506 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32); 2507 2508 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 2509 2510 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), 2511 PtrVT); 2512 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 2513 PtrVT); 2514 2515 uint64_t FrameOffset = PtrVT.getSizeInBits()/8; 2516 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT); 2517 2518 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; 2519 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT); 2520 2521 uint64_t FPROffset = 1; 2522 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT); 2523 2524 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 2525 2526 // Store first byte : number of int regs 2527 SDValue firstStore = DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, 2528 Op.getOperand(1), 2529 MachinePointerInfo(SV), 2530 MVT::i8, false, false, 0); 2531 uint64_t nextOffset = FPROffset; 2532 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), 2533 ConstFPROffset); 2534 2535 // Store second byte : number of float regs 2536 SDValue secondStore = 2537 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, 2538 MachinePointerInfo(SV, nextOffset), MVT::i8, 2539 false, false, 0); 2540 nextOffset += StackOffset; 2541 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); 2542 2543 // Store second word : arguments given on stack 2544 SDValue thirdStore = 2545 DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, 2546 MachinePointerInfo(SV, nextOffset), 2547 false, false, 0); 2548 nextOffset += FrameOffset; 2549 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); 2550 2551 // Store third word : arguments given in registers 2552 return DAG.getStore(thirdStore, dl, FR, nextPtr, 2553 MachinePointerInfo(SV, nextOffset), 2554 false, false, 0); 2555 2556 } 2557 2558 #include "PPCGenCallingConv.inc" 2559 2560 // Function whose sole purpose is to kill compiler warnings 2561 // stemming from unused functions included from PPCGenCallingConv.inc. 2562 CCAssignFn *PPCTargetLowering::useFastISelCCs(unsigned Flag) const { 2563 return Flag ? CC_PPC64_ELF_FIS : RetCC_PPC64_ELF_FIS; 2564 } 2565 2566 bool llvm::CC_PPC32_SVR4_Custom_Dummy(unsigned &ValNo, MVT &ValVT, MVT &LocVT, 2567 CCValAssign::LocInfo &LocInfo, 2568 ISD::ArgFlagsTy &ArgFlags, 2569 CCState &State) { 2570 return true; 2571 } 2572 2573 bool llvm::CC_PPC32_SVR4_Custom_AlignArgRegs(unsigned &ValNo, MVT &ValVT, 2574 MVT &LocVT, 2575 CCValAssign::LocInfo &LocInfo, 2576 ISD::ArgFlagsTy &ArgFlags, 2577 CCState &State) { 2578 static const MCPhysReg ArgRegs[] = { 2579 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 2580 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 2581 }; 2582 const unsigned NumArgRegs = array_lengthof(ArgRegs); 2583 2584 unsigned RegNum = State.getFirstUnallocated(ArgRegs); 2585 2586 // Skip one register if the first unallocated register has an even register 2587 // number and there are still argument registers available which have not been 2588 // allocated yet. RegNum is actually an index into ArgRegs, which means we 2589 // need to skip a register if RegNum is odd. 2590 if (RegNum != NumArgRegs && RegNum % 2 == 1) { 2591 State.AllocateReg(ArgRegs[RegNum]); 2592 } 2593 2594 // Always return false here, as this function only makes sure that the first 2595 // unallocated register has an odd register number and does not actually 2596 // allocate a register for the current argument. 2597 return false; 2598 } 2599 2600 bool llvm::CC_PPC32_SVR4_Custom_AlignFPArgRegs(unsigned &ValNo, MVT &ValVT, 2601 MVT &LocVT, 2602 CCValAssign::LocInfo &LocInfo, 2603 ISD::ArgFlagsTy &ArgFlags, 2604 CCState &State) { 2605 static const MCPhysReg ArgRegs[] = { 2606 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 2607 PPC::F8 2608 }; 2609 2610 const unsigned NumArgRegs = array_lengthof(ArgRegs); 2611 2612 unsigned RegNum = State.getFirstUnallocated(ArgRegs); 2613 2614 // If there is only one Floating-point register left we need to put both f64 2615 // values of a split ppc_fp128 value on the stack. 2616 if (RegNum != NumArgRegs && ArgRegs[RegNum] == PPC::F8) { 2617 State.AllocateReg(ArgRegs[RegNum]); 2618 } 2619 2620 // Always return false here, as this function only makes sure that the two f64 2621 // values a ppc_fp128 value is split into are both passed in registers or both 2622 // passed on the stack and does not actually allocate a register for the 2623 // current argument. 2624 return false; 2625 } 2626 2627 /// FPR - The set of FP registers that should be allocated for arguments, 2628 /// on Darwin. 2629 static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, 2630 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, 2631 PPC::F11, PPC::F12, PPC::F13}; 2632 2633 /// QFPR - The set of QPX registers that should be allocated for arguments. 2634 static const MCPhysReg QFPR[] = { 2635 PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7, 2636 PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13}; 2637 2638 /// CalculateStackSlotSize - Calculates the size reserved for this argument on 2639 /// the stack. 2640 static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, 2641 unsigned PtrByteSize) { 2642 unsigned ArgSize = ArgVT.getStoreSize(); 2643 if (Flags.isByVal()) 2644 ArgSize = Flags.getByValSize(); 2645 2646 // Round up to multiples of the pointer size, except for array members, 2647 // which are always packed. 2648 if (!Flags.isInConsecutiveRegs()) 2649 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 2650 2651 return ArgSize; 2652 } 2653 2654 /// CalculateStackSlotAlignment - Calculates the alignment of this argument 2655 /// on the stack. 2656 static unsigned CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, 2657 ISD::ArgFlagsTy Flags, 2658 unsigned PtrByteSize) { 2659 unsigned Align = PtrByteSize; 2660 2661 // Altivec parameters are padded to a 16 byte boundary. 2662 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 2663 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 2664 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 2665 ArgVT == MVT::v1i128) 2666 Align = 16; 2667 // QPX vector types stored in double-precision are padded to a 32 byte 2668 // boundary. 2669 else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1) 2670 Align = 32; 2671 2672 // ByVal parameters are aligned as requested. 2673 if (Flags.isByVal()) { 2674 unsigned BVAlign = Flags.getByValAlign(); 2675 if (BVAlign > PtrByteSize) { 2676 if (BVAlign % PtrByteSize != 0) 2677 llvm_unreachable( 2678 "ByVal alignment is not a multiple of the pointer size"); 2679 2680 Align = BVAlign; 2681 } 2682 } 2683 2684 // Array members are always packed to their original alignment. 2685 if (Flags.isInConsecutiveRegs()) { 2686 // If the array member was split into multiple registers, the first 2687 // needs to be aligned to the size of the full type. (Except for 2688 // ppcf128, which is only aligned as its f64 components.) 2689 if (Flags.isSplit() && OrigVT != MVT::ppcf128) 2690 Align = OrigVT.getStoreSize(); 2691 else 2692 Align = ArgVT.getStoreSize(); 2693 } 2694 2695 return Align; 2696 } 2697 2698 /// CalculateStackSlotUsed - Return whether this argument will use its 2699 /// stack slot (instead of being passed in registers). ArgOffset, 2700 /// AvailableFPRs, and AvailableVRs must hold the current argument 2701 /// position, and will be updated to account for this argument. 2702 static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, 2703 ISD::ArgFlagsTy Flags, 2704 unsigned PtrByteSize, 2705 unsigned LinkageSize, 2706 unsigned ParamAreaSize, 2707 unsigned &ArgOffset, 2708 unsigned &AvailableFPRs, 2709 unsigned &AvailableVRs, bool HasQPX) { 2710 bool UseMemory = false; 2711 2712 // Respect alignment of argument on the stack. 2713 unsigned Align = 2714 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 2715 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 2716 // If there's no space left in the argument save area, we must 2717 // use memory (this check also catches zero-sized arguments). 2718 if (ArgOffset >= LinkageSize + ParamAreaSize) 2719 UseMemory = true; 2720 2721 // Allocate argument on the stack. 2722 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 2723 if (Flags.isInConsecutiveRegsLast()) 2724 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 2725 // If we overran the argument save area, we must use memory 2726 // (this check catches arguments passed partially in memory) 2727 if (ArgOffset > LinkageSize + ParamAreaSize) 2728 UseMemory = true; 2729 2730 // However, if the argument is actually passed in an FPR or a VR, 2731 // we don't use memory after all. 2732 if (!Flags.isByVal()) { 2733 if (ArgVT == MVT::f32 || ArgVT == MVT::f64 || 2734 // QPX registers overlap with the scalar FP registers. 2735 (HasQPX && (ArgVT == MVT::v4f32 || 2736 ArgVT == MVT::v4f64 || 2737 ArgVT == MVT::v4i1))) 2738 if (AvailableFPRs > 0) { 2739 --AvailableFPRs; 2740 return false; 2741 } 2742 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 2743 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 2744 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 2745 ArgVT == MVT::v1i128) 2746 if (AvailableVRs > 0) { 2747 --AvailableVRs; 2748 return false; 2749 } 2750 } 2751 2752 return UseMemory; 2753 } 2754 2755 /// EnsureStackAlignment - Round stack frame size up from NumBytes to 2756 /// ensure minimum alignment required for target. 2757 static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering, 2758 unsigned NumBytes) { 2759 unsigned TargetAlign = Lowering->getStackAlignment(); 2760 unsigned AlignMask = TargetAlign - 1; 2761 NumBytes = (NumBytes + AlignMask) & ~AlignMask; 2762 return NumBytes; 2763 } 2764 2765 SDValue 2766 PPCTargetLowering::LowerFormalArguments(SDValue Chain, 2767 CallingConv::ID CallConv, bool isVarArg, 2768 const SmallVectorImpl<ISD::InputArg> 2769 &Ins, 2770 SDLoc dl, SelectionDAG &DAG, 2771 SmallVectorImpl<SDValue> &InVals) 2772 const { 2773 if (Subtarget.isSVR4ABI()) { 2774 if (Subtarget.isPPC64()) 2775 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, 2776 dl, DAG, InVals); 2777 else 2778 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, 2779 dl, DAG, InVals); 2780 } else { 2781 return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, 2782 dl, DAG, InVals); 2783 } 2784 } 2785 2786 SDValue 2787 PPCTargetLowering::LowerFormalArguments_32SVR4( 2788 SDValue Chain, 2789 CallingConv::ID CallConv, bool isVarArg, 2790 const SmallVectorImpl<ISD::InputArg> 2791 &Ins, 2792 SDLoc dl, SelectionDAG &DAG, 2793 SmallVectorImpl<SDValue> &InVals) const { 2794 2795 // 32-bit SVR4 ABI Stack Frame Layout: 2796 // +-----------------------------------+ 2797 // +--> | Back chain | 2798 // | +-----------------------------------+ 2799 // | | Floating-point register save area | 2800 // | +-----------------------------------+ 2801 // | | General register save area | 2802 // | +-----------------------------------+ 2803 // | | CR save word | 2804 // | +-----------------------------------+ 2805 // | | VRSAVE save word | 2806 // | +-----------------------------------+ 2807 // | | Alignment padding | 2808 // | +-----------------------------------+ 2809 // | | Vector register save area | 2810 // | +-----------------------------------+ 2811 // | | Local variable space | 2812 // | +-----------------------------------+ 2813 // | | Parameter list area | 2814 // | +-----------------------------------+ 2815 // | | LR save word | 2816 // | +-----------------------------------+ 2817 // SP--> +--- | Back chain | 2818 // +-----------------------------------+ 2819 // 2820 // Specifications: 2821 // System V Application Binary Interface PowerPC Processor Supplement 2822 // AltiVec Technology Programming Interface Manual 2823 2824 MachineFunction &MF = DAG.getMachineFunction(); 2825 MachineFrameInfo *MFI = MF.getFrameInfo(); 2826 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2827 2828 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 2829 // Potential tail calls could cause overwriting of argument stack slots. 2830 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 2831 (CallConv == CallingConv::Fast)); 2832 unsigned PtrByteSize = 4; 2833 2834 // Assign locations to all of the incoming arguments. 2835 SmallVector<CCValAssign, 16> ArgLocs; 2836 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 2837 *DAG.getContext()); 2838 2839 // Reserve space for the linkage area on the stack. 2840 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 2841 CCInfo.AllocateStack(LinkageSize, PtrByteSize); 2842 2843 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); 2844 2845 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2846 CCValAssign &VA = ArgLocs[i]; 2847 2848 // Arguments stored in registers. 2849 if (VA.isRegLoc()) { 2850 const TargetRegisterClass *RC; 2851 EVT ValVT = VA.getValVT(); 2852 2853 switch (ValVT.getSimpleVT().SimpleTy) { 2854 default: 2855 llvm_unreachable("ValVT not supported by formal arguments Lowering"); 2856 case MVT::i1: 2857 case MVT::i32: 2858 RC = &PPC::GPRCRegClass; 2859 break; 2860 case MVT::f32: 2861 if (Subtarget.hasP8Vector()) 2862 RC = &PPC::VSSRCRegClass; 2863 else 2864 RC = &PPC::F4RCRegClass; 2865 break; 2866 case MVT::f64: 2867 if (Subtarget.hasVSX()) 2868 RC = &PPC::VSFRCRegClass; 2869 else 2870 RC = &PPC::F8RCRegClass; 2871 break; 2872 case MVT::v16i8: 2873 case MVT::v8i16: 2874 case MVT::v4i32: 2875 RC = &PPC::VRRCRegClass; 2876 break; 2877 case MVT::v4f32: 2878 RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass; 2879 break; 2880 case MVT::v2f64: 2881 case MVT::v2i64: 2882 RC = &PPC::VSHRCRegClass; 2883 break; 2884 case MVT::v4f64: 2885 RC = &PPC::QFRCRegClass; 2886 break; 2887 case MVT::v4i1: 2888 RC = &PPC::QBRCRegClass; 2889 break; 2890 } 2891 2892 // Transform the arguments stored in physical registers into virtual ones. 2893 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 2894 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, 2895 ValVT == MVT::i1 ? MVT::i32 : ValVT); 2896 2897 if (ValVT == MVT::i1) 2898 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); 2899 2900 InVals.push_back(ArgValue); 2901 } else { 2902 // Argument stored in memory. 2903 assert(VA.isMemLoc()); 2904 2905 unsigned ArgSize = VA.getLocVT().getStoreSize(); 2906 int FI = MFI->CreateFixedObject(ArgSize, VA.getLocMemOffset(), 2907 isImmutable); 2908 2909 // Create load nodes to retrieve arguments from the stack. 2910 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2911 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN, 2912 MachinePointerInfo(), 2913 false, false, false, 0)); 2914 } 2915 } 2916 2917 // Assign locations to all of the incoming aggregate by value arguments. 2918 // Aggregates passed by value are stored in the local variable space of the 2919 // caller's stack frame, right above the parameter list area. 2920 SmallVector<CCValAssign, 16> ByValArgLocs; 2921 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 2922 ByValArgLocs, *DAG.getContext()); 2923 2924 // Reserve stack space for the allocations in CCInfo. 2925 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 2926 2927 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); 2928 2929 // Area that is at least reserved in the caller of this function. 2930 unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); 2931 MinReservedArea = std::max(MinReservedArea, LinkageSize); 2932 2933 // Set the size that is at least reserved in caller of this function. Tail 2934 // call optimized function's reserved stack space needs to be aligned so that 2935 // taking the difference between two stack areas will result in an aligned 2936 // stack. 2937 MinReservedArea = 2938 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 2939 FuncInfo->setMinReservedArea(MinReservedArea); 2940 2941 SmallVector<SDValue, 8> MemOps; 2942 2943 // If the function takes variable number of arguments, make a frame index for 2944 // the start of the first vararg value... for expansion of llvm.va_start. 2945 if (isVarArg) { 2946 static const MCPhysReg GPArgRegs[] = { 2947 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 2948 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 2949 }; 2950 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); 2951 2952 static const MCPhysReg FPArgRegs[] = { 2953 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 2954 PPC::F8 2955 }; 2956 unsigned NumFPArgRegs = array_lengthof(FPArgRegs); 2957 2958 if (Subtarget.useSoftFloat()) 2959 NumFPArgRegs = 0; 2960 2961 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs)); 2962 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs)); 2963 2964 // Make room for NumGPArgRegs and NumFPArgRegs. 2965 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + 2966 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8; 2967 2968 FuncInfo->setVarArgsStackOffset( 2969 MFI->CreateFixedObject(PtrVT.getSizeInBits()/8, 2970 CCInfo.getNextStackOffset(), true)); 2971 2972 FuncInfo->setVarArgsFrameIndex(MFI->CreateStackObject(Depth, 8, false)); 2973 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 2974 2975 // The fixed integer arguments of a variadic function are stored to the 2976 // VarArgsFrameIndex on the stack so that they may be loaded by deferencing 2977 // the result of va_next. 2978 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { 2979 // Get an existing live-in vreg, or add a new one. 2980 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); 2981 if (!VReg) 2982 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); 2983 2984 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2985 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2986 MachinePointerInfo(), false, false, 0); 2987 MemOps.push_back(Store); 2988 // Increment the address by four for the next argument to store 2989 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 2990 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 2991 } 2992 2993 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 2994 // is set. 2995 // The double arguments are stored to the VarArgsFrameIndex 2996 // on the stack. 2997 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { 2998 // Get an existing live-in vreg, or add a new one. 2999 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); 3000 if (!VReg) 3001 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); 3002 3003 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); 3004 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3005 MachinePointerInfo(), false, false, 0); 3006 MemOps.push_back(Store); 3007 // Increment the address by eight for the next argument to store 3008 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl, 3009 PtrVT); 3010 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3011 } 3012 } 3013 3014 if (!MemOps.empty()) 3015 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3016 3017 return Chain; 3018 } 3019 3020 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3021 // value to MVT::i64 and then truncate to the correct register size. 3022 SDValue 3023 PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT, 3024 SelectionDAG &DAG, SDValue ArgVal, 3025 SDLoc dl) const { 3026 if (Flags.isSExt()) 3027 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, 3028 DAG.getValueType(ObjectVT)); 3029 else if (Flags.isZExt()) 3030 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, 3031 DAG.getValueType(ObjectVT)); 3032 3033 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); 3034 } 3035 3036 SDValue 3037 PPCTargetLowering::LowerFormalArguments_64SVR4( 3038 SDValue Chain, 3039 CallingConv::ID CallConv, bool isVarArg, 3040 const SmallVectorImpl<ISD::InputArg> 3041 &Ins, 3042 SDLoc dl, SelectionDAG &DAG, 3043 SmallVectorImpl<SDValue> &InVals) const { 3044 // TODO: add description of PPC stack frame format, or at least some docs. 3045 // 3046 bool isELFv2ABI = Subtarget.isELFv2ABI(); 3047 bool isLittleEndian = Subtarget.isLittleEndian(); 3048 MachineFunction &MF = DAG.getMachineFunction(); 3049 MachineFrameInfo *MFI = MF.getFrameInfo(); 3050 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3051 3052 assert(!(CallConv == CallingConv::Fast && isVarArg) && 3053 "fastcc not supported on varargs functions"); 3054 3055 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 3056 // Potential tail calls could cause overwriting of argument stack slots. 3057 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 3058 (CallConv == CallingConv::Fast)); 3059 unsigned PtrByteSize = 8; 3060 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 3061 3062 static const MCPhysReg GPR[] = { 3063 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 3064 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 3065 }; 3066 static const MCPhysReg VR[] = { 3067 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 3068 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 3069 }; 3070 static const MCPhysReg VSRH[] = { 3071 PPC::VSH2, PPC::VSH3, PPC::VSH4, PPC::VSH5, PPC::VSH6, PPC::VSH7, PPC::VSH8, 3072 PPC::VSH9, PPC::VSH10, PPC::VSH11, PPC::VSH12, PPC::VSH13 3073 }; 3074 3075 const unsigned Num_GPR_Regs = array_lengthof(GPR); 3076 const unsigned Num_FPR_Regs = 13; 3077 const unsigned Num_VR_Regs = array_lengthof(VR); 3078 const unsigned Num_QFPR_Regs = Num_FPR_Regs; 3079 3080 // Do a first pass over the arguments to determine whether the ABI 3081 // guarantees that our caller has allocated the parameter save area 3082 // on its stack frame. In the ELFv1 ABI, this is always the case; 3083 // in the ELFv2 ABI, it is true if this is a vararg function or if 3084 // any parameter is located in a stack slot. 3085 3086 bool HasParameterArea = !isELFv2ABI || isVarArg; 3087 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize; 3088 unsigned NumBytes = LinkageSize; 3089 unsigned AvailableFPRs = Num_FPR_Regs; 3090 unsigned AvailableVRs = Num_VR_Regs; 3091 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 3092 if (Ins[i].Flags.isNest()) 3093 continue; 3094 3095 if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, 3096 PtrByteSize, LinkageSize, ParamAreaSize, 3097 NumBytes, AvailableFPRs, AvailableVRs, 3098 Subtarget.hasQPX())) 3099 HasParameterArea = true; 3100 } 3101 3102 // Add DAG nodes to load the arguments or copy them out of registers. On 3103 // entry to a function on PPC, the arguments start after the linkage area, 3104 // although the first ones are often in registers. 3105 3106 unsigned ArgOffset = LinkageSize; 3107 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 3108 unsigned &QFPR_idx = FPR_idx; 3109 SmallVector<SDValue, 8> MemOps; 3110 Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin(); 3111 unsigned CurArgIdx = 0; 3112 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 3113 SDValue ArgVal; 3114 bool needsLoad = false; 3115 EVT ObjectVT = Ins[ArgNo].VT; 3116 EVT OrigVT = Ins[ArgNo].ArgVT; 3117 unsigned ObjSize = ObjectVT.getStoreSize(); 3118 unsigned ArgSize = ObjSize; 3119 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 3120 if (Ins[ArgNo].isOrigArg()) { 3121 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 3122 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 3123 } 3124 // We re-align the argument offset for each argument, except when using the 3125 // fast calling convention, when we need to make sure we do that only when 3126 // we'll actually use a stack slot. 3127 unsigned CurArgOffset, Align; 3128 auto ComputeArgOffset = [&]() { 3129 /* Respect alignment of argument on the stack. */ 3130 Align = CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize); 3131 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 3132 CurArgOffset = ArgOffset; 3133 }; 3134 3135 if (CallConv != CallingConv::Fast) { 3136 ComputeArgOffset(); 3137 3138 /* Compute GPR index associated with argument offset. */ 3139 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 3140 GPR_idx = std::min(GPR_idx, Num_GPR_Regs); 3141 } 3142 3143 // FIXME the codegen can be much improved in some cases. 3144 // We do not have to keep everything in memory. 3145 if (Flags.isByVal()) { 3146 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 3147 3148 if (CallConv == CallingConv::Fast) 3149 ComputeArgOffset(); 3150 3151 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 3152 ObjSize = Flags.getByValSize(); 3153 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3154 // Empty aggregate parameters do not take up registers. Examples: 3155 // struct { } a; 3156 // union { } b; 3157 // int c[0]; 3158 // etc. However, we have to provide a place-holder in InVals, so 3159 // pretend we have an 8-byte item at the current address for that 3160 // purpose. 3161 if (!ObjSize) { 3162 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true); 3163 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3164 InVals.push_back(FIN); 3165 continue; 3166 } 3167 3168 // Create a stack object covering all stack doublewords occupied 3169 // by the argument. If the argument is (fully or partially) on 3170 // the stack, or if the argument is fully in registers but the 3171 // caller has allocated the parameter save anyway, we can refer 3172 // directly to the caller's stack frame. Otherwise, create a 3173 // local copy in our own frame. 3174 int FI; 3175 if (HasParameterArea || 3176 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize) 3177 FI = MFI->CreateFixedObject(ArgSize, ArgOffset, false, true); 3178 else 3179 FI = MFI->CreateStackObject(ArgSize, Align, false); 3180 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3181 3182 // Handle aggregates smaller than 8 bytes. 3183 if (ObjSize < PtrByteSize) { 3184 // The value of the object is its address, which differs from the 3185 // address of the enclosing doubleword on big-endian systems. 3186 SDValue Arg = FIN; 3187 if (!isLittleEndian) { 3188 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT); 3189 Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff); 3190 } 3191 InVals.push_back(Arg); 3192 3193 if (GPR_idx != Num_GPR_Regs) { 3194 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3195 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3196 SDValue Store; 3197 3198 if (ObjSize==1 || ObjSize==2 || ObjSize==4) { 3199 EVT ObjType = (ObjSize == 1 ? MVT::i8 : 3200 (ObjSize == 2 ? MVT::i16 : MVT::i32)); 3201 Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg, 3202 MachinePointerInfo(&*FuncArg), ObjType, 3203 false, false, 0); 3204 } else { 3205 // For sizes that don't fit a truncating store (3, 5, 6, 7), 3206 // store the whole register as-is to the parameter save area 3207 // slot. 3208 Store = 3209 DAG.getStore(Val.getValue(1), dl, Val, FIN, 3210 MachinePointerInfo(&*FuncArg), false, false, 0); 3211 } 3212 3213 MemOps.push_back(Store); 3214 } 3215 // Whether we copied from a register or not, advance the offset 3216 // into the parameter save area by a full doubleword. 3217 ArgOffset += PtrByteSize; 3218 continue; 3219 } 3220 3221 // The value of the object is its address, which is the address of 3222 // its first stack doubleword. 3223 InVals.push_back(FIN); 3224 3225 // Store whatever pieces of the object are in registers to memory. 3226 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 3227 if (GPR_idx == Num_GPR_Regs) 3228 break; 3229 3230 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3231 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3232 SDValue Addr = FIN; 3233 if (j) { 3234 SDValue Off = DAG.getConstant(j, dl, PtrVT); 3235 Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off); 3236 } 3237 SDValue Store = 3238 DAG.getStore(Val.getValue(1), dl, Val, Addr, 3239 MachinePointerInfo(&*FuncArg, j), false, false, 0); 3240 MemOps.push_back(Store); 3241 ++GPR_idx; 3242 } 3243 ArgOffset += ArgSize; 3244 continue; 3245 } 3246 3247 switch (ObjectVT.getSimpleVT().SimpleTy) { 3248 default: llvm_unreachable("Unhandled argument type!"); 3249 case MVT::i1: 3250 case MVT::i32: 3251 case MVT::i64: 3252 if (Flags.isNest()) { 3253 // The 'nest' parameter, if any, is passed in R11. 3254 unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass); 3255 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3256 3257 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3258 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3259 3260 break; 3261 } 3262 3263 // These can be scalar arguments or elements of an integer array type 3264 // passed directly. Clang may use those instead of "byval" aggregate 3265 // types to avoid forcing arguments to memory unnecessarily. 3266 if (GPR_idx != Num_GPR_Regs) { 3267 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3268 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3269 3270 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3271 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3272 // value to MVT::i64 and then truncate to the correct register size. 3273 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3274 } else { 3275 if (CallConv == CallingConv::Fast) 3276 ComputeArgOffset(); 3277 3278 needsLoad = true; 3279 ArgSize = PtrByteSize; 3280 } 3281 if (CallConv != CallingConv::Fast || needsLoad) 3282 ArgOffset += 8; 3283 break; 3284 3285 case MVT::f32: 3286 case MVT::f64: 3287 // These can be scalar arguments or elements of a float array type 3288 // passed directly. The latter are used to implement ELFv2 homogenous 3289 // float aggregates. 3290 if (FPR_idx != Num_FPR_Regs) { 3291 unsigned VReg; 3292 3293 if (ObjectVT == MVT::f32) 3294 VReg = MF.addLiveIn(FPR[FPR_idx], 3295 Subtarget.hasP8Vector() 3296 ? &PPC::VSSRCRegClass 3297 : &PPC::F4RCRegClass); 3298 else 3299 VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX() 3300 ? &PPC::VSFRCRegClass 3301 : &PPC::F8RCRegClass); 3302 3303 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3304 ++FPR_idx; 3305 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) { 3306 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 3307 // once we support fp <-> gpr moves. 3308 3309 // This can only ever happen in the presence of f32 array types, 3310 // since otherwise we never run out of FPRs before running out 3311 // of GPRs. 3312 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3313 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3314 3315 if (ObjectVT == MVT::f32) { 3316 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0)) 3317 ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal, 3318 DAG.getConstant(32, dl, MVT::i32)); 3319 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal); 3320 } 3321 3322 ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal); 3323 } else { 3324 if (CallConv == CallingConv::Fast) 3325 ComputeArgOffset(); 3326 3327 needsLoad = true; 3328 } 3329 3330 // When passing an array of floats, the array occupies consecutive 3331 // space in the argument area; only round up to the next doubleword 3332 // at the end of the array. Otherwise, each float takes 8 bytes. 3333 if (CallConv != CallingConv::Fast || needsLoad) { 3334 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize; 3335 ArgOffset += ArgSize; 3336 if (Flags.isInConsecutiveRegsLast()) 3337 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3338 } 3339 break; 3340 case MVT::v4f32: 3341 case MVT::v4i32: 3342 case MVT::v8i16: 3343 case MVT::v16i8: 3344 case MVT::v2f64: 3345 case MVT::v2i64: 3346 case MVT::v1i128: 3347 if (!Subtarget.hasQPX()) { 3348 // These can be scalar arguments or elements of a vector array type 3349 // passed directly. The latter are used to implement ELFv2 homogenous 3350 // vector aggregates. 3351 if (VR_idx != Num_VR_Regs) { 3352 unsigned VReg = (ObjectVT == MVT::v2f64 || ObjectVT == MVT::v2i64) ? 3353 MF.addLiveIn(VSRH[VR_idx], &PPC::VSHRCRegClass) : 3354 MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 3355 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3356 ++VR_idx; 3357 } else { 3358 if (CallConv == CallingConv::Fast) 3359 ComputeArgOffset(); 3360 3361 needsLoad = true; 3362 } 3363 if (CallConv != CallingConv::Fast || needsLoad) 3364 ArgOffset += 16; 3365 break; 3366 } // not QPX 3367 3368 assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 && 3369 "Invalid QPX parameter type"); 3370 /* fall through */ 3371 3372 case MVT::v4f64: 3373 case MVT::v4i1: 3374 // QPX vectors are treated like their scalar floating-point subregisters 3375 // (except that they're larger). 3376 unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32; 3377 if (QFPR_idx != Num_QFPR_Regs) { 3378 const TargetRegisterClass *RC; 3379 switch (ObjectVT.getSimpleVT().SimpleTy) { 3380 case MVT::v4f64: RC = &PPC::QFRCRegClass; break; 3381 case MVT::v4f32: RC = &PPC::QSRCRegClass; break; 3382 default: RC = &PPC::QBRCRegClass; break; 3383 } 3384 3385 unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC); 3386 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3387 ++QFPR_idx; 3388 } else { 3389 if (CallConv == CallingConv::Fast) 3390 ComputeArgOffset(); 3391 needsLoad = true; 3392 } 3393 if (CallConv != CallingConv::Fast || needsLoad) 3394 ArgOffset += Sz; 3395 break; 3396 } 3397 3398 // We need to load the argument to a virtual register if we determined 3399 // above that we ran out of physical registers of the appropriate type. 3400 if (needsLoad) { 3401 if (ObjSize < ArgSize && !isLittleEndian) 3402 CurArgOffset += ArgSize - ObjSize; 3403 int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, isImmutable); 3404 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3405 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(), 3406 false, false, false, 0); 3407 } 3408 3409 InVals.push_back(ArgVal); 3410 } 3411 3412 // Area that is at least reserved in the caller of this function. 3413 unsigned MinReservedArea; 3414 if (HasParameterArea) 3415 MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize); 3416 else 3417 MinReservedArea = LinkageSize; 3418 3419 // Set the size that is at least reserved in caller of this function. Tail 3420 // call optimized functions' reserved stack space needs to be aligned so that 3421 // taking the difference between two stack areas will result in an aligned 3422 // stack. 3423 MinReservedArea = 3424 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 3425 FuncInfo->setMinReservedArea(MinReservedArea); 3426 3427 // If the function takes variable number of arguments, make a frame index for 3428 // the start of the first vararg value... for expansion of llvm.va_start. 3429 if (isVarArg) { 3430 int Depth = ArgOffset; 3431 3432 FuncInfo->setVarArgsFrameIndex( 3433 MFI->CreateFixedObject(PtrByteSize, Depth, true)); 3434 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3435 3436 // If this function is vararg, store any remaining integer argument regs 3437 // to their spots on the stack so that they may be loaded by deferencing the 3438 // result of va_next. 3439 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 3440 GPR_idx < Num_GPR_Regs; ++GPR_idx) { 3441 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3442 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3443 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3444 MachinePointerInfo(), false, false, 0); 3445 MemOps.push_back(Store); 3446 // Increment the address by four for the next argument to store 3447 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); 3448 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3449 } 3450 } 3451 3452 if (!MemOps.empty()) 3453 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3454 3455 return Chain; 3456 } 3457 3458 SDValue 3459 PPCTargetLowering::LowerFormalArguments_Darwin( 3460 SDValue Chain, 3461 CallingConv::ID CallConv, bool isVarArg, 3462 const SmallVectorImpl<ISD::InputArg> 3463 &Ins, 3464 SDLoc dl, SelectionDAG &DAG, 3465 SmallVectorImpl<SDValue> &InVals) const { 3466 // TODO: add description of PPC stack frame format, or at least some docs. 3467 // 3468 MachineFunction &MF = DAG.getMachineFunction(); 3469 MachineFrameInfo *MFI = MF.getFrameInfo(); 3470 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3471 3472 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 3473 bool isPPC64 = PtrVT == MVT::i64; 3474 // Potential tail calls could cause overwriting of argument stack slots. 3475 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 3476 (CallConv == CallingConv::Fast)); 3477 unsigned PtrByteSize = isPPC64 ? 8 : 4; 3478 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 3479 unsigned ArgOffset = LinkageSize; 3480 // Area that is at least reserved in caller of this function. 3481 unsigned MinReservedArea = ArgOffset; 3482 3483 static const MCPhysReg GPR_32[] = { // 32-bit registers. 3484 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 3485 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 3486 }; 3487 static const MCPhysReg GPR_64[] = { // 64-bit registers. 3488 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 3489 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 3490 }; 3491 static const MCPhysReg VR[] = { 3492 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 3493 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 3494 }; 3495 3496 const unsigned Num_GPR_Regs = array_lengthof(GPR_32); 3497 const unsigned Num_FPR_Regs = 13; 3498 const unsigned Num_VR_Regs = array_lengthof( VR); 3499 3500 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 3501 3502 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 3503 3504 // In 32-bit non-varargs functions, the stack space for vectors is after the 3505 // stack space for non-vectors. We do not use this space unless we have 3506 // too many vectors to fit in registers, something that only occurs in 3507 // constructed examples:), but we have to walk the arglist to figure 3508 // that out...for the pathological case, compute VecArgOffset as the 3509 // start of the vector parameter area. Computing VecArgOffset is the 3510 // entire point of the following loop. 3511 unsigned VecArgOffset = ArgOffset; 3512 if (!isVarArg && !isPPC64) { 3513 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; 3514 ++ArgNo) { 3515 EVT ObjectVT = Ins[ArgNo].VT; 3516 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 3517 3518 if (Flags.isByVal()) { 3519 // ObjSize is the true size, ArgSize rounded up to multiple of regs. 3520 unsigned ObjSize = Flags.getByValSize(); 3521 unsigned ArgSize = 3522 ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3523 VecArgOffset += ArgSize; 3524 continue; 3525 } 3526 3527 switch(ObjectVT.getSimpleVT().SimpleTy) { 3528 default: llvm_unreachable("Unhandled argument type!"); 3529 case MVT::i1: 3530 case MVT::i32: 3531 case MVT::f32: 3532 VecArgOffset += 4; 3533 break; 3534 case MVT::i64: // PPC64 3535 case MVT::f64: 3536 // FIXME: We are guaranteed to be !isPPC64 at this point. 3537 // Does MVT::i64 apply? 3538 VecArgOffset += 8; 3539 break; 3540 case MVT::v4f32: 3541 case MVT::v4i32: 3542 case MVT::v8i16: 3543 case MVT::v16i8: 3544 // Nothing to do, we're only looking at Nonvector args here. 3545 break; 3546 } 3547 } 3548 } 3549 // We've found where the vector parameter area in memory is. Skip the 3550 // first 12 parameters; these don't use that memory. 3551 VecArgOffset = ((VecArgOffset+15)/16)*16; 3552 VecArgOffset += 12*16; 3553 3554 // Add DAG nodes to load the arguments or copy them out of registers. On 3555 // entry to a function on PPC, the arguments start after the linkage area, 3556 // although the first ones are often in registers. 3557 3558 SmallVector<SDValue, 8> MemOps; 3559 unsigned nAltivecParamsAtEnd = 0; 3560 Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin(); 3561 unsigned CurArgIdx = 0; 3562 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 3563 SDValue ArgVal; 3564 bool needsLoad = false; 3565 EVT ObjectVT = Ins[ArgNo].VT; 3566 unsigned ObjSize = ObjectVT.getSizeInBits()/8; 3567 unsigned ArgSize = ObjSize; 3568 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 3569 if (Ins[ArgNo].isOrigArg()) { 3570 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 3571 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 3572 } 3573 unsigned CurArgOffset = ArgOffset; 3574 3575 // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. 3576 if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || 3577 ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { 3578 if (isVarArg || isPPC64) { 3579 MinReservedArea = ((MinReservedArea+15)/16)*16; 3580 MinReservedArea += CalculateStackSlotSize(ObjectVT, 3581 Flags, 3582 PtrByteSize); 3583 } else nAltivecParamsAtEnd++; 3584 } else 3585 // Calculate min reserved area. 3586 MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, 3587 Flags, 3588 PtrByteSize); 3589 3590 // FIXME the codegen can be much improved in some cases. 3591 // We do not have to keep everything in memory. 3592 if (Flags.isByVal()) { 3593 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 3594 3595 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 3596 ObjSize = Flags.getByValSize(); 3597 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3598 // Objects of size 1 and 2 are right justified, everything else is 3599 // left justified. This means the memory address is adjusted forwards. 3600 if (ObjSize==1 || ObjSize==2) { 3601 CurArgOffset = CurArgOffset + (4 - ObjSize); 3602 } 3603 // The value of the object is its address. 3604 int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, false, true); 3605 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3606 InVals.push_back(FIN); 3607 if (ObjSize==1 || ObjSize==2) { 3608 if (GPR_idx != Num_GPR_Regs) { 3609 unsigned VReg; 3610 if (isPPC64) 3611 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3612 else 3613 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 3614 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3615 EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; 3616 SDValue Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, 3617 MachinePointerInfo(&*FuncArg), 3618 ObjType, false, false, 0); 3619 MemOps.push_back(Store); 3620 ++GPR_idx; 3621 } 3622 3623 ArgOffset += PtrByteSize; 3624 3625 continue; 3626 } 3627 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 3628 // Store whatever pieces of the object are in registers 3629 // to memory. ArgOffset will be the address of the beginning 3630 // of the object. 3631 if (GPR_idx != Num_GPR_Regs) { 3632 unsigned VReg; 3633 if (isPPC64) 3634 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3635 else 3636 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 3637 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true); 3638 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3639 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3640 SDValue Store = 3641 DAG.getStore(Val.getValue(1), dl, Val, FIN, 3642 MachinePointerInfo(&*FuncArg, j), false, false, 0); 3643 MemOps.push_back(Store); 3644 ++GPR_idx; 3645 ArgOffset += PtrByteSize; 3646 } else { 3647 ArgOffset += ArgSize - (ArgOffset-CurArgOffset); 3648 break; 3649 } 3650 } 3651 continue; 3652 } 3653 3654 switch (ObjectVT.getSimpleVT().SimpleTy) { 3655 default: llvm_unreachable("Unhandled argument type!"); 3656 case MVT::i1: 3657 case MVT::i32: 3658 if (!isPPC64) { 3659 if (GPR_idx != Num_GPR_Regs) { 3660 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 3661 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); 3662 3663 if (ObjectVT == MVT::i1) 3664 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); 3665 3666 ++GPR_idx; 3667 } else { 3668 needsLoad = true; 3669 ArgSize = PtrByteSize; 3670 } 3671 // All int arguments reserve stack space in the Darwin ABI. 3672 ArgOffset += PtrByteSize; 3673 break; 3674 } 3675 // FALLTHROUGH 3676 case MVT::i64: // PPC64 3677 if (GPR_idx != Num_GPR_Regs) { 3678 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3679 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3680 3681 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3682 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3683 // value to MVT::i64 and then truncate to the correct register size. 3684 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3685 3686 ++GPR_idx; 3687 } else { 3688 needsLoad = true; 3689 ArgSize = PtrByteSize; 3690 } 3691 // All int arguments reserve stack space in the Darwin ABI. 3692 ArgOffset += 8; 3693 break; 3694 3695 case MVT::f32: 3696 case MVT::f64: 3697 // Every 4 bytes of argument space consumes one of the GPRs available for 3698 // argument passing. 3699 if (GPR_idx != Num_GPR_Regs) { 3700 ++GPR_idx; 3701 if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) 3702 ++GPR_idx; 3703 } 3704 if (FPR_idx != Num_FPR_Regs) { 3705 unsigned VReg; 3706 3707 if (ObjectVT == MVT::f32) 3708 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); 3709 else 3710 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); 3711 3712 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3713 ++FPR_idx; 3714 } else { 3715 needsLoad = true; 3716 } 3717 3718 // All FP arguments reserve stack space in the Darwin ABI. 3719 ArgOffset += isPPC64 ? 8 : ObjSize; 3720 break; 3721 case MVT::v4f32: 3722 case MVT::v4i32: 3723 case MVT::v8i16: 3724 case MVT::v16i8: 3725 // Note that vector arguments in registers don't reserve stack space, 3726 // except in varargs functions. 3727 if (VR_idx != Num_VR_Regs) { 3728 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 3729 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3730 if (isVarArg) { 3731 while ((ArgOffset % 16) != 0) { 3732 ArgOffset += PtrByteSize; 3733 if (GPR_idx != Num_GPR_Regs) 3734 GPR_idx++; 3735 } 3736 ArgOffset += 16; 3737 GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? 3738 } 3739 ++VR_idx; 3740 } else { 3741 if (!isVarArg && !isPPC64) { 3742 // Vectors go after all the nonvectors. 3743 CurArgOffset = VecArgOffset; 3744 VecArgOffset += 16; 3745 } else { 3746 // Vectors are aligned. 3747 ArgOffset = ((ArgOffset+15)/16)*16; 3748 CurArgOffset = ArgOffset; 3749 ArgOffset += 16; 3750 } 3751 needsLoad = true; 3752 } 3753 break; 3754 } 3755 3756 // We need to load the argument to a virtual register if we determined above 3757 // that we ran out of physical registers of the appropriate type. 3758 if (needsLoad) { 3759 int FI = MFI->CreateFixedObject(ObjSize, 3760 CurArgOffset + (ArgSize - ObjSize), 3761 isImmutable); 3762 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3763 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(), 3764 false, false, false, 0); 3765 } 3766 3767 InVals.push_back(ArgVal); 3768 } 3769 3770 // Allow for Altivec parameters at the end, if needed. 3771 if (nAltivecParamsAtEnd) { 3772 MinReservedArea = ((MinReservedArea+15)/16)*16; 3773 MinReservedArea += 16*nAltivecParamsAtEnd; 3774 } 3775 3776 // Area that is at least reserved in the caller of this function. 3777 MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize); 3778 3779 // Set the size that is at least reserved in caller of this function. Tail 3780 // call optimized functions' reserved stack space needs to be aligned so that 3781 // taking the difference between two stack areas will result in an aligned 3782 // stack. 3783 MinReservedArea = 3784 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 3785 FuncInfo->setMinReservedArea(MinReservedArea); 3786 3787 // If the function takes variable number of arguments, make a frame index for 3788 // the start of the first vararg value... for expansion of llvm.va_start. 3789 if (isVarArg) { 3790 int Depth = ArgOffset; 3791 3792 FuncInfo->setVarArgsFrameIndex( 3793 MFI->CreateFixedObject(PtrVT.getSizeInBits()/8, 3794 Depth, true)); 3795 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3796 3797 // If this function is vararg, store any remaining integer argument regs 3798 // to their spots on the stack so that they may be loaded by deferencing the 3799 // result of va_next. 3800 for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { 3801 unsigned VReg; 3802 3803 if (isPPC64) 3804 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3805 else 3806 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 3807 3808 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3809 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3810 MachinePointerInfo(), false, false, 0); 3811 MemOps.push_back(Store); 3812 // Increment the address by four for the next argument to store 3813 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 3814 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3815 } 3816 } 3817 3818 if (!MemOps.empty()) 3819 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3820 3821 return Chain; 3822 } 3823 3824 /// CalculateTailCallSPDiff - Get the amount the stack pointer has to be 3825 /// adjusted to accommodate the arguments for the tailcall. 3826 static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, 3827 unsigned ParamSize) { 3828 3829 if (!isTailCall) return 0; 3830 3831 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>(); 3832 unsigned CallerMinReservedArea = FI->getMinReservedArea(); 3833 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; 3834 // Remember only if the new adjustement is bigger. 3835 if (SPDiff < FI->getTailCallSPDelta()) 3836 FI->setTailCallSPDelta(SPDiff); 3837 3838 return SPDiff; 3839 } 3840 3841 /// IsEligibleForTailCallOptimization - Check whether the call is eligible 3842 /// for tail call optimization. Targets which want to do tail call 3843 /// optimization should implement this function. 3844 bool 3845 PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 3846 CallingConv::ID CalleeCC, 3847 bool isVarArg, 3848 const SmallVectorImpl<ISD::InputArg> &Ins, 3849 SelectionDAG& DAG) const { 3850 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 3851 return false; 3852 3853 // Variable argument functions are not supported. 3854 if (isVarArg) 3855 return false; 3856 3857 MachineFunction &MF = DAG.getMachineFunction(); 3858 CallingConv::ID CallerCC = MF.getFunction()->getCallingConv(); 3859 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 3860 // Functions containing by val parameters are not supported. 3861 for (unsigned i = 0; i != Ins.size(); i++) { 3862 ISD::ArgFlagsTy Flags = Ins[i].Flags; 3863 if (Flags.isByVal()) return false; 3864 } 3865 3866 // Non-PIC/GOT tail calls are supported. 3867 if (getTargetMachine().getRelocationModel() != Reloc::PIC_) 3868 return true; 3869 3870 // At the moment we can only do local tail calls (in same module, hidden 3871 // or protected) if we are generating PIC. 3872 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 3873 return G->getGlobal()->hasHiddenVisibility() 3874 || G->getGlobal()->hasProtectedVisibility(); 3875 } 3876 3877 return false; 3878 } 3879 3880 /// isCallCompatibleAddress - Return the immediate to use if the specified 3881 /// 32-bit value is representable in the immediate field of a BxA instruction. 3882 static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { 3883 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 3884 if (!C) return nullptr; 3885 3886 int Addr = C->getZExtValue(); 3887 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. 3888 SignExtend32<26>(Addr) != Addr) 3889 return nullptr; // Top 6 bits have to be sext of immediate. 3890 3891 return DAG.getConstant((int)C->getZExtValue() >> 2, SDLoc(Op), 3892 DAG.getTargetLoweringInfo().getPointerTy( 3893 DAG.getDataLayout())).getNode(); 3894 } 3895 3896 namespace { 3897 3898 struct TailCallArgumentInfo { 3899 SDValue Arg; 3900 SDValue FrameIdxOp; 3901 int FrameIdx; 3902 3903 TailCallArgumentInfo() : FrameIdx(0) {} 3904 }; 3905 } 3906 3907 /// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. 3908 static void 3909 StoreTailCallArgumentsToStackSlot(SelectionDAG &DAG, 3910 SDValue Chain, 3911 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs, 3912 SmallVectorImpl<SDValue> &MemOpChains, 3913 SDLoc dl) { 3914 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { 3915 SDValue Arg = TailCallArgs[i].Arg; 3916 SDValue FIN = TailCallArgs[i].FrameIdxOp; 3917 int FI = TailCallArgs[i].FrameIdx; 3918 // Store relative to framepointer. 3919 MemOpChains.push_back(DAG.getStore( 3920 Chain, dl, Arg, FIN, 3921 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), false, 3922 false, 0)); 3923 } 3924 } 3925 3926 /// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to 3927 /// the appropriate stack slot for the tail call optimized function call. 3928 static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, 3929 MachineFunction &MF, 3930 SDValue Chain, 3931 SDValue OldRetAddr, 3932 SDValue OldFP, 3933 int SPDiff, 3934 bool isPPC64, 3935 bool isDarwinABI, 3936 SDLoc dl) { 3937 if (SPDiff) { 3938 // Calculate the new stack slot for the return address. 3939 int SlotSize = isPPC64 ? 8 : 4; 3940 const PPCFrameLowering *FL = 3941 MF.getSubtarget<PPCSubtarget>().getFrameLowering(); 3942 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset(); 3943 int NewRetAddr = MF.getFrameInfo()->CreateFixedObject(SlotSize, 3944 NewRetAddrLoc, true); 3945 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 3946 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); 3947 Chain = DAG.getStore( 3948 Chain, dl, OldRetAddr, NewRetAddrFrIdx, 3949 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), NewRetAddr), 3950 false, false, 0); 3951 3952 // When using the 32/64-bit SVR4 ABI there is no need to move the FP stack 3953 // slot as the FP is never overwritten. 3954 if (isDarwinABI) { 3955 int NewFPLoc = SPDiff + FL->getFramePointerSaveOffset(); 3956 int NewFPIdx = MF.getFrameInfo()->CreateFixedObject(SlotSize, NewFPLoc, 3957 true); 3958 SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT); 3959 Chain = DAG.getStore( 3960 Chain, dl, OldFP, NewFramePtrIdx, 3961 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), NewFPIdx), 3962 false, false, 0); 3963 } 3964 } 3965 return Chain; 3966 } 3967 3968 /// CalculateTailCallArgDest - Remember Argument for later processing. Calculate 3969 /// the position of the argument. 3970 static void 3971 CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, 3972 SDValue Arg, int SPDiff, unsigned ArgOffset, 3973 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) { 3974 int Offset = ArgOffset + SPDiff; 3975 uint32_t OpSize = (Arg.getValueType().getSizeInBits()+7)/8; 3976 int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true); 3977 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 3978 SDValue FIN = DAG.getFrameIndex(FI, VT); 3979 TailCallArgumentInfo Info; 3980 Info.Arg = Arg; 3981 Info.FrameIdxOp = FIN; 3982 Info.FrameIdx = FI; 3983 TailCallArguments.push_back(Info); 3984 } 3985 3986 /// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address 3987 /// stack slot. Returns the chain as result and the loaded frame pointers in 3988 /// LROpOut/FPOpout. Used when tail calling. 3989 SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(SelectionDAG & DAG, 3990 int SPDiff, 3991 SDValue Chain, 3992 SDValue &LROpOut, 3993 SDValue &FPOpOut, 3994 bool isDarwinABI, 3995 SDLoc dl) const { 3996 if (SPDiff) { 3997 // Load the LR and FP stack slot for later adjusting. 3998 EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 3999 LROpOut = getReturnAddrFrameIndex(DAG); 4000 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo(), 4001 false, false, false, 0); 4002 Chain = SDValue(LROpOut.getNode(), 1); 4003 4004 // When using the 32/64-bit SVR4 ABI there is no need to load the FP stack 4005 // slot as the FP is never overwritten. 4006 if (isDarwinABI) { 4007 FPOpOut = getFramePointerFrameIndex(DAG); 4008 FPOpOut = DAG.getLoad(VT, dl, Chain, FPOpOut, MachinePointerInfo(), 4009 false, false, false, 0); 4010 Chain = SDValue(FPOpOut.getNode(), 1); 4011 } 4012 } 4013 return Chain; 4014 } 4015 4016 /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 4017 /// by "Src" to address "Dst" of size "Size". Alignment information is 4018 /// specified by the specific parameter attribute. The copy will be passed as 4019 /// a byval function parameter. 4020 /// Sometimes what we are copying is the end of a larger object, the part that 4021 /// does not fit in registers. 4022 static SDValue 4023 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 4024 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, 4025 SDLoc dl) { 4026 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); 4027 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 4028 false, false, false, MachinePointerInfo(), 4029 MachinePointerInfo()); 4030 } 4031 4032 /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of 4033 /// tail calls. 4034 static void 4035 LowerMemOpCallTo(SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, 4036 SDValue Arg, SDValue PtrOff, int SPDiff, 4037 unsigned ArgOffset, bool isPPC64, bool isTailCall, 4038 bool isVector, SmallVectorImpl<SDValue> &MemOpChains, 4039 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, 4040 SDLoc dl) { 4041 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4042 if (!isTailCall) { 4043 if (isVector) { 4044 SDValue StackPtr; 4045 if (isPPC64) 4046 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 4047 else 4048 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 4049 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 4050 DAG.getConstant(ArgOffset, dl, PtrVT)); 4051 } 4052 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, 4053 MachinePointerInfo(), false, false, 0)); 4054 // Calculate and remember argument location. 4055 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, 4056 TailCallArguments); 4057 } 4058 4059 static 4060 void PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, 4061 SDLoc dl, bool isPPC64, int SPDiff, unsigned NumBytes, 4062 SDValue LROp, SDValue FPOp, bool isDarwinABI, 4063 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) { 4064 MachineFunction &MF = DAG.getMachineFunction(); 4065 4066 // Emit a sequence of copyto/copyfrom virtual registers for arguments that 4067 // might overwrite each other in case of tail call optimization. 4068 SmallVector<SDValue, 8> MemOpChains2; 4069 // Do not flag preceding copytoreg stuff together with the following stuff. 4070 InFlag = SDValue(); 4071 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, 4072 MemOpChains2, dl); 4073 if (!MemOpChains2.empty()) 4074 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); 4075 4076 // Store the return address to the appropriate stack slot. 4077 Chain = EmitTailCallStoreFPAndRetAddr(DAG, MF, Chain, LROp, FPOp, SPDiff, 4078 isPPC64, isDarwinABI, dl); 4079 4080 // Emit callseq_end just before tailcall node. 4081 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 4082 DAG.getIntPtrConstant(0, dl, true), InFlag, dl); 4083 InFlag = Chain.getValue(1); 4084 } 4085 4086 // Is this global address that of a function that can be called by name? (as 4087 // opposed to something that must hold a descriptor for an indirect call). 4088 static bool isFunctionGlobalAddress(SDValue Callee) { 4089 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 4090 if (Callee.getOpcode() == ISD::GlobalTLSAddress || 4091 Callee.getOpcode() == ISD::TargetGlobalTLSAddress) 4092 return false; 4093 4094 return G->getGlobal()->getType()->getElementType()->isFunctionTy(); 4095 } 4096 4097 return false; 4098 } 4099 4100 static 4101 unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, 4102 SDValue &Chain, SDValue CallSeqStart, SDLoc dl, int SPDiff, 4103 bool isTailCall, bool IsPatchPoint, bool hasNest, 4104 SmallVectorImpl<std::pair<unsigned, SDValue> > &RegsToPass, 4105 SmallVectorImpl<SDValue> &Ops, std::vector<EVT> &NodeTys, 4106 ImmutableCallSite *CS, const PPCSubtarget &Subtarget) { 4107 4108 bool isPPC64 = Subtarget.isPPC64(); 4109 bool isSVR4ABI = Subtarget.isSVR4ABI(); 4110 bool isELFv2ABI = Subtarget.isELFv2ABI(); 4111 4112 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4113 NodeTys.push_back(MVT::Other); // Returns a chain 4114 NodeTys.push_back(MVT::Glue); // Returns a flag for retval copy to use. 4115 4116 unsigned CallOpc = PPCISD::CALL; 4117 4118 bool needIndirectCall = true; 4119 if (!isSVR4ABI || !isPPC64) 4120 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) { 4121 // If this is an absolute destination address, use the munged value. 4122 Callee = SDValue(Dest, 0); 4123 needIndirectCall = false; 4124 } 4125 4126 if (isFunctionGlobalAddress(Callee)) { 4127 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee); 4128 // A call to a TLS address is actually an indirect call to a 4129 // thread-specific pointer. 4130 unsigned OpFlags = 0; 4131 if ((DAG.getTarget().getRelocationModel() != Reloc::Static && 4132 (Subtarget.getTargetTriple().isMacOSX() && 4133 Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5)) && 4134 !G->getGlobal()->isStrongDefinitionForLinker()) || 4135 (Subtarget.isTargetELF() && !isPPC64 && 4136 !G->getGlobal()->hasLocalLinkage() && 4137 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) { 4138 // PC-relative references to external symbols should go through $stub, 4139 // unless we're building with the leopard linker or later, which 4140 // automatically synthesizes these stubs. 4141 OpFlags = PPCII::MO_PLT_OR_STUB; 4142 } 4143 4144 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, 4145 // every direct call is) turn it into a TargetGlobalAddress / 4146 // TargetExternalSymbol node so that legalize doesn't hack it. 4147 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, 4148 Callee.getValueType(), 0, OpFlags); 4149 needIndirectCall = false; 4150 } 4151 4152 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 4153 unsigned char OpFlags = 0; 4154 4155 if ((DAG.getTarget().getRelocationModel() != Reloc::Static && 4156 (Subtarget.getTargetTriple().isMacOSX() && 4157 Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5))) || 4158 (Subtarget.isTargetELF() && !isPPC64 && 4159 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) { 4160 // PC-relative references to external symbols should go through $stub, 4161 // unless we're building with the leopard linker or later, which 4162 // automatically synthesizes these stubs. 4163 OpFlags = PPCII::MO_PLT_OR_STUB; 4164 } 4165 4166 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), Callee.getValueType(), 4167 OpFlags); 4168 needIndirectCall = false; 4169 } 4170 4171 if (IsPatchPoint) { 4172 // We'll form an invalid direct call when lowering a patchpoint; the full 4173 // sequence for an indirect call is complicated, and many of the 4174 // instructions introduced might have side effects (and, thus, can't be 4175 // removed later). The call itself will be removed as soon as the 4176 // argument/return lowering is complete, so the fact that it has the wrong 4177 // kind of operands should not really matter. 4178 needIndirectCall = false; 4179 } 4180 4181 if (needIndirectCall) { 4182 // Otherwise, this is an indirect call. We have to use a MTCTR/BCTRL pair 4183 // to do the call, we can't use PPCISD::CALL. 4184 SDValue MTCTROps[] = {Chain, Callee, InFlag}; 4185 4186 if (isSVR4ABI && isPPC64 && !isELFv2ABI) { 4187 // Function pointers in the 64-bit SVR4 ABI do not point to the function 4188 // entry point, but to the function descriptor (the function entry point 4189 // address is part of the function descriptor though). 4190 // The function descriptor is a three doubleword structure with the 4191 // following fields: function entry point, TOC base address and 4192 // environment pointer. 4193 // Thus for a call through a function pointer, the following actions need 4194 // to be performed: 4195 // 1. Save the TOC of the caller in the TOC save area of its stack 4196 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). 4197 // 2. Load the address of the function entry point from the function 4198 // descriptor. 4199 // 3. Load the TOC of the callee from the function descriptor into r2. 4200 // 4. Load the environment pointer from the function descriptor into 4201 // r11. 4202 // 5. Branch to the function entry point address. 4203 // 6. On return of the callee, the TOC of the caller needs to be 4204 // restored (this is done in FinishCall()). 4205 // 4206 // The loads are scheduled at the beginning of the call sequence, and the 4207 // register copies are flagged together to ensure that no other 4208 // operations can be scheduled in between. E.g. without flagging the 4209 // copies together, a TOC access in the caller could be scheduled between 4210 // the assignment of the callee TOC and the branch to the callee, which 4211 // results in the TOC access going through the TOC of the callee instead 4212 // of going through the TOC of the caller, which leads to incorrect code. 4213 4214 // Load the address of the function entry point from the function 4215 // descriptor. 4216 SDValue LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-1); 4217 if (LDChain.getValueType() == MVT::Glue) 4218 LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-2); 4219 4220 bool LoadsInv = Subtarget.hasInvariantFunctionDescriptors(); 4221 4222 MachinePointerInfo MPI(CS ? CS->getCalledValue() : nullptr); 4223 SDValue LoadFuncPtr = DAG.getLoad(MVT::i64, dl, LDChain, Callee, MPI, 4224 false, false, LoadsInv, 8); 4225 4226 // Load environment pointer into r11. 4227 SDValue PtrOff = DAG.getIntPtrConstant(16, dl); 4228 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff); 4229 SDValue LoadEnvPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddPtr, 4230 MPI.getWithOffset(16), false, false, 4231 LoadsInv, 8); 4232 4233 SDValue TOCOff = DAG.getIntPtrConstant(8, dl); 4234 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, TOCOff); 4235 SDValue TOCPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddTOC, 4236 MPI.getWithOffset(8), false, false, 4237 LoadsInv, 8); 4238 4239 setUsesTOCBasePtr(DAG); 4240 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, PPC::X2, TOCPtr, 4241 InFlag); 4242 Chain = TOCVal.getValue(0); 4243 InFlag = TOCVal.getValue(1); 4244 4245 // If the function call has an explicit 'nest' parameter, it takes the 4246 // place of the environment pointer. 4247 if (!hasNest) { 4248 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr, 4249 InFlag); 4250 4251 Chain = EnvVal.getValue(0); 4252 InFlag = EnvVal.getValue(1); 4253 } 4254 4255 MTCTROps[0] = Chain; 4256 MTCTROps[1] = LoadFuncPtr; 4257 MTCTROps[2] = InFlag; 4258 } 4259 4260 Chain = DAG.getNode(PPCISD::MTCTR, dl, NodeTys, 4261 makeArrayRef(MTCTROps, InFlag.getNode() ? 3 : 2)); 4262 InFlag = Chain.getValue(1); 4263 4264 NodeTys.clear(); 4265 NodeTys.push_back(MVT::Other); 4266 NodeTys.push_back(MVT::Glue); 4267 Ops.push_back(Chain); 4268 CallOpc = PPCISD::BCTRL; 4269 Callee.setNode(nullptr); 4270 // Add use of X11 (holding environment pointer) 4271 if (isSVR4ABI && isPPC64 && !isELFv2ABI && !hasNest) 4272 Ops.push_back(DAG.getRegister(PPC::X11, PtrVT)); 4273 // Add CTR register as callee so a bctr can be emitted later. 4274 if (isTailCall) 4275 Ops.push_back(DAG.getRegister(isPPC64 ? PPC::CTR8 : PPC::CTR, PtrVT)); 4276 } 4277 4278 // If this is a direct call, pass the chain and the callee. 4279 if (Callee.getNode()) { 4280 Ops.push_back(Chain); 4281 Ops.push_back(Callee); 4282 } 4283 // If this is a tail call add stack pointer delta. 4284 if (isTailCall) 4285 Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32)); 4286 4287 // Add argument registers to the end of the list so that they are known live 4288 // into the call. 4289 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 4290 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 4291 RegsToPass[i].second.getValueType())); 4292 4293 // All calls, in both the ELF V1 and V2 ABIs, need the TOC register live 4294 // into the call. 4295 if (isSVR4ABI && isPPC64 && !IsPatchPoint) { 4296 setUsesTOCBasePtr(DAG); 4297 Ops.push_back(DAG.getRegister(PPC::X2, PtrVT)); 4298 } 4299 4300 return CallOpc; 4301 } 4302 4303 static 4304 bool isLocalCall(const SDValue &Callee) 4305 { 4306 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 4307 return G->getGlobal()->isStrongDefinitionForLinker(); 4308 return false; 4309 } 4310 4311 SDValue 4312 PPCTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag, 4313 CallingConv::ID CallConv, bool isVarArg, 4314 const SmallVectorImpl<ISD::InputArg> &Ins, 4315 SDLoc dl, SelectionDAG &DAG, 4316 SmallVectorImpl<SDValue> &InVals) const { 4317 4318 SmallVector<CCValAssign, 16> RVLocs; 4319 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 4320 *DAG.getContext()); 4321 CCRetInfo.AnalyzeCallResult(Ins, RetCC_PPC); 4322 4323 // Copy all of the result registers out of their specified physreg. 4324 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 4325 CCValAssign &VA = RVLocs[i]; 4326 assert(VA.isRegLoc() && "Can only return in registers!"); 4327 4328 SDValue Val = DAG.getCopyFromReg(Chain, dl, 4329 VA.getLocReg(), VA.getLocVT(), InFlag); 4330 Chain = Val.getValue(1); 4331 InFlag = Val.getValue(2); 4332 4333 switch (VA.getLocInfo()) { 4334 default: llvm_unreachable("Unknown loc info!"); 4335 case CCValAssign::Full: break; 4336 case CCValAssign::AExt: 4337 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 4338 break; 4339 case CCValAssign::ZExt: 4340 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, 4341 DAG.getValueType(VA.getValVT())); 4342 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 4343 break; 4344 case CCValAssign::SExt: 4345 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, 4346 DAG.getValueType(VA.getValVT())); 4347 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 4348 break; 4349 } 4350 4351 InVals.push_back(Val); 4352 } 4353 4354 return Chain; 4355 } 4356 4357 SDValue 4358 PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl, 4359 bool isTailCall, bool isVarArg, bool IsPatchPoint, 4360 bool hasNest, SelectionDAG &DAG, 4361 SmallVector<std::pair<unsigned, SDValue>, 8> 4362 &RegsToPass, 4363 SDValue InFlag, SDValue Chain, 4364 SDValue CallSeqStart, SDValue &Callee, 4365 int SPDiff, unsigned NumBytes, 4366 const SmallVectorImpl<ISD::InputArg> &Ins, 4367 SmallVectorImpl<SDValue> &InVals, 4368 ImmutableCallSite *CS) const { 4369 4370 std::vector<EVT> NodeTys; 4371 SmallVector<SDValue, 8> Ops; 4372 unsigned CallOpc = PrepareCall(DAG, Callee, InFlag, Chain, CallSeqStart, dl, 4373 SPDiff, isTailCall, IsPatchPoint, hasNest, 4374 RegsToPass, Ops, NodeTys, CS, Subtarget); 4375 4376 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls 4377 if (isVarArg && Subtarget.isSVR4ABI() && !Subtarget.isPPC64()) 4378 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); 4379 4380 // When performing tail call optimization the callee pops its arguments off 4381 // the stack. Account for this here so these bytes can be pushed back on in 4382 // PPCFrameLowering::eliminateCallFramePseudoInstr. 4383 int BytesCalleePops = 4384 (CallConv == CallingConv::Fast && 4385 getTargetMachine().Options.GuaranteedTailCallOpt) ? NumBytes : 0; 4386 4387 // Add a register mask operand representing the call-preserved registers. 4388 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 4389 const uint32_t *Mask = 4390 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallConv); 4391 assert(Mask && "Missing call preserved mask for calling convention"); 4392 Ops.push_back(DAG.getRegisterMask(Mask)); 4393 4394 if (InFlag.getNode()) 4395 Ops.push_back(InFlag); 4396 4397 // Emit tail call. 4398 if (isTailCall) { 4399 assert(((Callee.getOpcode() == ISD::Register && 4400 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || 4401 Callee.getOpcode() == ISD::TargetExternalSymbol || 4402 Callee.getOpcode() == ISD::TargetGlobalAddress || 4403 isa<ConstantSDNode>(Callee)) && 4404 "Expecting an global address, external symbol, absolute value or register"); 4405 4406 DAG.getMachineFunction().getFrameInfo()->setHasTailCall(); 4407 return DAG.getNode(PPCISD::TC_RETURN, dl, MVT::Other, Ops); 4408 } 4409 4410 // Add a NOP immediately after the branch instruction when using the 64-bit 4411 // SVR4 ABI. At link time, if caller and callee are in a different module and 4412 // thus have a different TOC, the call will be replaced with a call to a stub 4413 // function which saves the current TOC, loads the TOC of the callee and 4414 // branches to the callee. The NOP will be replaced with a load instruction 4415 // which restores the TOC of the caller from the TOC save slot of the current 4416 // stack frame. If caller and callee belong to the same module (and have the 4417 // same TOC), the NOP will remain unchanged. 4418 4419 if (!isTailCall && Subtarget.isSVR4ABI()&& Subtarget.isPPC64() && 4420 !IsPatchPoint) { 4421 if (CallOpc == PPCISD::BCTRL) { 4422 // This is a call through a function pointer. 4423 // Restore the caller TOC from the save area into R2. 4424 // See PrepareCall() for more information about calls through function 4425 // pointers in the 64-bit SVR4 ABI. 4426 // We are using a target-specific load with r2 hard coded, because the 4427 // result of a target-independent load would never go directly into r2, 4428 // since r2 is a reserved register (which prevents the register allocator 4429 // from allocating it), resulting in an additional register being 4430 // allocated and an unnecessary move instruction being generated. 4431 CallOpc = PPCISD::BCTRL_LOAD_TOC; 4432 4433 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4434 SDValue StackPtr = DAG.getRegister(PPC::X1, PtrVT); 4435 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 4436 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 4437 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, TOCOff); 4438 4439 // The address needs to go after the chain input but before the flag (or 4440 // any other variadic arguments). 4441 Ops.insert(std::next(Ops.begin()), AddTOC); 4442 } else if ((CallOpc == PPCISD::CALL) && 4443 (!isLocalCall(Callee) || 4444 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) 4445 // Otherwise insert NOP for non-local calls. 4446 CallOpc = PPCISD::CALL_NOP; 4447 } 4448 4449 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); 4450 InFlag = Chain.getValue(1); 4451 4452 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 4453 DAG.getIntPtrConstant(BytesCalleePops, dl, true), 4454 InFlag, dl); 4455 if (!Ins.empty()) 4456 InFlag = Chain.getValue(1); 4457 4458 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, 4459 Ins, dl, DAG, InVals); 4460 } 4461 4462 SDValue 4463 PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 4464 SmallVectorImpl<SDValue> &InVals) const { 4465 SelectionDAG &DAG = CLI.DAG; 4466 SDLoc &dl = CLI.DL; 4467 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 4468 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 4469 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 4470 SDValue Chain = CLI.Chain; 4471 SDValue Callee = CLI.Callee; 4472 bool &isTailCall = CLI.IsTailCall; 4473 CallingConv::ID CallConv = CLI.CallConv; 4474 bool isVarArg = CLI.IsVarArg; 4475 bool IsPatchPoint = CLI.IsPatchPoint; 4476 ImmutableCallSite *CS = CLI.CS; 4477 4478 if (isTailCall) 4479 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, 4480 Ins, DAG); 4481 4482 if (!isTailCall && CS && CS->isMustTailCall()) 4483 report_fatal_error("failed to perform tail call elimination on a call " 4484 "site marked musttail"); 4485 4486 if (Subtarget.isSVR4ABI()) { 4487 if (Subtarget.isPPC64()) 4488 return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg, 4489 isTailCall, IsPatchPoint, Outs, OutVals, Ins, 4490 dl, DAG, InVals, CS); 4491 else 4492 return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg, 4493 isTailCall, IsPatchPoint, Outs, OutVals, Ins, 4494 dl, DAG, InVals, CS); 4495 } 4496 4497 return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg, 4498 isTailCall, IsPatchPoint, Outs, OutVals, Ins, 4499 dl, DAG, InVals, CS); 4500 } 4501 4502 SDValue 4503 PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee, 4504 CallingConv::ID CallConv, bool isVarArg, 4505 bool isTailCall, bool IsPatchPoint, 4506 const SmallVectorImpl<ISD::OutputArg> &Outs, 4507 const SmallVectorImpl<SDValue> &OutVals, 4508 const SmallVectorImpl<ISD::InputArg> &Ins, 4509 SDLoc dl, SelectionDAG &DAG, 4510 SmallVectorImpl<SDValue> &InVals, 4511 ImmutableCallSite *CS) const { 4512 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description 4513 // of the 32-bit SVR4 ABI stack frame layout. 4514 4515 assert((CallConv == CallingConv::C || 4516 CallConv == CallingConv::Fast) && "Unknown calling convention!"); 4517 4518 unsigned PtrByteSize = 4; 4519 4520 MachineFunction &MF = DAG.getMachineFunction(); 4521 4522 // Mark this function as potentially containing a function that contains a 4523 // tail call. As a consequence the frame pointer will be used for dynamicalloc 4524 // and restoring the callers stack pointer in this functions epilog. This is 4525 // done because by tail calling the called function might overwrite the value 4526 // in this function's (MF) stack pointer stack slot 0(SP). 4527 if (getTargetMachine().Options.GuaranteedTailCallOpt && 4528 CallConv == CallingConv::Fast) 4529 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 4530 4531 // Count how many bytes are to be pushed on the stack, including the linkage 4532 // area, parameter list area and the part of the local variable space which 4533 // contains copies of aggregates which are passed by value. 4534 4535 // Assign locations to all of the outgoing arguments. 4536 SmallVector<CCValAssign, 16> ArgLocs; 4537 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 4538 *DAG.getContext()); 4539 4540 // Reserve space for the linkage area on the stack. 4541 CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(), 4542 PtrByteSize); 4543 4544 if (isVarArg) { 4545 // Handle fixed and variable vector arguments differently. 4546 // Fixed vector arguments go into registers as long as registers are 4547 // available. Variable vector arguments always go into memory. 4548 unsigned NumArgs = Outs.size(); 4549 4550 for (unsigned i = 0; i != NumArgs; ++i) { 4551 MVT ArgVT = Outs[i].VT; 4552 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 4553 bool Result; 4554 4555 if (Outs[i].IsFixed) { 4556 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, 4557 CCInfo); 4558 } else { 4559 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, 4560 ArgFlags, CCInfo); 4561 } 4562 4563 if (Result) { 4564 #ifndef NDEBUG 4565 errs() << "Call operand #" << i << " has unhandled type " 4566 << EVT(ArgVT).getEVTString() << "\n"; 4567 #endif 4568 llvm_unreachable(nullptr); 4569 } 4570 } 4571 } else { 4572 // All arguments are treated the same. 4573 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); 4574 } 4575 4576 // Assign locations to all of the outgoing aggregate by value arguments. 4577 SmallVector<CCValAssign, 16> ByValArgLocs; 4578 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 4579 ByValArgLocs, *DAG.getContext()); 4580 4581 // Reserve stack space for the allocations in CCInfo. 4582 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 4583 4584 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); 4585 4586 // Size of the linkage area, parameter list area and the part of the local 4587 // space variable where copies of aggregates which are passed by value are 4588 // stored. 4589 unsigned NumBytes = CCByValInfo.getNextStackOffset(); 4590 4591 // Calculate by how many bytes the stack has to be adjusted in case of tail 4592 // call optimization. 4593 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 4594 4595 // Adjust the stack pointer for the new arguments... 4596 // These operations are automatically eliminated by the prolog/epilog pass 4597 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 4598 dl); 4599 SDValue CallSeqStart = Chain; 4600 4601 // Load the return address and frame pointer so it can be moved somewhere else 4602 // later. 4603 SDValue LROp, FPOp; 4604 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, false, 4605 dl); 4606 4607 // Set up a copy of the stack pointer for use loading and storing any 4608 // arguments that may not fit in the registers available for argument 4609 // passing. 4610 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 4611 4612 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 4613 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 4614 SmallVector<SDValue, 8> MemOpChains; 4615 4616 bool seenFloatArg = false; 4617 // Walk the register/memloc assignments, inserting copies/loads. 4618 for (unsigned i = 0, j = 0, e = ArgLocs.size(); 4619 i != e; 4620 ++i) { 4621 CCValAssign &VA = ArgLocs[i]; 4622 SDValue Arg = OutVals[i]; 4623 ISD::ArgFlagsTy Flags = Outs[i].Flags; 4624 4625 if (Flags.isByVal()) { 4626 // Argument is an aggregate which is passed by value, thus we need to 4627 // create a copy of it in the local variable space of the current stack 4628 // frame (which is the stack frame of the caller) and pass the address of 4629 // this copy to the callee. 4630 assert((j < ByValArgLocs.size()) && "Index out of bounds!"); 4631 CCValAssign &ByValVA = ByValArgLocs[j++]; 4632 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); 4633 4634 // Memory reserved in the local variable space of the callers stack frame. 4635 unsigned LocMemOffset = ByValVA.getLocMemOffset(); 4636 4637 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 4638 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 4639 StackPtr, PtrOff); 4640 4641 // Create a copy of the argument in the local area of the current 4642 // stack frame. 4643 SDValue MemcpyCall = 4644 CreateCopyOfByValArgument(Arg, PtrOff, 4645 CallSeqStart.getNode()->getOperand(0), 4646 Flags, DAG, dl); 4647 4648 // This must go outside the CALLSEQ_START..END. 4649 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, 4650 CallSeqStart.getNode()->getOperand(1), 4651 SDLoc(MemcpyCall)); 4652 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 4653 NewCallSeqStart.getNode()); 4654 Chain = CallSeqStart = NewCallSeqStart; 4655 4656 // Pass the address of the aggregate copy on the stack either in a 4657 // physical register or in the parameter list area of the current stack 4658 // frame to the callee. 4659 Arg = PtrOff; 4660 } 4661 4662 if (VA.isRegLoc()) { 4663 if (Arg.getValueType() == MVT::i1) 4664 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Arg); 4665 4666 seenFloatArg |= VA.getLocVT().isFloatingPoint(); 4667 // Put argument in a physical register. 4668 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 4669 } else { 4670 // Put argument in the parameter list area of the current stack frame. 4671 assert(VA.isMemLoc()); 4672 unsigned LocMemOffset = VA.getLocMemOffset(); 4673 4674 if (!isTailCall) { 4675 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 4676 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 4677 StackPtr, PtrOff); 4678 4679 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, 4680 MachinePointerInfo(), 4681 false, false, 0)); 4682 } else { 4683 // Calculate and remember argument location. 4684 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, 4685 TailCallArguments); 4686 } 4687 } 4688 } 4689 4690 if (!MemOpChains.empty()) 4691 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 4692 4693 // Build a sequence of copy-to-reg nodes chained together with token chain 4694 // and flag operands which copy the outgoing args into the appropriate regs. 4695 SDValue InFlag; 4696 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 4697 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 4698 RegsToPass[i].second, InFlag); 4699 InFlag = Chain.getValue(1); 4700 } 4701 4702 // Set CR bit 6 to true if this is a vararg call with floating args passed in 4703 // registers. 4704 if (isVarArg) { 4705 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 4706 SDValue Ops[] = { Chain, InFlag }; 4707 4708 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, 4709 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); 4710 4711 InFlag = Chain.getValue(1); 4712 } 4713 4714 if (isTailCall) 4715 PrepareTailCall(DAG, InFlag, Chain, dl, false, SPDiff, NumBytes, LROp, FPOp, 4716 false, TailCallArguments); 4717 4718 return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, 4719 /* unused except on PPC64 ELFv1 */ false, DAG, 4720 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 4721 NumBytes, Ins, InVals, CS); 4722 } 4723 4724 // Copy an argument into memory, being careful to do this outside the 4725 // call sequence for the call to which the argument belongs. 4726 SDValue 4727 PPCTargetLowering::createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff, 4728 SDValue CallSeqStart, 4729 ISD::ArgFlagsTy Flags, 4730 SelectionDAG &DAG, 4731 SDLoc dl) const { 4732 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, 4733 CallSeqStart.getNode()->getOperand(0), 4734 Flags, DAG, dl); 4735 // The MEMCPY must go outside the CALLSEQ_START..END. 4736 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, 4737 CallSeqStart.getNode()->getOperand(1), 4738 SDLoc(MemcpyCall)); 4739 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 4740 NewCallSeqStart.getNode()); 4741 return NewCallSeqStart; 4742 } 4743 4744 SDValue 4745 PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee, 4746 CallingConv::ID CallConv, bool isVarArg, 4747 bool isTailCall, bool IsPatchPoint, 4748 const SmallVectorImpl<ISD::OutputArg> &Outs, 4749 const SmallVectorImpl<SDValue> &OutVals, 4750 const SmallVectorImpl<ISD::InputArg> &Ins, 4751 SDLoc dl, SelectionDAG &DAG, 4752 SmallVectorImpl<SDValue> &InVals, 4753 ImmutableCallSite *CS) const { 4754 4755 bool isELFv2ABI = Subtarget.isELFv2ABI(); 4756 bool isLittleEndian = Subtarget.isLittleEndian(); 4757 unsigned NumOps = Outs.size(); 4758 bool hasNest = false; 4759 4760 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4761 unsigned PtrByteSize = 8; 4762 4763 MachineFunction &MF = DAG.getMachineFunction(); 4764 4765 // Mark this function as potentially containing a function that contains a 4766 // tail call. As a consequence the frame pointer will be used for dynamicalloc 4767 // and restoring the callers stack pointer in this functions epilog. This is 4768 // done because by tail calling the called function might overwrite the value 4769 // in this function's (MF) stack pointer stack slot 0(SP). 4770 if (getTargetMachine().Options.GuaranteedTailCallOpt && 4771 CallConv == CallingConv::Fast) 4772 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 4773 4774 assert(!(CallConv == CallingConv::Fast && isVarArg) && 4775 "fastcc not supported on varargs functions"); 4776 4777 // Count how many bytes are to be pushed on the stack, including the linkage 4778 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes 4779 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage 4780 // area is 32 bytes reserved space for [SP][CR][LR][TOC]. 4781 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4782 unsigned NumBytes = LinkageSize; 4783 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 4784 unsigned &QFPR_idx = FPR_idx; 4785 4786 static const MCPhysReg GPR[] = { 4787 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4788 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4789 }; 4790 static const MCPhysReg VR[] = { 4791 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4792 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4793 }; 4794 static const MCPhysReg VSRH[] = { 4795 PPC::VSH2, PPC::VSH3, PPC::VSH4, PPC::VSH5, PPC::VSH6, PPC::VSH7, PPC::VSH8, 4796 PPC::VSH9, PPC::VSH10, PPC::VSH11, PPC::VSH12, PPC::VSH13 4797 }; 4798 4799 const unsigned NumGPRs = array_lengthof(GPR); 4800 const unsigned NumFPRs = 13; 4801 const unsigned NumVRs = array_lengthof(VR); 4802 const unsigned NumQFPRs = NumFPRs; 4803 4804 // When using the fast calling convention, we don't provide backing for 4805 // arguments that will be in registers. 4806 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0; 4807 4808 // Add up all the space actually used. 4809 for (unsigned i = 0; i != NumOps; ++i) { 4810 ISD::ArgFlagsTy Flags = Outs[i].Flags; 4811 EVT ArgVT = Outs[i].VT; 4812 EVT OrigVT = Outs[i].ArgVT; 4813 4814 if (Flags.isNest()) 4815 continue; 4816 4817 if (CallConv == CallingConv::Fast) { 4818 if (Flags.isByVal()) 4819 NumGPRsUsed += (Flags.getByValSize()+7)/8; 4820 else 4821 switch (ArgVT.getSimpleVT().SimpleTy) { 4822 default: llvm_unreachable("Unexpected ValueType for argument!"); 4823 case MVT::i1: 4824 case MVT::i32: 4825 case MVT::i64: 4826 if (++NumGPRsUsed <= NumGPRs) 4827 continue; 4828 break; 4829 case MVT::v4i32: 4830 case MVT::v8i16: 4831 case MVT::v16i8: 4832 case MVT::v2f64: 4833 case MVT::v2i64: 4834 case MVT::v1i128: 4835 if (++NumVRsUsed <= NumVRs) 4836 continue; 4837 break; 4838 case MVT::v4f32: 4839 // When using QPX, this is handled like a FP register, otherwise, it 4840 // is an Altivec register. 4841 if (Subtarget.hasQPX()) { 4842 if (++NumFPRsUsed <= NumFPRs) 4843 continue; 4844 } else { 4845 if (++NumVRsUsed <= NumVRs) 4846 continue; 4847 } 4848 break; 4849 case MVT::f32: 4850 case MVT::f64: 4851 case MVT::v4f64: // QPX 4852 case MVT::v4i1: // QPX 4853 if (++NumFPRsUsed <= NumFPRs) 4854 continue; 4855 break; 4856 } 4857 } 4858 4859 /* Respect alignment of argument on the stack. */ 4860 unsigned Align = 4861 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 4862 NumBytes = ((NumBytes + Align - 1) / Align) * Align; 4863 4864 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 4865 if (Flags.isInConsecutiveRegsLast()) 4866 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4867 } 4868 4869 unsigned NumBytesActuallyUsed = NumBytes; 4870 4871 // The prolog code of the callee may store up to 8 GPR argument registers to 4872 // the stack, allowing va_start to index over them in memory if its varargs. 4873 // Because we cannot tell if this is needed on the caller side, we have to 4874 // conservatively assume that it is needed. As such, make sure we have at 4875 // least enough stack space for the caller to store the 8 GPRs. 4876 // FIXME: On ELFv2, it may be unnecessary to allocate the parameter area. 4877 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 4878 4879 // Tail call needs the stack to be aligned. 4880 if (getTargetMachine().Options.GuaranteedTailCallOpt && 4881 CallConv == CallingConv::Fast) 4882 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 4883 4884 // Calculate by how many bytes the stack has to be adjusted in case of tail 4885 // call optimization. 4886 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 4887 4888 // To protect arguments on the stack from being clobbered in a tail call, 4889 // force all the loads to happen before doing any other lowering. 4890 if (isTailCall) 4891 Chain = DAG.getStackArgumentTokenFactor(Chain); 4892 4893 // Adjust the stack pointer for the new arguments... 4894 // These operations are automatically eliminated by the prolog/epilog pass 4895 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 4896 dl); 4897 SDValue CallSeqStart = Chain; 4898 4899 // Load the return address and frame pointer so it can be move somewhere else 4900 // later. 4901 SDValue LROp, FPOp; 4902 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true, 4903 dl); 4904 4905 // Set up a copy of the stack pointer for use loading and storing any 4906 // arguments that may not fit in the registers available for argument 4907 // passing. 4908 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 4909 4910 // Figure out which arguments are going to go in registers, and which in 4911 // memory. Also, if this is a vararg function, floating point operations 4912 // must be stored to our stack, and loaded into integer regs as well, if 4913 // any integer regs are available for argument passing. 4914 unsigned ArgOffset = LinkageSize; 4915 4916 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 4917 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 4918 4919 SmallVector<SDValue, 8> MemOpChains; 4920 for (unsigned i = 0; i != NumOps; ++i) { 4921 SDValue Arg = OutVals[i]; 4922 ISD::ArgFlagsTy Flags = Outs[i].Flags; 4923 EVT ArgVT = Outs[i].VT; 4924 EVT OrigVT = Outs[i].ArgVT; 4925 4926 // PtrOff will be used to store the current argument to the stack if a 4927 // register cannot be found for it. 4928 SDValue PtrOff; 4929 4930 // We re-align the argument offset for each argument, except when using the 4931 // fast calling convention, when we need to make sure we do that only when 4932 // we'll actually use a stack slot. 4933 auto ComputePtrOff = [&]() { 4934 /* Respect alignment of argument on the stack. */ 4935 unsigned Align = 4936 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 4937 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 4938 4939 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 4940 4941 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 4942 }; 4943 4944 if (CallConv != CallingConv::Fast) { 4945 ComputePtrOff(); 4946 4947 /* Compute GPR index associated with argument offset. */ 4948 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 4949 GPR_idx = std::min(GPR_idx, NumGPRs); 4950 } 4951 4952 // Promote integers to 64-bit values. 4953 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { 4954 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 4955 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4956 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 4957 } 4958 4959 // FIXME memcpy is used way more than necessary. Correctness first. 4960 // Note: "by value" is code for passing a structure by value, not 4961 // basic types. 4962 if (Flags.isByVal()) { 4963 // Note: Size includes alignment padding, so 4964 // struct x { short a; char b; } 4965 // will have Size = 4. With #pragma pack(1), it will have Size = 3. 4966 // These are the proper values we need for right-justifying the 4967 // aggregate in a parameter register. 4968 unsigned Size = Flags.getByValSize(); 4969 4970 // An empty aggregate parameter takes up no storage and no 4971 // registers. 4972 if (Size == 0) 4973 continue; 4974 4975 if (CallConv == CallingConv::Fast) 4976 ComputePtrOff(); 4977 4978 // All aggregates smaller than 8 bytes must be passed right-justified. 4979 if (Size==1 || Size==2 || Size==4) { 4980 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); 4981 if (GPR_idx != NumGPRs) { 4982 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 4983 MachinePointerInfo(), VT, 4984 false, false, false, 0); 4985 MemOpChains.push_back(Load.getValue(1)); 4986 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4987 4988 ArgOffset += PtrByteSize; 4989 continue; 4990 } 4991 } 4992 4993 if (GPR_idx == NumGPRs && Size < 8) { 4994 SDValue AddPtr = PtrOff; 4995 if (!isLittleEndian) { 4996 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 4997 PtrOff.getValueType()); 4998 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 4999 } 5000 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 5001 CallSeqStart, 5002 Flags, DAG, dl); 5003 ArgOffset += PtrByteSize; 5004 continue; 5005 } 5006 // Copy entire object into memory. There are cases where gcc-generated 5007 // code assumes it is there, even if it could be put entirely into 5008 // registers. (This is not what the doc says.) 5009 5010 // FIXME: The above statement is likely due to a misunderstanding of the 5011 // documents. All arguments must be copied into the parameter area BY 5012 // THE CALLEE in the event that the callee takes the address of any 5013 // formal argument. That has not yet been implemented. However, it is 5014 // reasonable to use the stack area as a staging area for the register 5015 // load. 5016 5017 // Skip this for small aggregates, as we will use the same slot for a 5018 // right-justified copy, below. 5019 if (Size >= 8) 5020 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 5021 CallSeqStart, 5022 Flags, DAG, dl); 5023 5024 // When a register is available, pass a small aggregate right-justified. 5025 if (Size < 8 && GPR_idx != NumGPRs) { 5026 // The easiest way to get this right-justified in a register 5027 // is to copy the structure into the rightmost portion of a 5028 // local variable slot, then load the whole slot into the 5029 // register. 5030 // FIXME: The memcpy seems to produce pretty awful code for 5031 // small aggregates, particularly for packed ones. 5032 // FIXME: It would be preferable to use the slot in the 5033 // parameter save area instead of a new local variable. 5034 SDValue AddPtr = PtrOff; 5035 if (!isLittleEndian) { 5036 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType()); 5037 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 5038 } 5039 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 5040 CallSeqStart, 5041 Flags, DAG, dl); 5042 5043 // Load the slot into the register. 5044 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, PtrOff, 5045 MachinePointerInfo(), 5046 false, false, false, 0); 5047 MemOpChains.push_back(Load.getValue(1)); 5048 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5049 5050 // Done with this argument. 5051 ArgOffset += PtrByteSize; 5052 continue; 5053 } 5054 5055 // For aggregates larger than PtrByteSize, copy the pieces of the 5056 // object that fit into registers from the parameter save area. 5057 for (unsigned j=0; j<Size; j+=PtrByteSize) { 5058 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 5059 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 5060 if (GPR_idx != NumGPRs) { 5061 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg, 5062 MachinePointerInfo(), 5063 false, false, false, 0); 5064 MemOpChains.push_back(Load.getValue(1)); 5065 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5066 ArgOffset += PtrByteSize; 5067 } else { 5068 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 5069 break; 5070 } 5071 } 5072 continue; 5073 } 5074 5075 switch (Arg.getSimpleValueType().SimpleTy) { 5076 default: llvm_unreachable("Unexpected ValueType for argument!"); 5077 case MVT::i1: 5078 case MVT::i32: 5079 case MVT::i64: 5080 if (Flags.isNest()) { 5081 // The 'nest' parameter, if any, is passed in R11. 5082 RegsToPass.push_back(std::make_pair(PPC::X11, Arg)); 5083 hasNest = true; 5084 break; 5085 } 5086 5087 // These can be scalar arguments or elements of an integer array type 5088 // passed directly. Clang may use those instead of "byval" aggregate 5089 // types to avoid forcing arguments to memory unnecessarily. 5090 if (GPR_idx != NumGPRs) { 5091 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 5092 } else { 5093 if (CallConv == CallingConv::Fast) 5094 ComputePtrOff(); 5095 5096 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5097 true, isTailCall, false, MemOpChains, 5098 TailCallArguments, dl); 5099 if (CallConv == CallingConv::Fast) 5100 ArgOffset += PtrByteSize; 5101 } 5102 if (CallConv != CallingConv::Fast) 5103 ArgOffset += PtrByteSize; 5104 break; 5105 case MVT::f32: 5106 case MVT::f64: { 5107 // These can be scalar arguments or elements of a float array type 5108 // passed directly. The latter are used to implement ELFv2 homogenous 5109 // float aggregates. 5110 5111 // Named arguments go into FPRs first, and once they overflow, the 5112 // remaining arguments go into GPRs and then the parameter save area. 5113 // Unnamed arguments for vararg functions always go to GPRs and 5114 // then the parameter save area. For now, put all arguments to vararg 5115 // routines always in both locations (FPR *and* GPR or stack slot). 5116 bool NeedGPROrStack = isVarArg || FPR_idx == NumFPRs; 5117 bool NeededLoad = false; 5118 5119 // First load the argument into the next available FPR. 5120 if (FPR_idx != NumFPRs) 5121 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 5122 5123 // Next, load the argument into GPR or stack slot if needed. 5124 if (!NeedGPROrStack) 5125 ; 5126 else if (GPR_idx != NumGPRs && CallConv != CallingConv::Fast) { 5127 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 5128 // once we support fp <-> gpr moves. 5129 5130 // In the non-vararg case, this can only ever happen in the 5131 // presence of f32 array types, since otherwise we never run 5132 // out of FPRs before running out of GPRs. 5133 SDValue ArgVal; 5134 5135 // Double values are always passed in a single GPR. 5136 if (Arg.getValueType() != MVT::f32) { 5137 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); 5138 5139 // Non-array float values are extended and passed in a GPR. 5140 } else if (!Flags.isInConsecutiveRegs()) { 5141 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 5142 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 5143 5144 // If we have an array of floats, we collect every odd element 5145 // together with its predecessor into one GPR. 5146 } else if (ArgOffset % PtrByteSize != 0) { 5147 SDValue Lo, Hi; 5148 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]); 5149 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 5150 if (!isLittleEndian) 5151 std::swap(Lo, Hi); 5152 ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 5153 5154 // The final element, if even, goes into the first half of a GPR. 5155 } else if (Flags.isInConsecutiveRegsLast()) { 5156 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 5157 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 5158 if (!isLittleEndian) 5159 ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal, 5160 DAG.getConstant(32, dl, MVT::i32)); 5161 5162 // Non-final even elements are skipped; they will be handled 5163 // together the with subsequent argument on the next go-around. 5164 } else 5165 ArgVal = SDValue(); 5166 5167 if (ArgVal.getNode()) 5168 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal)); 5169 } else { 5170 if (CallConv == CallingConv::Fast) 5171 ComputePtrOff(); 5172 5173 // Single-precision floating-point values are mapped to the 5174 // second (rightmost) word of the stack doubleword. 5175 if (Arg.getValueType() == MVT::f32 && 5176 !isLittleEndian && !Flags.isInConsecutiveRegs()) { 5177 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 5178 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 5179 } 5180 5181 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5182 true, isTailCall, false, MemOpChains, 5183 TailCallArguments, dl); 5184 5185 NeededLoad = true; 5186 } 5187 // When passing an array of floats, the array occupies consecutive 5188 // space in the argument area; only round up to the next doubleword 5189 // at the end of the array. Otherwise, each float takes 8 bytes. 5190 if (CallConv != CallingConv::Fast || NeededLoad) { 5191 ArgOffset += (Arg.getValueType() == MVT::f32 && 5192 Flags.isInConsecutiveRegs()) ? 4 : 8; 5193 if (Flags.isInConsecutiveRegsLast()) 5194 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 5195 } 5196 break; 5197 } 5198 case MVT::v4f32: 5199 case MVT::v4i32: 5200 case MVT::v8i16: 5201 case MVT::v16i8: 5202 case MVT::v2f64: 5203 case MVT::v2i64: 5204 case MVT::v1i128: 5205 if (!Subtarget.hasQPX()) { 5206 // These can be scalar arguments or elements of a vector array type 5207 // passed directly. The latter are used to implement ELFv2 homogenous 5208 // vector aggregates. 5209 5210 // For a varargs call, named arguments go into VRs or on the stack as 5211 // usual; unnamed arguments always go to the stack or the corresponding 5212 // GPRs when within range. For now, we always put the value in both 5213 // locations (or even all three). 5214 if (isVarArg) { 5215 // We could elide this store in the case where the object fits 5216 // entirely in R registers. Maybe later. 5217 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 5218 MachinePointerInfo(), false, false, 0); 5219 MemOpChains.push_back(Store); 5220 if (VR_idx != NumVRs) { 5221 SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, 5222 MachinePointerInfo(), 5223 false, false, false, 0); 5224 MemOpChains.push_back(Load.getValue(1)); 5225 5226 unsigned VReg = (Arg.getSimpleValueType() == MVT::v2f64 || 5227 Arg.getSimpleValueType() == MVT::v2i64) ? 5228 VSRH[VR_idx] : VR[VR_idx]; 5229 ++VR_idx; 5230 5231 RegsToPass.push_back(std::make_pair(VReg, Load)); 5232 } 5233 ArgOffset += 16; 5234 for (unsigned i=0; i<16; i+=PtrByteSize) { 5235 if (GPR_idx == NumGPRs) 5236 break; 5237 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 5238 DAG.getConstant(i, dl, PtrVT)); 5239 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(), 5240 false, false, false, 0); 5241 MemOpChains.push_back(Load.getValue(1)); 5242 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5243 } 5244 break; 5245 } 5246 5247 // Non-varargs Altivec params go into VRs or on the stack. 5248 if (VR_idx != NumVRs) { 5249 unsigned VReg = (Arg.getSimpleValueType() == MVT::v2f64 || 5250 Arg.getSimpleValueType() == MVT::v2i64) ? 5251 VSRH[VR_idx] : VR[VR_idx]; 5252 ++VR_idx; 5253 5254 RegsToPass.push_back(std::make_pair(VReg, Arg)); 5255 } else { 5256 if (CallConv == CallingConv::Fast) 5257 ComputePtrOff(); 5258 5259 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5260 true, isTailCall, true, MemOpChains, 5261 TailCallArguments, dl); 5262 if (CallConv == CallingConv::Fast) 5263 ArgOffset += 16; 5264 } 5265 5266 if (CallConv != CallingConv::Fast) 5267 ArgOffset += 16; 5268 break; 5269 } // not QPX 5270 5271 assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 && 5272 "Invalid QPX parameter type"); 5273 5274 /* fall through */ 5275 case MVT::v4f64: 5276 case MVT::v4i1: { 5277 bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32; 5278 if (isVarArg) { 5279 // We could elide this store in the case where the object fits 5280 // entirely in R registers. Maybe later. 5281 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 5282 MachinePointerInfo(), false, false, 0); 5283 MemOpChains.push_back(Store); 5284 if (QFPR_idx != NumQFPRs) { 5285 SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, 5286 Store, PtrOff, MachinePointerInfo(), 5287 false, false, false, 0); 5288 MemOpChains.push_back(Load.getValue(1)); 5289 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load)); 5290 } 5291 ArgOffset += (IsF32 ? 16 : 32); 5292 for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) { 5293 if (GPR_idx == NumGPRs) 5294 break; 5295 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 5296 DAG.getConstant(i, dl, PtrVT)); 5297 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(), 5298 false, false, false, 0); 5299 MemOpChains.push_back(Load.getValue(1)); 5300 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5301 } 5302 break; 5303 } 5304 5305 // Non-varargs QPX params go into registers or on the stack. 5306 if (QFPR_idx != NumQFPRs) { 5307 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg)); 5308 } else { 5309 if (CallConv == CallingConv::Fast) 5310 ComputePtrOff(); 5311 5312 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5313 true, isTailCall, true, MemOpChains, 5314 TailCallArguments, dl); 5315 if (CallConv == CallingConv::Fast) 5316 ArgOffset += (IsF32 ? 16 : 32); 5317 } 5318 5319 if (CallConv != CallingConv::Fast) 5320 ArgOffset += (IsF32 ? 16 : 32); 5321 break; 5322 } 5323 } 5324 } 5325 5326 assert(NumBytesActuallyUsed == ArgOffset); 5327 (void)NumBytesActuallyUsed; 5328 5329 if (!MemOpChains.empty()) 5330 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 5331 5332 // Check if this is an indirect call (MTCTR/BCTRL). 5333 // See PrepareCall() for more information about calls through function 5334 // pointers in the 64-bit SVR4 ABI. 5335 if (!isTailCall && !IsPatchPoint && 5336 !isFunctionGlobalAddress(Callee) && 5337 !isa<ExternalSymbolSDNode>(Callee)) { 5338 // Load r2 into a virtual register and store it to the TOC save area. 5339 setUsesTOCBasePtr(DAG); 5340 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); 5341 // TOC save area offset. 5342 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 5343 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 5344 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 5345 Chain = DAG.getStore( 5346 Val.getValue(1), dl, Val, AddPtr, 5347 MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset), 5348 false, false, 0); 5349 // In the ELFv2 ABI, R12 must contain the address of an indirect callee. 5350 // This does not mean the MTCTR instruction must use R12; it's easier 5351 // to model this as an extra parameter, so do that. 5352 if (isELFv2ABI && !IsPatchPoint) 5353 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); 5354 } 5355 5356 // Build a sequence of copy-to-reg nodes chained together with token chain 5357 // and flag operands which copy the outgoing args into the appropriate regs. 5358 SDValue InFlag; 5359 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 5360 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 5361 RegsToPass[i].second, InFlag); 5362 InFlag = Chain.getValue(1); 5363 } 5364 5365 if (isTailCall) 5366 PrepareTailCall(DAG, InFlag, Chain, dl, true, SPDiff, NumBytes, LROp, 5367 FPOp, true, TailCallArguments); 5368 5369 return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, hasNest, 5370 DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, 5371 SPDiff, NumBytes, Ins, InVals, CS); 5372 } 5373 5374 SDValue 5375 PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee, 5376 CallingConv::ID CallConv, bool isVarArg, 5377 bool isTailCall, bool IsPatchPoint, 5378 const SmallVectorImpl<ISD::OutputArg> &Outs, 5379 const SmallVectorImpl<SDValue> &OutVals, 5380 const SmallVectorImpl<ISD::InputArg> &Ins, 5381 SDLoc dl, SelectionDAG &DAG, 5382 SmallVectorImpl<SDValue> &InVals, 5383 ImmutableCallSite *CS) const { 5384 5385 unsigned NumOps = Outs.size(); 5386 5387 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 5388 bool isPPC64 = PtrVT == MVT::i64; 5389 unsigned PtrByteSize = isPPC64 ? 8 : 4; 5390 5391 MachineFunction &MF = DAG.getMachineFunction(); 5392 5393 // Mark this function as potentially containing a function that contains a 5394 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5395 // and restoring the callers stack pointer in this functions epilog. This is 5396 // done because by tail calling the called function might overwrite the value 5397 // in this function's (MF) stack pointer stack slot 0(SP). 5398 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5399 CallConv == CallingConv::Fast) 5400 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5401 5402 // Count how many bytes are to be pushed on the stack, including the linkage 5403 // area, and parameter passing area. We start with 24/48 bytes, which is 5404 // prereserved space for [SP][CR][LR][3 x unused]. 5405 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 5406 unsigned NumBytes = LinkageSize; 5407 5408 // Add up all the space actually used. 5409 // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually 5410 // they all go in registers, but we must reserve stack space for them for 5411 // possible use by the caller. In varargs or 64-bit calls, parameters are 5412 // assigned stack space in order, with padding so Altivec parameters are 5413 // 16-byte aligned. 5414 unsigned nAltivecParamsAtEnd = 0; 5415 for (unsigned i = 0; i != NumOps; ++i) { 5416 ISD::ArgFlagsTy Flags = Outs[i].Flags; 5417 EVT ArgVT = Outs[i].VT; 5418 // Varargs Altivec parameters are padded to a 16 byte boundary. 5419 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 5420 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 5421 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) { 5422 if (!isVarArg && !isPPC64) { 5423 // Non-varargs Altivec parameters go after all the non-Altivec 5424 // parameters; handle those later so we know how much padding we need. 5425 nAltivecParamsAtEnd++; 5426 continue; 5427 } 5428 // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. 5429 NumBytes = ((NumBytes+15)/16)*16; 5430 } 5431 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 5432 } 5433 5434 // Allow for Altivec parameters at the end, if needed. 5435 if (nAltivecParamsAtEnd) { 5436 NumBytes = ((NumBytes+15)/16)*16; 5437 NumBytes += 16*nAltivecParamsAtEnd; 5438 } 5439 5440 // The prolog code of the callee may store up to 8 GPR argument registers to 5441 // the stack, allowing va_start to index over them in memory if its varargs. 5442 // Because we cannot tell if this is needed on the caller side, we have to 5443 // conservatively assume that it is needed. As such, make sure we have at 5444 // least enough stack space for the caller to store the 8 GPRs. 5445 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 5446 5447 // Tail call needs the stack to be aligned. 5448 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5449 CallConv == CallingConv::Fast) 5450 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 5451 5452 // Calculate by how many bytes the stack has to be adjusted in case of tail 5453 // call optimization. 5454 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 5455 5456 // To protect arguments on the stack from being clobbered in a tail call, 5457 // force all the loads to happen before doing any other lowering. 5458 if (isTailCall) 5459 Chain = DAG.getStackArgumentTokenFactor(Chain); 5460 5461 // Adjust the stack pointer for the new arguments... 5462 // These operations are automatically eliminated by the prolog/epilog pass 5463 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 5464 dl); 5465 SDValue CallSeqStart = Chain; 5466 5467 // Load the return address and frame pointer so it can be move somewhere else 5468 // later. 5469 SDValue LROp, FPOp; 5470 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true, 5471 dl); 5472 5473 // Set up a copy of the stack pointer for use loading and storing any 5474 // arguments that may not fit in the registers available for argument 5475 // passing. 5476 SDValue StackPtr; 5477 if (isPPC64) 5478 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 5479 else 5480 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 5481 5482 // Figure out which arguments are going to go in registers, and which in 5483 // memory. Also, if this is a vararg function, floating point operations 5484 // must be stored to our stack, and loaded into integer regs as well, if 5485 // any integer regs are available for argument passing. 5486 unsigned ArgOffset = LinkageSize; 5487 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 5488 5489 static const MCPhysReg GPR_32[] = { // 32-bit registers. 5490 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 5491 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 5492 }; 5493 static const MCPhysReg GPR_64[] = { // 64-bit registers. 5494 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 5495 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 5496 }; 5497 static const MCPhysReg VR[] = { 5498 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 5499 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 5500 }; 5501 const unsigned NumGPRs = array_lengthof(GPR_32); 5502 const unsigned NumFPRs = 13; 5503 const unsigned NumVRs = array_lengthof(VR); 5504 5505 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 5506 5507 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 5508 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 5509 5510 SmallVector<SDValue, 8> MemOpChains; 5511 for (unsigned i = 0; i != NumOps; ++i) { 5512 SDValue Arg = OutVals[i]; 5513 ISD::ArgFlagsTy Flags = Outs[i].Flags; 5514 5515 // PtrOff will be used to store the current argument to the stack if a 5516 // register cannot be found for it. 5517 SDValue PtrOff; 5518 5519 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 5520 5521 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 5522 5523 // On PPC64, promote integers to 64-bit values. 5524 if (isPPC64 && Arg.getValueType() == MVT::i32) { 5525 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 5526 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 5527 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 5528 } 5529 5530 // FIXME memcpy is used way more than necessary. Correctness first. 5531 // Note: "by value" is code for passing a structure by value, not 5532 // basic types. 5533 if (Flags.isByVal()) { 5534 unsigned Size = Flags.getByValSize(); 5535 // Very small objects are passed right-justified. Everything else is 5536 // passed left-justified. 5537 if (Size==1 || Size==2) { 5538 EVT VT = (Size==1) ? MVT::i8 : MVT::i16; 5539 if (GPR_idx != NumGPRs) { 5540 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 5541 MachinePointerInfo(), VT, 5542 false, false, false, 0); 5543 MemOpChains.push_back(Load.getValue(1)); 5544 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5545 5546 ArgOffset += PtrByteSize; 5547 } else { 5548 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 5549 PtrOff.getValueType()); 5550 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 5551 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 5552 CallSeqStart, 5553 Flags, DAG, dl); 5554 ArgOffset += PtrByteSize; 5555 } 5556 continue; 5557 } 5558 // Copy entire object into memory. There are cases where gcc-generated 5559 // code assumes it is there, even if it could be put entirely into 5560 // registers. (This is not what the doc says.) 5561 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 5562 CallSeqStart, 5563 Flags, DAG, dl); 5564 5565 // For small aggregates (Darwin only) and aggregates >= PtrByteSize, 5566 // copy the pieces of the object that fit into registers from the 5567 // parameter save area. 5568 for (unsigned j=0; j<Size; j+=PtrByteSize) { 5569 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 5570 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 5571 if (GPR_idx != NumGPRs) { 5572 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg, 5573 MachinePointerInfo(), 5574 false, false, false, 0); 5575 MemOpChains.push_back(Load.getValue(1)); 5576 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5577 ArgOffset += PtrByteSize; 5578 } else { 5579 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 5580 break; 5581 } 5582 } 5583 continue; 5584 } 5585 5586 switch (Arg.getSimpleValueType().SimpleTy) { 5587 default: llvm_unreachable("Unexpected ValueType for argument!"); 5588 case MVT::i1: 5589 case MVT::i32: 5590 case MVT::i64: 5591 if (GPR_idx != NumGPRs) { 5592 if (Arg.getValueType() == MVT::i1) 5593 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, PtrVT, Arg); 5594 5595 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 5596 } else { 5597 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5598 isPPC64, isTailCall, false, MemOpChains, 5599 TailCallArguments, dl); 5600 } 5601 ArgOffset += PtrByteSize; 5602 break; 5603 case MVT::f32: 5604 case MVT::f64: 5605 if (FPR_idx != NumFPRs) { 5606 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 5607 5608 if (isVarArg) { 5609 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 5610 MachinePointerInfo(), false, false, 0); 5611 MemOpChains.push_back(Store); 5612 5613 // Float varargs are always shadowed in available integer registers 5614 if (GPR_idx != NumGPRs) { 5615 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff, 5616 MachinePointerInfo(), false, false, 5617 false, 0); 5618 MemOpChains.push_back(Load.getValue(1)); 5619 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5620 } 5621 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){ 5622 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 5623 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 5624 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff, 5625 MachinePointerInfo(), 5626 false, false, false, 0); 5627 MemOpChains.push_back(Load.getValue(1)); 5628 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5629 } 5630 } else { 5631 // If we have any FPRs remaining, we may also have GPRs remaining. 5632 // Args passed in FPRs consume either 1 (f32) or 2 (f64) available 5633 // GPRs. 5634 if (GPR_idx != NumGPRs) 5635 ++GPR_idx; 5636 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && 5637 !isPPC64) // PPC64 has 64-bit GPR's obviously :) 5638 ++GPR_idx; 5639 } 5640 } else 5641 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5642 isPPC64, isTailCall, false, MemOpChains, 5643 TailCallArguments, dl); 5644 if (isPPC64) 5645 ArgOffset += 8; 5646 else 5647 ArgOffset += Arg.getValueType() == MVT::f32 ? 4 : 8; 5648 break; 5649 case MVT::v4f32: 5650 case MVT::v4i32: 5651 case MVT::v8i16: 5652 case MVT::v16i8: 5653 if (isVarArg) { 5654 // These go aligned on the stack, or in the corresponding R registers 5655 // when within range. The Darwin PPC ABI doc claims they also go in 5656 // V registers; in fact gcc does this only for arguments that are 5657 // prototyped, not for those that match the ... We do it for all 5658 // arguments, seems to work. 5659 while (ArgOffset % 16 !=0) { 5660 ArgOffset += PtrByteSize; 5661 if (GPR_idx != NumGPRs) 5662 GPR_idx++; 5663 } 5664 // We could elide this store in the case where the object fits 5665 // entirely in R registers. Maybe later. 5666 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 5667 DAG.getConstant(ArgOffset, dl, PtrVT)); 5668 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 5669 MachinePointerInfo(), false, false, 0); 5670 MemOpChains.push_back(Store); 5671 if (VR_idx != NumVRs) { 5672 SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, 5673 MachinePointerInfo(), 5674 false, false, false, 0); 5675 MemOpChains.push_back(Load.getValue(1)); 5676 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 5677 } 5678 ArgOffset += 16; 5679 for (unsigned i=0; i<16; i+=PtrByteSize) { 5680 if (GPR_idx == NumGPRs) 5681 break; 5682 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 5683 DAG.getConstant(i, dl, PtrVT)); 5684 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(), 5685 false, false, false, 0); 5686 MemOpChains.push_back(Load.getValue(1)); 5687 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5688 } 5689 break; 5690 } 5691 5692 // Non-varargs Altivec params generally go in registers, but have 5693 // stack space allocated at the end. 5694 if (VR_idx != NumVRs) { 5695 // Doesn't have GPR space allocated. 5696 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 5697 } else if (nAltivecParamsAtEnd==0) { 5698 // We are emitting Altivec params in order. 5699 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5700 isPPC64, isTailCall, true, MemOpChains, 5701 TailCallArguments, dl); 5702 ArgOffset += 16; 5703 } 5704 break; 5705 } 5706 } 5707 // If all Altivec parameters fit in registers, as they usually do, 5708 // they get stack space following the non-Altivec parameters. We 5709 // don't track this here because nobody below needs it. 5710 // If there are more Altivec parameters than fit in registers emit 5711 // the stores here. 5712 if (!isVarArg && nAltivecParamsAtEnd > NumVRs) { 5713 unsigned j = 0; 5714 // Offset is aligned; skip 1st 12 params which go in V registers. 5715 ArgOffset = ((ArgOffset+15)/16)*16; 5716 ArgOffset += 12*16; 5717 for (unsigned i = 0; i != NumOps; ++i) { 5718 SDValue Arg = OutVals[i]; 5719 EVT ArgType = Outs[i].VT; 5720 if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || 5721 ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { 5722 if (++j > NumVRs) { 5723 SDValue PtrOff; 5724 // We are emitting Altivec params in order. 5725 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 5726 isPPC64, isTailCall, true, MemOpChains, 5727 TailCallArguments, dl); 5728 ArgOffset += 16; 5729 } 5730 } 5731 } 5732 } 5733 5734 if (!MemOpChains.empty()) 5735 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 5736 5737 // On Darwin, R12 must contain the address of an indirect callee. This does 5738 // not mean the MTCTR instruction must use R12; it's easier to model this as 5739 // an extra parameter, so do that. 5740 if (!isTailCall && 5741 !isFunctionGlobalAddress(Callee) && 5742 !isa<ExternalSymbolSDNode>(Callee) && 5743 !isBLACompatibleAddress(Callee, DAG)) 5744 RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : 5745 PPC::R12), Callee)); 5746 5747 // Build a sequence of copy-to-reg nodes chained together with token chain 5748 // and flag operands which copy the outgoing args into the appropriate regs. 5749 SDValue InFlag; 5750 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 5751 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 5752 RegsToPass[i].second, InFlag); 5753 InFlag = Chain.getValue(1); 5754 } 5755 5756 if (isTailCall) 5757 PrepareTailCall(DAG, InFlag, Chain, dl, isPPC64, SPDiff, NumBytes, LROp, 5758 FPOp, true, TailCallArguments); 5759 5760 return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, 5761 /* unused except on PPC64 ELFv1 */ false, DAG, 5762 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 5763 NumBytes, Ins, InVals, CS); 5764 } 5765 5766 bool 5767 PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 5768 MachineFunction &MF, bool isVarArg, 5769 const SmallVectorImpl<ISD::OutputArg> &Outs, 5770 LLVMContext &Context) const { 5771 SmallVector<CCValAssign, 16> RVLocs; 5772 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 5773 return CCInfo.CheckReturn(Outs, RetCC_PPC); 5774 } 5775 5776 SDValue 5777 PPCTargetLowering::LowerReturn(SDValue Chain, 5778 CallingConv::ID CallConv, bool isVarArg, 5779 const SmallVectorImpl<ISD::OutputArg> &Outs, 5780 const SmallVectorImpl<SDValue> &OutVals, 5781 SDLoc dl, SelectionDAG &DAG) const { 5782 5783 SmallVector<CCValAssign, 16> RVLocs; 5784 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 5785 *DAG.getContext()); 5786 CCInfo.AnalyzeReturn(Outs, RetCC_PPC); 5787 5788 SDValue Flag; 5789 SmallVector<SDValue, 4> RetOps(1, Chain); 5790 5791 // Copy the result values into the output registers. 5792 for (unsigned i = 0; i != RVLocs.size(); ++i) { 5793 CCValAssign &VA = RVLocs[i]; 5794 assert(VA.isRegLoc() && "Can only return in registers!"); 5795 5796 SDValue Arg = OutVals[i]; 5797 5798 switch (VA.getLocInfo()) { 5799 default: llvm_unreachable("Unknown loc info!"); 5800 case CCValAssign::Full: break; 5801 case CCValAssign::AExt: 5802 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 5803 break; 5804 case CCValAssign::ZExt: 5805 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 5806 break; 5807 case CCValAssign::SExt: 5808 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 5809 break; 5810 } 5811 5812 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 5813 Flag = Chain.getValue(1); 5814 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 5815 } 5816 5817 RetOps[0] = Chain; // Update chain. 5818 5819 // Add the flag if we have it. 5820 if (Flag.getNode()) 5821 RetOps.push_back(Flag); 5822 5823 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); 5824 } 5825 5826 SDValue PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET( 5827 SDValue Op, SelectionDAG &DAG, const PPCSubtarget &Subtarget) const { 5828 SDLoc dl(Op); 5829 5830 // Get the corect type for integers. 5831 EVT IntVT = Op.getValueType(); 5832 5833 // Get the inputs. 5834 SDValue Chain = Op.getOperand(0); 5835 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 5836 // Build a DYNAREAOFFSET node. 5837 SDValue Ops[2] = {Chain, FPSIdx}; 5838 SDVTList VTs = DAG.getVTList(IntVT); 5839 return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops); 5840 } 5841 5842 SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG, 5843 const PPCSubtarget &Subtarget) const { 5844 // When we pop the dynamic allocation we need to restore the SP link. 5845 SDLoc dl(Op); 5846 5847 // Get the corect type for pointers. 5848 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 5849 5850 // Construct the stack pointer operand. 5851 bool isPPC64 = Subtarget.isPPC64(); 5852 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; 5853 SDValue StackPtr = DAG.getRegister(SP, PtrVT); 5854 5855 // Get the operands for the STACKRESTORE. 5856 SDValue Chain = Op.getOperand(0); 5857 SDValue SaveSP = Op.getOperand(1); 5858 5859 // Load the old link SP. 5860 SDValue LoadLinkSP = DAG.getLoad(PtrVT, dl, Chain, StackPtr, 5861 MachinePointerInfo(), 5862 false, false, false, 0); 5863 5864 // Restore the stack pointer. 5865 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); 5866 5867 // Store the old link SP. 5868 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo(), 5869 false, false, 0); 5870 } 5871 5872 SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const { 5873 MachineFunction &MF = DAG.getMachineFunction(); 5874 bool isPPC64 = Subtarget.isPPC64(); 5875 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 5876 5877 // Get current frame pointer save index. The users of this index will be 5878 // primarily DYNALLOC instructions. 5879 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 5880 int RASI = FI->getReturnAddrSaveIndex(); 5881 5882 // If the frame pointer save index hasn't been defined yet. 5883 if (!RASI) { 5884 // Find out what the fix offset of the frame pointer save area. 5885 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset(); 5886 // Allocate the frame index for frame pointer save area. 5887 RASI = MF.getFrameInfo()->CreateFixedObject(isPPC64? 8 : 4, LROffset, false); 5888 // Save the result. 5889 FI->setReturnAddrSaveIndex(RASI); 5890 } 5891 return DAG.getFrameIndex(RASI, PtrVT); 5892 } 5893 5894 SDValue 5895 PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { 5896 MachineFunction &MF = DAG.getMachineFunction(); 5897 bool isPPC64 = Subtarget.isPPC64(); 5898 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 5899 5900 // Get current frame pointer save index. The users of this index will be 5901 // primarily DYNALLOC instructions. 5902 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 5903 int FPSI = FI->getFramePointerSaveIndex(); 5904 5905 // If the frame pointer save index hasn't been defined yet. 5906 if (!FPSI) { 5907 // Find out what the fix offset of the frame pointer save area. 5908 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset(); 5909 // Allocate the frame index for frame pointer save area. 5910 FPSI = MF.getFrameInfo()->CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); 5911 // Save the result. 5912 FI->setFramePointerSaveIndex(FPSI); 5913 } 5914 return DAG.getFrameIndex(FPSI, PtrVT); 5915 } 5916 5917 SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 5918 SelectionDAG &DAG, 5919 const PPCSubtarget &Subtarget) const { 5920 // Get the inputs. 5921 SDValue Chain = Op.getOperand(0); 5922 SDValue Size = Op.getOperand(1); 5923 SDLoc dl(Op); 5924 5925 // Get the corect type for pointers. 5926 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 5927 // Negate the size. 5928 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, 5929 DAG.getConstant(0, dl, PtrVT), Size); 5930 // Construct a node for the frame pointer save index. 5931 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 5932 // Build a DYNALLOC node. 5933 SDValue Ops[3] = { Chain, NegSize, FPSIdx }; 5934 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); 5935 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); 5936 } 5937 5938 SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, 5939 SelectionDAG &DAG) const { 5940 SDLoc DL(Op); 5941 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, 5942 DAG.getVTList(MVT::i32, MVT::Other), 5943 Op.getOperand(0), Op.getOperand(1)); 5944 } 5945 5946 SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, 5947 SelectionDAG &DAG) const { 5948 SDLoc DL(Op); 5949 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, 5950 Op.getOperand(0), Op.getOperand(1)); 5951 } 5952 5953 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 5954 if (Op.getValueType().isVector()) 5955 return LowerVectorLoad(Op, DAG); 5956 5957 assert(Op.getValueType() == MVT::i1 && 5958 "Custom lowering only for i1 loads"); 5959 5960 // First, load 8 bits into 32 bits, then truncate to 1 bit. 5961 5962 SDLoc dl(Op); 5963 LoadSDNode *LD = cast<LoadSDNode>(Op); 5964 5965 SDValue Chain = LD->getChain(); 5966 SDValue BasePtr = LD->getBasePtr(); 5967 MachineMemOperand *MMO = LD->getMemOperand(); 5968 5969 SDValue NewLD = 5970 DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain, 5971 BasePtr, MVT::i8, MMO); 5972 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); 5973 5974 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; 5975 return DAG.getMergeValues(Ops, dl); 5976 } 5977 5978 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 5979 if (Op.getOperand(1).getValueType().isVector()) 5980 return LowerVectorStore(Op, DAG); 5981 5982 assert(Op.getOperand(1).getValueType() == MVT::i1 && 5983 "Custom lowering only for i1 stores"); 5984 5985 // First, zero extend to 32 bits, then use a truncating store to 8 bits. 5986 5987 SDLoc dl(Op); 5988 StoreSDNode *ST = cast<StoreSDNode>(Op); 5989 5990 SDValue Chain = ST->getChain(); 5991 SDValue BasePtr = ST->getBasePtr(); 5992 SDValue Value = ST->getValue(); 5993 MachineMemOperand *MMO = ST->getMemOperand(); 5994 5995 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()), 5996 Value); 5997 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); 5998 } 5999 6000 // FIXME: Remove this once the ANDI glue bug is fixed: 6001 SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { 6002 assert(Op.getValueType() == MVT::i1 && 6003 "Custom lowering only for i1 results"); 6004 6005 SDLoc DL(Op); 6006 return DAG.getNode(PPCISD::ANDIo_1_GT_BIT, DL, MVT::i1, 6007 Op.getOperand(0)); 6008 } 6009 6010 /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when 6011 /// possible. 6012 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 6013 // Not FP? Not a fsel. 6014 if (!Op.getOperand(0).getValueType().isFloatingPoint() || 6015 !Op.getOperand(2).getValueType().isFloatingPoint()) 6016 return Op; 6017 6018 // We might be able to do better than this under some circumstances, but in 6019 // general, fsel-based lowering of select is a finite-math-only optimization. 6020 // For more information, see section F.3 of the 2.06 ISA specification. 6021 if (!DAG.getTarget().Options.NoInfsFPMath || 6022 !DAG.getTarget().Options.NoNaNsFPMath) 6023 return Op; 6024 // TODO: Propagate flags from the select rather than global settings. 6025 SDNodeFlags Flags; 6026 Flags.setNoInfs(true); 6027 Flags.setNoNaNs(true); 6028 6029 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 6030 6031 EVT ResVT = Op.getValueType(); 6032 EVT CmpVT = Op.getOperand(0).getValueType(); 6033 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 6034 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); 6035 SDLoc dl(Op); 6036 6037 // If the RHS of the comparison is a 0.0, we don't need to do the 6038 // subtraction at all. 6039 SDValue Sel1; 6040 if (isFloatingPointZero(RHS)) 6041 switch (CC) { 6042 default: break; // SETUO etc aren't handled by fsel. 6043 case ISD::SETNE: 6044 std::swap(TV, FV); 6045 case ISD::SETEQ: 6046 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 6047 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 6048 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 6049 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 6050 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 6051 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 6052 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); 6053 case ISD::SETULT: 6054 case ISD::SETLT: 6055 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 6056 case ISD::SETOGE: 6057 case ISD::SETGE: 6058 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 6059 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 6060 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 6061 case ISD::SETUGT: 6062 case ISD::SETGT: 6063 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 6064 case ISD::SETOLE: 6065 case ISD::SETLE: 6066 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 6067 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 6068 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 6069 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); 6070 } 6071 6072 SDValue Cmp; 6073 switch (CC) { 6074 default: break; // SETUO etc aren't handled by fsel. 6075 case ISD::SETNE: 6076 std::swap(TV, FV); 6077 case ISD::SETEQ: 6078 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, &Flags); 6079 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 6080 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 6081 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 6082 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 6083 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 6084 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 6085 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); 6086 case ISD::SETULT: 6087 case ISD::SETLT: 6088 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, &Flags); 6089 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 6090 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 6091 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 6092 case ISD::SETOGE: 6093 case ISD::SETGE: 6094 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, &Flags); 6095 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 6096 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 6097 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 6098 case ISD::SETUGT: 6099 case ISD::SETGT: 6100 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, &Flags); 6101 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 6102 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 6103 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 6104 case ISD::SETOLE: 6105 case ISD::SETLE: 6106 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, &Flags); 6107 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 6108 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 6109 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 6110 } 6111 return Op; 6112 } 6113 6114 void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, 6115 SelectionDAG &DAG, 6116 SDLoc dl) const { 6117 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 6118 SDValue Src = Op.getOperand(0); 6119 if (Src.getValueType() == MVT::f32) 6120 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 6121 6122 SDValue Tmp; 6123 switch (Op.getSimpleValueType().SimpleTy) { 6124 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 6125 case MVT::i32: 6126 Tmp = DAG.getNode( 6127 Op.getOpcode() == ISD::FP_TO_SINT 6128 ? PPCISD::FCTIWZ 6129 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), 6130 dl, MVT::f64, Src); 6131 break; 6132 case MVT::i64: 6133 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && 6134 "i64 FP_TO_UINT is supported only with FPCVT"); 6135 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 6136 PPCISD::FCTIDUZ, 6137 dl, MVT::f64, Src); 6138 break; 6139 } 6140 6141 // Convert the FP value to an int value through memory. 6142 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && 6143 (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()); 6144 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); 6145 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex(); 6146 MachinePointerInfo MPI = 6147 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 6148 6149 // Emit a store to the stack slot. 6150 SDValue Chain; 6151 if (i32Stack) { 6152 MachineFunction &MF = DAG.getMachineFunction(); 6153 MachineMemOperand *MMO = 6154 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4); 6155 SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; 6156 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 6157 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); 6158 } else 6159 Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, 6160 MPI, false, false, 0); 6161 6162 // Result is a load from the stack slot. If loading 4 bytes, make sure to 6163 // add in a bias. 6164 if (Op.getValueType() == MVT::i32 && !i32Stack) { 6165 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, 6166 DAG.getConstant(4, dl, FIPtr.getValueType())); 6167 MPI = MPI.getWithOffset(4); 6168 } 6169 6170 RLI.Chain = Chain; 6171 RLI.Ptr = FIPtr; 6172 RLI.MPI = MPI; 6173 } 6174 6175 /// \brief Custom lowers floating point to integer conversions to use 6176 /// the direct move instructions available in ISA 2.07 to avoid the 6177 /// need for load/store combinations. 6178 SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, 6179 SelectionDAG &DAG, 6180 SDLoc dl) const { 6181 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 6182 SDValue Src = Op.getOperand(0); 6183 6184 if (Src.getValueType() == MVT::f32) 6185 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 6186 6187 SDValue Tmp; 6188 switch (Op.getSimpleValueType().SimpleTy) { 6189 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 6190 case MVT::i32: 6191 Tmp = DAG.getNode( 6192 Op.getOpcode() == ISD::FP_TO_SINT 6193 ? PPCISD::FCTIWZ 6194 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), 6195 dl, MVT::f64, Src); 6196 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp); 6197 break; 6198 case MVT::i64: 6199 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && 6200 "i64 FP_TO_UINT is supported only with FPCVT"); 6201 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 6202 PPCISD::FCTIDUZ, 6203 dl, MVT::f64, Src); 6204 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp); 6205 break; 6206 } 6207 return Tmp; 6208 } 6209 6210 SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, 6211 SDLoc dl) const { 6212 if (Subtarget.hasDirectMove() && Subtarget.isPPC64()) 6213 return LowerFP_TO_INTDirectMove(Op, DAG, dl); 6214 6215 ReuseLoadInfo RLI; 6216 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 6217 6218 return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, false, 6219 false, RLI.IsInvariant, RLI.Alignment, RLI.AAInfo, 6220 RLI.Ranges); 6221 } 6222 6223 // We're trying to insert a regular store, S, and then a load, L. If the 6224 // incoming value, O, is a load, we might just be able to have our load use the 6225 // address used by O. However, we don't know if anything else will store to 6226 // that address before we can load from it. To prevent this situation, we need 6227 // to insert our load, L, into the chain as a peer of O. To do this, we give L 6228 // the same chain operand as O, we create a token factor from the chain results 6229 // of O and L, and we replace all uses of O's chain result with that token 6230 // factor (see spliceIntoChain below for this last part). 6231 bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, 6232 ReuseLoadInfo &RLI, 6233 SelectionDAG &DAG, 6234 ISD::LoadExtType ET) const { 6235 SDLoc dl(Op); 6236 if (ET == ISD::NON_EXTLOAD && 6237 (Op.getOpcode() == ISD::FP_TO_UINT || 6238 Op.getOpcode() == ISD::FP_TO_SINT) && 6239 isOperationLegalOrCustom(Op.getOpcode(), 6240 Op.getOperand(0).getValueType())) { 6241 6242 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 6243 return true; 6244 } 6245 6246 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op); 6247 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() || 6248 LD->isNonTemporal()) 6249 return false; 6250 if (LD->getMemoryVT() != MemVT) 6251 return false; 6252 6253 RLI.Ptr = LD->getBasePtr(); 6254 if (LD->isIndexed() && LD->getOffset().getOpcode() != ISD::UNDEF) { 6255 assert(LD->getAddressingMode() == ISD::PRE_INC && 6256 "Non-pre-inc AM on PPC?"); 6257 RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr, 6258 LD->getOffset()); 6259 } 6260 6261 RLI.Chain = LD->getChain(); 6262 RLI.MPI = LD->getPointerInfo(); 6263 RLI.IsInvariant = LD->isInvariant(); 6264 RLI.Alignment = LD->getAlignment(); 6265 RLI.AAInfo = LD->getAAInfo(); 6266 RLI.Ranges = LD->getRanges(); 6267 6268 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1); 6269 return true; 6270 } 6271 6272 // Given the head of the old chain, ResChain, insert a token factor containing 6273 // it and NewResChain, and make users of ResChain now be users of that token 6274 // factor. 6275 void PPCTargetLowering::spliceIntoChain(SDValue ResChain, 6276 SDValue NewResChain, 6277 SelectionDAG &DAG) const { 6278 if (!ResChain) 6279 return; 6280 6281 SDLoc dl(NewResChain); 6282 6283 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 6284 NewResChain, DAG.getUNDEF(MVT::Other)); 6285 assert(TF.getNode() != NewResChain.getNode() && 6286 "A new TF really is required here"); 6287 6288 DAG.ReplaceAllUsesOfValueWith(ResChain, TF); 6289 DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain); 6290 } 6291 6292 /// \brief Custom lowers integer to floating point conversions to use 6293 /// the direct move instructions available in ISA 2.07 to avoid the 6294 /// need for load/store combinations. 6295 SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, 6296 SelectionDAG &DAG, 6297 SDLoc dl) const { 6298 assert((Op.getValueType() == MVT::f32 || 6299 Op.getValueType() == MVT::f64) && 6300 "Invalid floating point type as target of conversion"); 6301 assert(Subtarget.hasFPCVT() && 6302 "Int to FP conversions with direct moves require FPCVT"); 6303 SDValue FP; 6304 SDValue Src = Op.getOperand(0); 6305 bool SinglePrec = Op.getValueType() == MVT::f32; 6306 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; 6307 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP; 6308 unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) : 6309 (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU); 6310 6311 if (WordInt) { 6312 FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ, 6313 dl, MVT::f64, Src); 6314 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); 6315 } 6316 else { 6317 FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src); 6318 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); 6319 } 6320 6321 return FP; 6322 } 6323 6324 SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, 6325 SelectionDAG &DAG) const { 6326 SDLoc dl(Op); 6327 6328 if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) { 6329 if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64) 6330 return SDValue(); 6331 6332 SDValue Value = Op.getOperand(0); 6333 // The values are now known to be -1 (false) or 1 (true). To convert this 6334 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 6335 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 6336 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 6337 6338 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64); 6339 FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64, FPHalfs, FPHalfs, 6340 FPHalfs, FPHalfs); 6341 6342 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 6343 6344 if (Op.getValueType() != MVT::v4f64) 6345 Value = DAG.getNode(ISD::FP_ROUND, dl, 6346 Op.getValueType(), Value, 6347 DAG.getIntPtrConstant(1, dl)); 6348 return Value; 6349 } 6350 6351 // Don't handle ppc_fp128 here; let it be lowered to a libcall. 6352 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 6353 return SDValue(); 6354 6355 if (Op.getOperand(0).getValueType() == MVT::i1) 6356 return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), 6357 DAG.getConstantFP(1.0, dl, Op.getValueType()), 6358 DAG.getConstantFP(0.0, dl, Op.getValueType())); 6359 6360 // If we have direct moves, we can do all the conversion, skip the store/load 6361 // however, without FPCVT we can't do most conversions. 6362 if (Subtarget.hasDirectMove() && Subtarget.isPPC64() && Subtarget.hasFPCVT()) 6363 return LowerINT_TO_FPDirectMove(Op, DAG, dl); 6364 6365 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 6366 "UINT_TO_FP is supported only with FPCVT"); 6367 6368 // If we have FCFIDS, then use it when converting to single-precision. 6369 // Otherwise, convert to double-precision and then round. 6370 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 6371 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 6372 : PPCISD::FCFIDS) 6373 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 6374 : PPCISD::FCFID); 6375 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 6376 ? MVT::f32 6377 : MVT::f64; 6378 6379 if (Op.getOperand(0).getValueType() == MVT::i64) { 6380 SDValue SINT = Op.getOperand(0); 6381 // When converting to single-precision, we actually need to convert 6382 // to double-precision first and then round to single-precision. 6383 // To avoid double-rounding effects during that operation, we have 6384 // to prepare the input operand. Bits that might be truncated when 6385 // converting to double-precision are replaced by a bit that won't 6386 // be lost at this stage, but is below the single-precision rounding 6387 // position. 6388 // 6389 // However, if -enable-unsafe-fp-math is in effect, accept double 6390 // rounding to avoid the extra overhead. 6391 if (Op.getValueType() == MVT::f32 && 6392 !Subtarget.hasFPCVT() && 6393 !DAG.getTarget().Options.UnsafeFPMath) { 6394 6395 // Twiddle input to make sure the low 11 bits are zero. (If this 6396 // is the case, we are guaranteed the value will fit into the 53 bit 6397 // mantissa of an IEEE double-precision value without rounding.) 6398 // If any of those low 11 bits were not zero originally, make sure 6399 // bit 12 (value 2048) is set instead, so that the final rounding 6400 // to single-precision gets the correct result. 6401 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, 6402 SINT, DAG.getConstant(2047, dl, MVT::i64)); 6403 Round = DAG.getNode(ISD::ADD, dl, MVT::i64, 6404 Round, DAG.getConstant(2047, dl, MVT::i64)); 6405 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); 6406 Round = DAG.getNode(ISD::AND, dl, MVT::i64, 6407 Round, DAG.getConstant(-2048, dl, MVT::i64)); 6408 6409 // However, we cannot use that value unconditionally: if the magnitude 6410 // of the input value is small, the bit-twiddling we did above might 6411 // end up visibly changing the output. Fortunately, in that case, we 6412 // don't need to twiddle bits since the original input will convert 6413 // exactly to double-precision floating-point already. Therefore, 6414 // construct a conditional to use the original value if the top 11 6415 // bits are all sign-bit copies, and use the rounded value computed 6416 // above otherwise. 6417 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, 6418 SINT, DAG.getConstant(53, dl, MVT::i32)); 6419 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, 6420 Cond, DAG.getConstant(1, dl, MVT::i64)); 6421 Cond = DAG.getSetCC(dl, MVT::i32, 6422 Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT); 6423 6424 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); 6425 } 6426 6427 ReuseLoadInfo RLI; 6428 SDValue Bits; 6429 6430 MachineFunction &MF = DAG.getMachineFunction(); 6431 if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { 6432 Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, false, 6433 false, RLI.IsInvariant, RLI.Alignment, RLI.AAInfo, 6434 RLI.Ranges); 6435 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 6436 } else if (Subtarget.hasLFIWAX() && 6437 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { 6438 MachineMemOperand *MMO = 6439 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 6440 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 6441 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 6442 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl, 6443 DAG.getVTList(MVT::f64, MVT::Other), 6444 Ops, MVT::i32, MMO); 6445 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 6446 } else if (Subtarget.hasFPCVT() && 6447 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) { 6448 MachineMemOperand *MMO = 6449 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 6450 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 6451 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 6452 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl, 6453 DAG.getVTList(MVT::f64, MVT::Other), 6454 Ops, MVT::i32, MMO); 6455 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 6456 } else if (((Subtarget.hasLFIWAX() && 6457 SINT.getOpcode() == ISD::SIGN_EXTEND) || 6458 (Subtarget.hasFPCVT() && 6459 SINT.getOpcode() == ISD::ZERO_EXTEND)) && 6460 SINT.getOperand(0).getValueType() == MVT::i32) { 6461 MachineFrameInfo *FrameInfo = MF.getFrameInfo(); 6462 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 6463 6464 int FrameIdx = FrameInfo->CreateStackObject(4, 4, false); 6465 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 6466 6467 SDValue Store = DAG.getStore( 6468 DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx, 6469 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx), 6470 false, false, 0); 6471 6472 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 6473 "Expected an i32 store"); 6474 6475 RLI.Ptr = FIdx; 6476 RLI.Chain = Store; 6477 RLI.MPI = 6478 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 6479 RLI.Alignment = 4; 6480 6481 MachineMemOperand *MMO = 6482 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 6483 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 6484 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 6485 Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ? 6486 PPCISD::LFIWZX : PPCISD::LFIWAX, 6487 dl, DAG.getVTList(MVT::f64, MVT::Other), 6488 Ops, MVT::i32, MMO); 6489 } else 6490 Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); 6491 6492 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); 6493 6494 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) 6495 FP = DAG.getNode(ISD::FP_ROUND, dl, 6496 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 6497 return FP; 6498 } 6499 6500 assert(Op.getOperand(0).getValueType() == MVT::i32 && 6501 "Unhandled INT_TO_FP type in custom expander!"); 6502 // Since we only generate this in 64-bit mode, we can take advantage of 6503 // 64-bit registers. In particular, sign extend the input value into the 6504 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack 6505 // then lfd it and fcfid it. 6506 MachineFunction &MF = DAG.getMachineFunction(); 6507 MachineFrameInfo *FrameInfo = MF.getFrameInfo(); 6508 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 6509 6510 SDValue Ld; 6511 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { 6512 ReuseLoadInfo RLI; 6513 bool ReusingLoad; 6514 if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI, 6515 DAG))) { 6516 int FrameIdx = FrameInfo->CreateStackObject(4, 4, false); 6517 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 6518 6519 SDValue Store = DAG.getStore( 6520 DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 6521 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx), 6522 false, false, 0); 6523 6524 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 6525 "Expected an i32 store"); 6526 6527 RLI.Ptr = FIdx; 6528 RLI.Chain = Store; 6529 RLI.MPI = 6530 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 6531 RLI.Alignment = 4; 6532 } 6533 6534 MachineMemOperand *MMO = 6535 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 6536 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 6537 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 6538 Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? 6539 PPCISD::LFIWZX : PPCISD::LFIWAX, 6540 dl, DAG.getVTList(MVT::f64, MVT::Other), 6541 Ops, MVT::i32, MMO); 6542 if (ReusingLoad) 6543 spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); 6544 } else { 6545 assert(Subtarget.isPPC64() && 6546 "i32->FP without LFIWAX supported only on PPC64"); 6547 6548 int FrameIdx = FrameInfo->CreateStackObject(8, 8, false); 6549 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 6550 6551 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, 6552 Op.getOperand(0)); 6553 6554 // STD the extended value into the stack slot. 6555 SDValue Store = DAG.getStore( 6556 DAG.getEntryNode(), dl, Ext64, FIdx, 6557 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx), 6558 false, false, 0); 6559 6560 // Load the value as a double. 6561 Ld = DAG.getLoad( 6562 MVT::f64, dl, Store, FIdx, 6563 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx), 6564 false, false, false, 0); 6565 } 6566 6567 // FCFID it and return it. 6568 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); 6569 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) 6570 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, 6571 DAG.getIntPtrConstant(0, dl)); 6572 return FP; 6573 } 6574 6575 SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 6576 SelectionDAG &DAG) const { 6577 SDLoc dl(Op); 6578 /* 6579 The rounding mode is in bits 30:31 of FPSR, and has the following 6580 settings: 6581 00 Round to nearest 6582 01 Round to 0 6583 10 Round to +inf 6584 11 Round to -inf 6585 6586 FLT_ROUNDS, on the other hand, expects the following: 6587 -1 Undefined 6588 0 Round to 0 6589 1 Round to nearest 6590 2 Round to +inf 6591 3 Round to -inf 6592 6593 To perform the conversion, we do: 6594 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) 6595 */ 6596 6597 MachineFunction &MF = DAG.getMachineFunction(); 6598 EVT VT = Op.getValueType(); 6599 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 6600 6601 // Save FP Control Word to register 6602 EVT NodeTys[] = { 6603 MVT::f64, // return register 6604 MVT::Glue // unused in this context 6605 }; 6606 SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None); 6607 6608 // Save FP register to stack slot 6609 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8, false); 6610 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); 6611 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain, 6612 StackSlot, MachinePointerInfo(), false, false,0); 6613 6614 // Load FP Control Word from low 32 bits of stack slot. 6615 SDValue Four = DAG.getConstant(4, dl, PtrVT); 6616 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); 6617 SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo(), 6618 false, false, false, 0); 6619 6620 // Transform as necessary 6621 SDValue CWD1 = 6622 DAG.getNode(ISD::AND, dl, MVT::i32, 6623 CWD, DAG.getConstant(3, dl, MVT::i32)); 6624 SDValue CWD2 = 6625 DAG.getNode(ISD::SRL, dl, MVT::i32, 6626 DAG.getNode(ISD::AND, dl, MVT::i32, 6627 DAG.getNode(ISD::XOR, dl, MVT::i32, 6628 CWD, DAG.getConstant(3, dl, MVT::i32)), 6629 DAG.getConstant(3, dl, MVT::i32)), 6630 DAG.getConstant(1, dl, MVT::i32)); 6631 6632 SDValue RetVal = 6633 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); 6634 6635 return DAG.getNode((VT.getSizeInBits() < 16 ? 6636 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); 6637 } 6638 6639 SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { 6640 EVT VT = Op.getValueType(); 6641 unsigned BitWidth = VT.getSizeInBits(); 6642 SDLoc dl(Op); 6643 assert(Op.getNumOperands() == 3 && 6644 VT == Op.getOperand(1).getValueType() && 6645 "Unexpected SHL!"); 6646 6647 // Expand into a bunch of logical ops. Note that these ops 6648 // depend on the PPC behavior for oversized shift amounts. 6649 SDValue Lo = Op.getOperand(0); 6650 SDValue Hi = Op.getOperand(1); 6651 SDValue Amt = Op.getOperand(2); 6652 EVT AmtVT = Amt.getValueType(); 6653 6654 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 6655 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 6656 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); 6657 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); 6658 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); 6659 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 6660 DAG.getConstant(-BitWidth, dl, AmtVT)); 6661 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); 6662 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 6663 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); 6664 SDValue OutOps[] = { OutLo, OutHi }; 6665 return DAG.getMergeValues(OutOps, dl); 6666 } 6667 6668 SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { 6669 EVT VT = Op.getValueType(); 6670 SDLoc dl(Op); 6671 unsigned BitWidth = VT.getSizeInBits(); 6672 assert(Op.getNumOperands() == 3 && 6673 VT == Op.getOperand(1).getValueType() && 6674 "Unexpected SRL!"); 6675 6676 // Expand into a bunch of logical ops. Note that these ops 6677 // depend on the PPC behavior for oversized shift amounts. 6678 SDValue Lo = Op.getOperand(0); 6679 SDValue Hi = Op.getOperand(1); 6680 SDValue Amt = Op.getOperand(2); 6681 EVT AmtVT = Amt.getValueType(); 6682 6683 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 6684 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 6685 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 6686 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 6687 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 6688 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 6689 DAG.getConstant(-BitWidth, dl, AmtVT)); 6690 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); 6691 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 6692 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); 6693 SDValue OutOps[] = { OutLo, OutHi }; 6694 return DAG.getMergeValues(OutOps, dl); 6695 } 6696 6697 SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { 6698 SDLoc dl(Op); 6699 EVT VT = Op.getValueType(); 6700 unsigned BitWidth = VT.getSizeInBits(); 6701 assert(Op.getNumOperands() == 3 && 6702 VT == Op.getOperand(1).getValueType() && 6703 "Unexpected SRA!"); 6704 6705 // Expand into a bunch of logical ops, followed by a select_cc. 6706 SDValue Lo = Op.getOperand(0); 6707 SDValue Hi = Op.getOperand(1); 6708 SDValue Amt = Op.getOperand(2); 6709 EVT AmtVT = Amt.getValueType(); 6710 6711 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 6712 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 6713 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 6714 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 6715 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 6716 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 6717 DAG.getConstant(-BitWidth, dl, AmtVT)); 6718 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); 6719 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); 6720 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT), 6721 Tmp4, Tmp6, ISD::SETLE); 6722 SDValue OutOps[] = { OutLo, OutHi }; 6723 return DAG.getMergeValues(OutOps, dl); 6724 } 6725 6726 //===----------------------------------------------------------------------===// 6727 // Vector related lowering. 6728 // 6729 6730 /// BuildSplatI - Build a canonical splati of Val with an element size of 6731 /// SplatSize. Cast the result to VT. 6732 static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT, 6733 SelectionDAG &DAG, SDLoc dl) { 6734 assert(Val >= -16 && Val <= 15 && "vsplti is out of range!"); 6735 6736 static const MVT VTys[] = { // canonical VT to use for each size. 6737 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 6738 }; 6739 6740 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; 6741 6742 // Force vspltis[hw] -1 to vspltisb -1 to canonicalize. 6743 if (Val == -1) 6744 SplatSize = 1; 6745 6746 EVT CanonicalVT = VTys[SplatSize-1]; 6747 6748 // Build a canonical splat for this value. 6749 SDValue Elt = DAG.getConstant(Val, dl, MVT::i32); 6750 SmallVector<SDValue, 8> Ops; 6751 Ops.assign(CanonicalVT.getVectorNumElements(), Elt); 6752 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, CanonicalVT, Ops); 6753 return DAG.getNode(ISD::BITCAST, dl, ReqVT, Res); 6754 } 6755 6756 /// BuildIntrinsicOp - Return a unary operator intrinsic node with the 6757 /// specified intrinsic ID. 6758 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, 6759 SelectionDAG &DAG, SDLoc dl, 6760 EVT DestVT = MVT::Other) { 6761 if (DestVT == MVT::Other) DestVT = Op.getValueType(); 6762 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 6763 DAG.getConstant(IID, dl, MVT::i32), Op); 6764 } 6765 6766 /// BuildIntrinsicOp - Return a binary operator intrinsic node with the 6767 /// specified intrinsic ID. 6768 static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, 6769 SelectionDAG &DAG, SDLoc dl, 6770 EVT DestVT = MVT::Other) { 6771 if (DestVT == MVT::Other) DestVT = LHS.getValueType(); 6772 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 6773 DAG.getConstant(IID, dl, MVT::i32), LHS, RHS); 6774 } 6775 6776 /// BuildIntrinsicOp - Return a ternary operator intrinsic node with the 6777 /// specified intrinsic ID. 6778 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, 6779 SDValue Op2, SelectionDAG &DAG, 6780 SDLoc dl, EVT DestVT = MVT::Other) { 6781 if (DestVT == MVT::Other) DestVT = Op0.getValueType(); 6782 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 6783 DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2); 6784 } 6785 6786 /// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified 6787 /// amount. The result has the specified value type. 6788 static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, 6789 EVT VT, SelectionDAG &DAG, SDLoc dl) { 6790 // Force LHS/RHS to be the right type. 6791 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); 6792 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); 6793 6794 int Ops[16]; 6795 for (unsigned i = 0; i != 16; ++i) 6796 Ops[i] = i + Amt; 6797 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); 6798 return DAG.getNode(ISD::BITCAST, dl, VT, T); 6799 } 6800 6801 // If this is a case we can't handle, return null and let the default 6802 // expansion code take care of it. If we CAN select this case, and if it 6803 // selects to a single instruction, return Op. Otherwise, if we can codegen 6804 // this case more efficiently than a constant pool load, lower it to the 6805 // sequence of ops that should be used. 6806 SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, 6807 SelectionDAG &DAG) const { 6808 SDLoc dl(Op); 6809 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 6810 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); 6811 6812 if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) { 6813 // We first build an i32 vector, load it into a QPX register, 6814 // then convert it to a floating-point vector and compare it 6815 // to a zero vector to get the boolean result. 6816 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 6817 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false); 6818 MachinePointerInfo PtrInfo = 6819 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 6820 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 6821 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 6822 6823 assert(BVN->getNumOperands() == 4 && 6824 "BUILD_VECTOR for v4i1 does not have 4 operands"); 6825 6826 bool IsConst = true; 6827 for (unsigned i = 0; i < 4; ++i) { 6828 if (BVN->getOperand(i).getOpcode() == ISD::UNDEF) continue; 6829 if (!isa<ConstantSDNode>(BVN->getOperand(i))) { 6830 IsConst = false; 6831 break; 6832 } 6833 } 6834 6835 if (IsConst) { 6836 Constant *One = 6837 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0); 6838 Constant *NegOne = 6839 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0); 6840 6841 SmallVector<Constant*, 4> CV(4, NegOne); 6842 for (unsigned i = 0; i < 4; ++i) { 6843 if (BVN->getOperand(i).getOpcode() == ISD::UNDEF) 6844 CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext())); 6845 else if (isNullConstant(BVN->getOperand(i))) 6846 continue; 6847 else 6848 CV[i] = One; 6849 } 6850 6851 Constant *CP = ConstantVector::get(CV); 6852 SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), 6853 16 /* alignment */); 6854 6855 SmallVector<SDValue, 2> Ops; 6856 Ops.push_back(DAG.getEntryNode()); 6857 Ops.push_back(CPIdx); 6858 6859 SmallVector<EVT, 2> ValueVTs; 6860 ValueVTs.push_back(MVT::v4i1); 6861 ValueVTs.push_back(MVT::Other); // chain 6862 SDVTList VTs = DAG.getVTList(ValueVTs); 6863 6864 return DAG.getMemIntrinsicNode( 6865 PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32, 6866 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 6867 } 6868 6869 SmallVector<SDValue, 4> Stores; 6870 for (unsigned i = 0; i < 4; ++i) { 6871 if (BVN->getOperand(i).getOpcode() == ISD::UNDEF) continue; 6872 6873 unsigned Offset = 4*i; 6874 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 6875 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 6876 6877 unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize(); 6878 if (StoreSize > 4) { 6879 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 6880 BVN->getOperand(i), Idx, 6881 PtrInfo.getWithOffset(Offset), 6882 MVT::i32, false, false, 0)); 6883 } else { 6884 SDValue StoreValue = BVN->getOperand(i); 6885 if (StoreSize < 4) 6886 StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue); 6887 6888 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, 6889 StoreValue, Idx, 6890 PtrInfo.getWithOffset(Offset), 6891 false, false, 0)); 6892 } 6893 } 6894 6895 SDValue StoreChain; 6896 if (!Stores.empty()) 6897 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 6898 else 6899 StoreChain = DAG.getEntryNode(); 6900 6901 // Now load from v4i32 into the QPX register; this will extend it to 6902 // v4i64 but not yet convert it to a floating point. Nevertheless, this 6903 // is typed as v4f64 because the QPX register integer states are not 6904 // explicitly represented. 6905 6906 SmallVector<SDValue, 2> Ops; 6907 Ops.push_back(StoreChain); 6908 Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32)); 6909 Ops.push_back(FIdx); 6910 6911 SmallVector<EVT, 2> ValueVTs; 6912 ValueVTs.push_back(MVT::v4f64); 6913 ValueVTs.push_back(MVT::Other); // chain 6914 SDVTList VTs = DAG.getVTList(ValueVTs); 6915 6916 SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, 6917 dl, VTs, Ops, MVT::v4i32, PtrInfo); 6918 LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 6919 DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32), 6920 LoadedVect); 6921 6922 SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::f64); 6923 FPZeros = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64, 6924 FPZeros, FPZeros, FPZeros, FPZeros); 6925 6926 return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ); 6927 } 6928 6929 // All other QPX vectors are handled by generic code. 6930 if (Subtarget.hasQPX()) 6931 return SDValue(); 6932 6933 // Check if this is a splat of a constant value. 6934 APInt APSplatBits, APSplatUndef; 6935 unsigned SplatBitSize; 6936 bool HasAnyUndefs; 6937 if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, 6938 HasAnyUndefs, 0, !Subtarget.isLittleEndian()) || 6939 SplatBitSize > 32) 6940 return SDValue(); 6941 6942 unsigned SplatBits = APSplatBits.getZExtValue(); 6943 unsigned SplatUndef = APSplatUndef.getZExtValue(); 6944 unsigned SplatSize = SplatBitSize / 8; 6945 6946 // First, handle single instruction cases. 6947 6948 // All zeros? 6949 if (SplatBits == 0) { 6950 // Canonicalize all zero vectors to be v4i32. 6951 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { 6952 SDValue Z = DAG.getConstant(0, dl, MVT::i32); 6953 Z = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Z, Z, Z, Z); 6954 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); 6955 } 6956 return Op; 6957 } 6958 6959 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. 6960 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> 6961 (32-SplatBitSize)); 6962 if (SextVal >= -16 && SextVal <= 15) 6963 return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl); 6964 6965 // Two instruction sequences. 6966 6967 // If this value is in the range [-32,30] and is even, use: 6968 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) 6969 // If this value is in the range [17,31] and is odd, use: 6970 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) 6971 // If this value is in the range [-31,-17] and is odd, use: 6972 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) 6973 // Note the last two are three-instruction sequences. 6974 if (SextVal >= -32 && SextVal <= 31) { 6975 // To avoid having these optimizations undone by constant folding, 6976 // we convert to a pseudo that will be expanded later into one of 6977 // the above forms. 6978 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32); 6979 EVT VT = (SplatSize == 1 ? MVT::v16i8 : 6980 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); 6981 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32); 6982 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); 6983 if (VT == Op.getValueType()) 6984 return RetVal; 6985 else 6986 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); 6987 } 6988 6989 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is 6990 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important 6991 // for fneg/fabs. 6992 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { 6993 // Make -1 and vspltisw -1: 6994 SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl); 6995 6996 // Make the VSLW intrinsic, computing 0x8000_0000. 6997 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, 6998 OnesV, DAG, dl); 6999 7000 // xor by OnesV to invert it. 7001 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); 7002 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 7003 } 7004 7005 // Check to see if this is a wide variety of vsplti*, binop self cases. 7006 static const signed char SplatCsts[] = { 7007 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, 7008 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 7009 }; 7010 7011 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { 7012 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for 7013 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' 7014 int i = SplatCsts[idx]; 7015 7016 // Figure out what shift amount will be used by altivec if shifted by i in 7017 // this splat size. 7018 unsigned TypeShiftAmt = i & (SplatBitSize-1); 7019 7020 // vsplti + shl self. 7021 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { 7022 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 7023 static const unsigned IIDs[] = { // Intrinsic to use for each size. 7024 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, 7025 Intrinsic::ppc_altivec_vslw 7026 }; 7027 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 7028 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 7029 } 7030 7031 // vsplti + srl self. 7032 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 7033 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 7034 static const unsigned IIDs[] = { // Intrinsic to use for each size. 7035 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, 7036 Intrinsic::ppc_altivec_vsrw 7037 }; 7038 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 7039 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 7040 } 7041 7042 // vsplti + sra self. 7043 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 7044 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 7045 static const unsigned IIDs[] = { // Intrinsic to use for each size. 7046 Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, 7047 Intrinsic::ppc_altivec_vsraw 7048 }; 7049 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 7050 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 7051 } 7052 7053 // vsplti + rol self. 7054 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | 7055 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { 7056 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 7057 static const unsigned IIDs[] = { // Intrinsic to use for each size. 7058 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, 7059 Intrinsic::ppc_altivec_vrlw 7060 }; 7061 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 7062 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 7063 } 7064 7065 // t = vsplti c, result = vsldoi t, t, 1 7066 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { 7067 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 7068 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1; 7069 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 7070 } 7071 // t = vsplti c, result = vsldoi t, t, 2 7072 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { 7073 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 7074 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2; 7075 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 7076 } 7077 // t = vsplti c, result = vsldoi t, t, 3 7078 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { 7079 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 7080 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3; 7081 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 7082 } 7083 } 7084 7085 return SDValue(); 7086 } 7087 7088 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 7089 /// the specified operations to build the shuffle. 7090 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 7091 SDValue RHS, SelectionDAG &DAG, 7092 SDLoc dl) { 7093 unsigned OpNum = (PFEntry >> 26) & 0x0F; 7094 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 7095 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 7096 7097 enum { 7098 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 7099 OP_VMRGHW, 7100 OP_VMRGLW, 7101 OP_VSPLTISW0, 7102 OP_VSPLTISW1, 7103 OP_VSPLTISW2, 7104 OP_VSPLTISW3, 7105 OP_VSLDOI4, 7106 OP_VSLDOI8, 7107 OP_VSLDOI12 7108 }; 7109 7110 if (OpNum == OP_COPY) { 7111 if (LHSID == (1*9+2)*9+3) return LHS; 7112 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 7113 return RHS; 7114 } 7115 7116 SDValue OpLHS, OpRHS; 7117 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 7118 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 7119 7120 int ShufIdxs[16]; 7121 switch (OpNum) { 7122 default: llvm_unreachable("Unknown i32 permute!"); 7123 case OP_VMRGHW: 7124 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; 7125 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; 7126 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; 7127 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; 7128 break; 7129 case OP_VMRGLW: 7130 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; 7131 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; 7132 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; 7133 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; 7134 break; 7135 case OP_VSPLTISW0: 7136 for (unsigned i = 0; i != 16; ++i) 7137 ShufIdxs[i] = (i&3)+0; 7138 break; 7139 case OP_VSPLTISW1: 7140 for (unsigned i = 0; i != 16; ++i) 7141 ShufIdxs[i] = (i&3)+4; 7142 break; 7143 case OP_VSPLTISW2: 7144 for (unsigned i = 0; i != 16; ++i) 7145 ShufIdxs[i] = (i&3)+8; 7146 break; 7147 case OP_VSPLTISW3: 7148 for (unsigned i = 0; i != 16; ++i) 7149 ShufIdxs[i] = (i&3)+12; 7150 break; 7151 case OP_VSLDOI4: 7152 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); 7153 case OP_VSLDOI8: 7154 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); 7155 case OP_VSLDOI12: 7156 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); 7157 } 7158 EVT VT = OpLHS.getValueType(); 7159 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); 7160 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); 7161 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); 7162 return DAG.getNode(ISD::BITCAST, dl, VT, T); 7163 } 7164 7165 /// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this 7166 /// is a shuffle we can handle in a single instruction, return it. Otherwise, 7167 /// return the code it can be lowered into. Worst case, it can always be 7168 /// lowered into a vperm. 7169 SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 7170 SelectionDAG &DAG) const { 7171 SDLoc dl(Op); 7172 SDValue V1 = Op.getOperand(0); 7173 SDValue V2 = Op.getOperand(1); 7174 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 7175 EVT VT = Op.getValueType(); 7176 bool isLittleEndian = Subtarget.isLittleEndian(); 7177 7178 if (Subtarget.hasQPX()) { 7179 if (VT.getVectorNumElements() != 4) 7180 return SDValue(); 7181 7182 if (V2.getOpcode() == ISD::UNDEF) V2 = V1; 7183 7184 int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp); 7185 if (AlignIdx != -1) { 7186 return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2, 7187 DAG.getConstant(AlignIdx, dl, MVT::i32)); 7188 } else if (SVOp->isSplat()) { 7189 int SplatIdx = SVOp->getSplatIndex(); 7190 if (SplatIdx >= 4) { 7191 std::swap(V1, V2); 7192 SplatIdx -= 4; 7193 } 7194 7195 // FIXME: If SplatIdx == 0 and the input came from a load, then there is 7196 // nothing to do. 7197 7198 return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1, 7199 DAG.getConstant(SplatIdx, dl, MVT::i32)); 7200 } 7201 7202 // Lower this into a qvgpci/qvfperm pair. 7203 7204 // Compute the qvgpci literal 7205 unsigned idx = 0; 7206 for (unsigned i = 0; i < 4; ++i) { 7207 int m = SVOp->getMaskElt(i); 7208 unsigned mm = m >= 0 ? (unsigned) m : i; 7209 idx |= mm << (3-i)*3; 7210 } 7211 7212 SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64, 7213 DAG.getConstant(idx, dl, MVT::i32)); 7214 return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3); 7215 } 7216 7217 // Cases that are handled by instructions that take permute immediates 7218 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be 7219 // selected by the instruction selector. 7220 if (V2.getOpcode() == ISD::UNDEF) { 7221 if (PPC::isSplatShuffleMask(SVOp, 1) || 7222 PPC::isSplatShuffleMask(SVOp, 2) || 7223 PPC::isSplatShuffleMask(SVOp, 4) || 7224 PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) || 7225 PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) || 7226 PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 || 7227 PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) || 7228 PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) || 7229 PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) || 7230 PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) || 7231 PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) || 7232 PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) || 7233 (Subtarget.hasP8Altivec() && ( 7234 PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) || 7235 PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) || 7236 PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) { 7237 return Op; 7238 } 7239 } 7240 7241 // Altivec has a variety of "shuffle immediates" that take two vector inputs 7242 // and produce a fixed permutation. If any of these match, do not lower to 7243 // VPERM. 7244 unsigned int ShuffleKind = isLittleEndian ? 2 : 0; 7245 if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) || 7246 PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) || 7247 PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 || 7248 PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) || 7249 PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) || 7250 PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) || 7251 PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) || 7252 PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) || 7253 PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) || 7254 (Subtarget.hasP8Altivec() && ( 7255 PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) || 7256 PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) || 7257 PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG)))) 7258 return Op; 7259 7260 // Check to see if this is a shuffle of 4-byte values. If so, we can use our 7261 // perfect shuffle table to emit an optimal matching sequence. 7262 ArrayRef<int> PermMask = SVOp->getMask(); 7263 7264 unsigned PFIndexes[4]; 7265 bool isFourElementShuffle = true; 7266 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number 7267 unsigned EltNo = 8; // Start out undef. 7268 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. 7269 if (PermMask[i*4+j] < 0) 7270 continue; // Undef, ignore it. 7271 7272 unsigned ByteSource = PermMask[i*4+j]; 7273 if ((ByteSource & 3) != j) { 7274 isFourElementShuffle = false; 7275 break; 7276 } 7277 7278 if (EltNo == 8) { 7279 EltNo = ByteSource/4; 7280 } else if (EltNo != ByteSource/4) { 7281 isFourElementShuffle = false; 7282 break; 7283 } 7284 } 7285 PFIndexes[i] = EltNo; 7286 } 7287 7288 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the 7289 // perfect shuffle vector to determine if it is cost effective to do this as 7290 // discrete instructions, or whether we should use a vperm. 7291 // For now, we skip this for little endian until such time as we have a 7292 // little-endian perfect shuffle table. 7293 if (isFourElementShuffle && !isLittleEndian) { 7294 // Compute the index in the perfect shuffle table. 7295 unsigned PFTableIndex = 7296 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 7297 7298 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 7299 unsigned Cost = (PFEntry >> 30); 7300 7301 // Determining when to avoid vperm is tricky. Many things affect the cost 7302 // of vperm, particularly how many times the perm mask needs to be computed. 7303 // For example, if the perm mask can be hoisted out of a loop or is already 7304 // used (perhaps because there are multiple permutes with the same shuffle 7305 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of 7306 // the loop requires an extra register. 7307 // 7308 // As a compromise, we only emit discrete instructions if the shuffle can be 7309 // generated in 3 or fewer operations. When we have loop information 7310 // available, if this block is within a loop, we should avoid using vperm 7311 // for 3-operation perms and use a constant pool load instead. 7312 if (Cost < 3) 7313 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 7314 } 7315 7316 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant 7317 // vector that will get spilled to the constant pool. 7318 if (V2.getOpcode() == ISD::UNDEF) V2 = V1; 7319 7320 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except 7321 // that it is in input element units, not in bytes. Convert now. 7322 7323 // For little endian, the order of the input vectors is reversed, and 7324 // the permutation mask is complemented with respect to 31. This is 7325 // necessary to produce proper semantics with the big-endian-biased vperm 7326 // instruction. 7327 EVT EltVT = V1.getValueType().getVectorElementType(); 7328 unsigned BytesPerElement = EltVT.getSizeInBits()/8; 7329 7330 SmallVector<SDValue, 16> ResultMask; 7331 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 7332 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; 7333 7334 for (unsigned j = 0; j != BytesPerElement; ++j) 7335 if (isLittleEndian) 7336 ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j), 7337 dl, MVT::i32)); 7338 else 7339 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl, 7340 MVT::i32)); 7341 } 7342 7343 SDValue VPermMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8, 7344 ResultMask); 7345 if (isLittleEndian) 7346 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 7347 V2, V1, VPermMask); 7348 else 7349 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 7350 V1, V2, VPermMask); 7351 } 7352 7353 /// getVectorCompareInfo - Given an intrinsic, return false if it is not a 7354 /// vector comparison. If it is, return true and fill in Opc/isDot with 7355 /// information about the intrinsic. 7356 static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, 7357 bool &isDot, const PPCSubtarget &Subtarget) { 7358 unsigned IntrinsicID = 7359 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue(); 7360 CompareOpc = -1; 7361 isDot = false; 7362 switch (IntrinsicID) { 7363 default: return false; 7364 // Comparison predicates. 7365 case Intrinsic::ppc_altivec_vcmpbfp_p: CompareOpc = 966; isDot = 1; break; 7366 case Intrinsic::ppc_altivec_vcmpeqfp_p: CompareOpc = 198; isDot = 1; break; 7367 case Intrinsic::ppc_altivec_vcmpequb_p: CompareOpc = 6; isDot = 1; break; 7368 case Intrinsic::ppc_altivec_vcmpequh_p: CompareOpc = 70; isDot = 1; break; 7369 case Intrinsic::ppc_altivec_vcmpequw_p: CompareOpc = 134; isDot = 1; break; 7370 case Intrinsic::ppc_altivec_vcmpequd_p: 7371 if (Subtarget.hasP8Altivec()) { 7372 CompareOpc = 199; 7373 isDot = 1; 7374 } else 7375 return false; 7376 7377 break; 7378 case Intrinsic::ppc_altivec_vcmpgefp_p: CompareOpc = 454; isDot = 1; break; 7379 case Intrinsic::ppc_altivec_vcmpgtfp_p: CompareOpc = 710; isDot = 1; break; 7380 case Intrinsic::ppc_altivec_vcmpgtsb_p: CompareOpc = 774; isDot = 1; break; 7381 case Intrinsic::ppc_altivec_vcmpgtsh_p: CompareOpc = 838; isDot = 1; break; 7382 case Intrinsic::ppc_altivec_vcmpgtsw_p: CompareOpc = 902; isDot = 1; break; 7383 case Intrinsic::ppc_altivec_vcmpgtsd_p: 7384 if (Subtarget.hasP8Altivec()) { 7385 CompareOpc = 967; 7386 isDot = 1; 7387 } else 7388 return false; 7389 7390 break; 7391 case Intrinsic::ppc_altivec_vcmpgtub_p: CompareOpc = 518; isDot = 1; break; 7392 case Intrinsic::ppc_altivec_vcmpgtuh_p: CompareOpc = 582; isDot = 1; break; 7393 case Intrinsic::ppc_altivec_vcmpgtuw_p: CompareOpc = 646; isDot = 1; break; 7394 case Intrinsic::ppc_altivec_vcmpgtud_p: 7395 if (Subtarget.hasP8Altivec()) { 7396 CompareOpc = 711; 7397 isDot = 1; 7398 } else 7399 return false; 7400 7401 break; 7402 // VSX predicate comparisons use the same infrastructure 7403 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 7404 case Intrinsic::ppc_vsx_xvcmpgedp_p: 7405 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 7406 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 7407 case Intrinsic::ppc_vsx_xvcmpgesp_p: 7408 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 7409 if (Subtarget.hasVSX()) { 7410 switch (IntrinsicID) { 7411 case Intrinsic::ppc_vsx_xvcmpeqdp_p: CompareOpc = 99; break; 7412 case Intrinsic::ppc_vsx_xvcmpgedp_p: CompareOpc = 115; break; 7413 case Intrinsic::ppc_vsx_xvcmpgtdp_p: CompareOpc = 107; break; 7414 case Intrinsic::ppc_vsx_xvcmpeqsp_p: CompareOpc = 67; break; 7415 case Intrinsic::ppc_vsx_xvcmpgesp_p: CompareOpc = 83; break; 7416 case Intrinsic::ppc_vsx_xvcmpgtsp_p: CompareOpc = 75; break; 7417 } 7418 isDot = 1; 7419 } 7420 else 7421 return false; 7422 7423 break; 7424 7425 // Normal Comparisons. 7426 case Intrinsic::ppc_altivec_vcmpbfp: CompareOpc = 966; isDot = 0; break; 7427 case Intrinsic::ppc_altivec_vcmpeqfp: CompareOpc = 198; isDot = 0; break; 7428 case Intrinsic::ppc_altivec_vcmpequb: CompareOpc = 6; isDot = 0; break; 7429 case Intrinsic::ppc_altivec_vcmpequh: CompareOpc = 70; isDot = 0; break; 7430 case Intrinsic::ppc_altivec_vcmpequw: CompareOpc = 134; isDot = 0; break; 7431 case Intrinsic::ppc_altivec_vcmpequd: 7432 if (Subtarget.hasP8Altivec()) { 7433 CompareOpc = 199; 7434 isDot = 0; 7435 } else 7436 return false; 7437 7438 break; 7439 case Intrinsic::ppc_altivec_vcmpgefp: CompareOpc = 454; isDot = 0; break; 7440 case Intrinsic::ppc_altivec_vcmpgtfp: CompareOpc = 710; isDot = 0; break; 7441 case Intrinsic::ppc_altivec_vcmpgtsb: CompareOpc = 774; isDot = 0; break; 7442 case Intrinsic::ppc_altivec_vcmpgtsh: CompareOpc = 838; isDot = 0; break; 7443 case Intrinsic::ppc_altivec_vcmpgtsw: CompareOpc = 902; isDot = 0; break; 7444 case Intrinsic::ppc_altivec_vcmpgtsd: 7445 if (Subtarget.hasP8Altivec()) { 7446 CompareOpc = 967; 7447 isDot = 0; 7448 } else 7449 return false; 7450 7451 break; 7452 case Intrinsic::ppc_altivec_vcmpgtub: CompareOpc = 518; isDot = 0; break; 7453 case Intrinsic::ppc_altivec_vcmpgtuh: CompareOpc = 582; isDot = 0; break; 7454 case Intrinsic::ppc_altivec_vcmpgtuw: CompareOpc = 646; isDot = 0; break; 7455 case Intrinsic::ppc_altivec_vcmpgtud: 7456 if (Subtarget.hasP8Altivec()) { 7457 CompareOpc = 711; 7458 isDot = 0; 7459 } else 7460 return false; 7461 7462 break; 7463 } 7464 return true; 7465 } 7466 7467 /// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom 7468 /// lower, do it, otherwise return null. 7469 SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 7470 SelectionDAG &DAG) const { 7471 // If this is a lowered altivec predicate compare, CompareOpc is set to the 7472 // opcode number of the comparison. 7473 SDLoc dl(Op); 7474 int CompareOpc; 7475 bool isDot; 7476 if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) 7477 return SDValue(); // Don't custom lower most intrinsics. 7478 7479 // If this is a non-dot comparison, make the VCMP node and we are done. 7480 if (!isDot) { 7481 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), 7482 Op.getOperand(1), Op.getOperand(2), 7483 DAG.getConstant(CompareOpc, dl, MVT::i32)); 7484 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); 7485 } 7486 7487 // Create the PPCISD altivec 'dot' comparison node. 7488 SDValue Ops[] = { 7489 Op.getOperand(2), // LHS 7490 Op.getOperand(3), // RHS 7491 DAG.getConstant(CompareOpc, dl, MVT::i32) 7492 }; 7493 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; 7494 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 7495 7496 // Now that we have the comparison, emit a copy from the CR to a GPR. 7497 // This is flagged to the above dot comparison. 7498 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, 7499 DAG.getRegister(PPC::CR6, MVT::i32), 7500 CompNode.getValue(1)); 7501 7502 // Unpack the result based on how the target uses it. 7503 unsigned BitNo; // Bit # of CR6. 7504 bool InvertBit; // Invert result? 7505 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) { 7506 default: // Can't happen, don't crash on invalid number though. 7507 case 0: // Return the value of the EQ bit of CR6. 7508 BitNo = 0; InvertBit = false; 7509 break; 7510 case 1: // Return the inverted value of the EQ bit of CR6. 7511 BitNo = 0; InvertBit = true; 7512 break; 7513 case 2: // Return the value of the LT bit of CR6. 7514 BitNo = 2; InvertBit = false; 7515 break; 7516 case 3: // Return the inverted value of the LT bit of CR6. 7517 BitNo = 2; InvertBit = true; 7518 break; 7519 } 7520 7521 // Shift the bit into the low position. 7522 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, 7523 DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32)); 7524 // Isolate the bit. 7525 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, 7526 DAG.getConstant(1, dl, MVT::i32)); 7527 7528 // If we are supposed to, toggle the bit. 7529 if (InvertBit) 7530 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, 7531 DAG.getConstant(1, dl, MVT::i32)); 7532 return Flags; 7533 } 7534 7535 SDValue PPCTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 7536 SelectionDAG &DAG) const { 7537 SDLoc dl(Op); 7538 // For v2i64 (VSX), we can pattern patch the v2i32 case (using fp <-> int 7539 // instructions), but for smaller types, we need to first extend up to v2i32 7540 // before doing going farther. 7541 if (Op.getValueType() == MVT::v2i64) { 7542 EVT ExtVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 7543 if (ExtVT != MVT::v2i32) { 7544 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)); 7545 Op = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, Op, 7546 DAG.getValueType(EVT::getVectorVT(*DAG.getContext(), 7547 ExtVT.getVectorElementType(), 4))); 7548 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Op); 7549 Op = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v2i64, Op, 7550 DAG.getValueType(MVT::v2i32)); 7551 } 7552 7553 return Op; 7554 } 7555 7556 return SDValue(); 7557 } 7558 7559 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, 7560 SelectionDAG &DAG) const { 7561 SDLoc dl(Op); 7562 // Create a stack slot that is 16-byte aligned. 7563 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 7564 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false); 7565 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7566 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 7567 7568 // Store the input value into Value#0 of the stack slot. 7569 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, 7570 Op.getOperand(0), FIdx, MachinePointerInfo(), 7571 false, false, 0); 7572 // Load it out. 7573 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo(), 7574 false, false, false, 0); 7575 } 7576 7577 SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 7578 SelectionDAG &DAG) const { 7579 SDLoc dl(Op); 7580 SDNode *N = Op.getNode(); 7581 7582 assert(N->getOperand(0).getValueType() == MVT::v4i1 && 7583 "Unknown extract_vector_elt type"); 7584 7585 SDValue Value = N->getOperand(0); 7586 7587 // The first part of this is like the store lowering except that we don't 7588 // need to track the chain. 7589 7590 // The values are now known to be -1 (false) or 1 (true). To convert this 7591 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 7592 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 7593 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 7594 7595 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to 7596 // understand how to form the extending load. 7597 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64); 7598 FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64, 7599 FPHalfs, FPHalfs, FPHalfs, FPHalfs); 7600 7601 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 7602 7603 // Now convert to an integer and store. 7604 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 7605 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), 7606 Value); 7607 7608 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 7609 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false); 7610 MachinePointerInfo PtrInfo = 7611 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 7612 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7613 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 7614 7615 SDValue StoreChain = DAG.getEntryNode(); 7616 SmallVector<SDValue, 2> Ops; 7617 Ops.push_back(StoreChain); 7618 Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32)); 7619 Ops.push_back(Value); 7620 Ops.push_back(FIdx); 7621 7622 SmallVector<EVT, 2> ValueVTs; 7623 ValueVTs.push_back(MVT::Other); // chain 7624 SDVTList VTs = DAG.getVTList(ValueVTs); 7625 7626 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, 7627 dl, VTs, Ops, MVT::v4i32, PtrInfo); 7628 7629 // Extract the value requested. 7630 unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 7631 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 7632 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 7633 7634 SDValue IntVal = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, 7635 PtrInfo.getWithOffset(Offset), 7636 false, false, false, 0); 7637 7638 if (!Subtarget.useCRBits()) 7639 return IntVal; 7640 7641 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal); 7642 } 7643 7644 /// Lowering for QPX v4i1 loads 7645 SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, 7646 SelectionDAG &DAG) const { 7647 SDLoc dl(Op); 7648 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); 7649 SDValue LoadChain = LN->getChain(); 7650 SDValue BasePtr = LN->getBasePtr(); 7651 7652 if (Op.getValueType() == MVT::v4f64 || 7653 Op.getValueType() == MVT::v4f32) { 7654 EVT MemVT = LN->getMemoryVT(); 7655 unsigned Alignment = LN->getAlignment(); 7656 7657 // If this load is properly aligned, then it is legal. 7658 if (Alignment >= MemVT.getStoreSize()) 7659 return Op; 7660 7661 EVT ScalarVT = Op.getValueType().getScalarType(), 7662 ScalarMemVT = MemVT.getScalarType(); 7663 unsigned Stride = ScalarMemVT.getStoreSize(); 7664 7665 SmallVector<SDValue, 8> Vals, LoadChains; 7666 for (unsigned Idx = 0; Idx < 4; ++Idx) { 7667 SDValue Load; 7668 if (ScalarVT != ScalarMemVT) 7669 Load = 7670 DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain, 7671 BasePtr, 7672 LN->getPointerInfo().getWithOffset(Idx*Stride), 7673 ScalarMemVT, LN->isVolatile(), LN->isNonTemporal(), 7674 LN->isInvariant(), MinAlign(Alignment, Idx*Stride), 7675 LN->getAAInfo()); 7676 else 7677 Load = 7678 DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr, 7679 LN->getPointerInfo().getWithOffset(Idx*Stride), 7680 LN->isVolatile(), LN->isNonTemporal(), 7681 LN->isInvariant(), MinAlign(Alignment, Idx*Stride), 7682 LN->getAAInfo()); 7683 7684 if (Idx == 0 && LN->isIndexed()) { 7685 assert(LN->getAddressingMode() == ISD::PRE_INC && 7686 "Unknown addressing mode on vector load"); 7687 Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(), 7688 LN->getAddressingMode()); 7689 } 7690 7691 Vals.push_back(Load); 7692 LoadChains.push_back(Load.getValue(1)); 7693 7694 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 7695 DAG.getConstant(Stride, dl, 7696 BasePtr.getValueType())); 7697 } 7698 7699 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 7700 SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, 7701 Op.getValueType(), Vals); 7702 7703 if (LN->isIndexed()) { 7704 SDValue RetOps[] = { Value, Vals[0].getValue(1), TF }; 7705 return DAG.getMergeValues(RetOps, dl); 7706 } 7707 7708 SDValue RetOps[] = { Value, TF }; 7709 return DAG.getMergeValues(RetOps, dl); 7710 } 7711 7712 assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower"); 7713 assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported"); 7714 7715 // To lower v4i1 from a byte array, we load the byte elements of the 7716 // vector and then reuse the BUILD_VECTOR logic. 7717 7718 SmallVector<SDValue, 4> VectElmts, VectElmtChains; 7719 for (unsigned i = 0; i < 4; ++i) { 7720 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); 7721 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); 7722 7723 VectElmts.push_back(DAG.getExtLoad(ISD::EXTLOAD, 7724 dl, MVT::i32, LoadChain, Idx, 7725 LN->getPointerInfo().getWithOffset(i), 7726 MVT::i8 /* memory type */, 7727 LN->isVolatile(), LN->isNonTemporal(), 7728 LN->isInvariant(), 7729 1 /* alignment */, LN->getAAInfo())); 7730 VectElmtChains.push_back(VectElmts[i].getValue(1)); 7731 } 7732 7733 LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains); 7734 SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i1, VectElmts); 7735 7736 SDValue RVals[] = { Value, LoadChain }; 7737 return DAG.getMergeValues(RVals, dl); 7738 } 7739 7740 /// Lowering for QPX v4i1 stores 7741 SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, 7742 SelectionDAG &DAG) const { 7743 SDLoc dl(Op); 7744 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); 7745 SDValue StoreChain = SN->getChain(); 7746 SDValue BasePtr = SN->getBasePtr(); 7747 SDValue Value = SN->getValue(); 7748 7749 if (Value.getValueType() == MVT::v4f64 || 7750 Value.getValueType() == MVT::v4f32) { 7751 EVT MemVT = SN->getMemoryVT(); 7752 unsigned Alignment = SN->getAlignment(); 7753 7754 // If this store is properly aligned, then it is legal. 7755 if (Alignment >= MemVT.getStoreSize()) 7756 return Op; 7757 7758 EVT ScalarVT = Value.getValueType().getScalarType(), 7759 ScalarMemVT = MemVT.getScalarType(); 7760 unsigned Stride = ScalarMemVT.getStoreSize(); 7761 7762 SmallVector<SDValue, 8> Stores; 7763 for (unsigned Idx = 0; Idx < 4; ++Idx) { 7764 SDValue Ex = DAG.getNode( 7765 ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value, 7766 DAG.getConstant(Idx, dl, getVectorIdxTy(DAG.getDataLayout()))); 7767 SDValue Store; 7768 if (ScalarVT != ScalarMemVT) 7769 Store = 7770 DAG.getTruncStore(StoreChain, dl, Ex, BasePtr, 7771 SN->getPointerInfo().getWithOffset(Idx*Stride), 7772 ScalarMemVT, SN->isVolatile(), SN->isNonTemporal(), 7773 MinAlign(Alignment, Idx*Stride), SN->getAAInfo()); 7774 else 7775 Store = 7776 DAG.getStore(StoreChain, dl, Ex, BasePtr, 7777 SN->getPointerInfo().getWithOffset(Idx*Stride), 7778 SN->isVolatile(), SN->isNonTemporal(), 7779 MinAlign(Alignment, Idx*Stride), SN->getAAInfo()); 7780 7781 if (Idx == 0 && SN->isIndexed()) { 7782 assert(SN->getAddressingMode() == ISD::PRE_INC && 7783 "Unknown addressing mode on vector store"); 7784 Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(), 7785 SN->getAddressingMode()); 7786 } 7787 7788 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 7789 DAG.getConstant(Stride, dl, 7790 BasePtr.getValueType())); 7791 Stores.push_back(Store); 7792 } 7793 7794 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 7795 7796 if (SN->isIndexed()) { 7797 SDValue RetOps[] = { TF, Stores[0].getValue(1) }; 7798 return DAG.getMergeValues(RetOps, dl); 7799 } 7800 7801 return TF; 7802 } 7803 7804 assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported"); 7805 assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower"); 7806 7807 // The values are now known to be -1 (false) or 1 (true). To convert this 7808 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 7809 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 7810 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 7811 7812 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to 7813 // understand how to form the extending load. 7814 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64); 7815 FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64, 7816 FPHalfs, FPHalfs, FPHalfs, FPHalfs); 7817 7818 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 7819 7820 // Now convert to an integer and store. 7821 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 7822 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), 7823 Value); 7824 7825 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 7826 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false); 7827 MachinePointerInfo PtrInfo = 7828 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 7829 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7830 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 7831 7832 SmallVector<SDValue, 2> Ops; 7833 Ops.push_back(StoreChain); 7834 Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32)); 7835 Ops.push_back(Value); 7836 Ops.push_back(FIdx); 7837 7838 SmallVector<EVT, 2> ValueVTs; 7839 ValueVTs.push_back(MVT::Other); // chain 7840 SDVTList VTs = DAG.getVTList(ValueVTs); 7841 7842 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, 7843 dl, VTs, Ops, MVT::v4i32, PtrInfo); 7844 7845 // Move data into the byte array. 7846 SmallVector<SDValue, 4> Loads, LoadChains; 7847 for (unsigned i = 0; i < 4; ++i) { 7848 unsigned Offset = 4*i; 7849 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 7850 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 7851 7852 Loads.push_back(DAG.getLoad(MVT::i32, dl, StoreChain, Idx, 7853 PtrInfo.getWithOffset(Offset), 7854 false, false, false, 0)); 7855 LoadChains.push_back(Loads[i].getValue(1)); 7856 } 7857 7858 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 7859 7860 SmallVector<SDValue, 4> Stores; 7861 for (unsigned i = 0; i < 4; ++i) { 7862 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); 7863 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); 7864 7865 Stores.push_back(DAG.getTruncStore( 7866 StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i), 7867 MVT::i8 /* memory type */, SN->isNonTemporal(), SN->isVolatile(), 7868 1 /* alignment */, SN->getAAInfo())); 7869 } 7870 7871 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 7872 7873 return StoreChain; 7874 } 7875 7876 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { 7877 SDLoc dl(Op); 7878 if (Op.getValueType() == MVT::v4i32) { 7879 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 7880 7881 SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl); 7882 SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt. 7883 7884 SDValue RHSSwap = // = vrlw RHS, 16 7885 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); 7886 7887 // Shrinkify inputs to v8i16. 7888 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); 7889 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); 7890 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); 7891 7892 // Low parts multiplied together, generating 32-bit results (we ignore the 7893 // top parts). 7894 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, 7895 LHS, RHS, DAG, dl, MVT::v4i32); 7896 7897 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, 7898 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); 7899 // Shift the high parts up 16 bits. 7900 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, 7901 Neg16, DAG, dl); 7902 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); 7903 } else if (Op.getValueType() == MVT::v8i16) { 7904 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 7905 7906 SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl); 7907 7908 return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm, 7909 LHS, RHS, Zero, DAG, dl); 7910 } else if (Op.getValueType() == MVT::v16i8) { 7911 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 7912 bool isLittleEndian = Subtarget.isLittleEndian(); 7913 7914 // Multiply the even 8-bit parts, producing 16-bit sums. 7915 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, 7916 LHS, RHS, DAG, dl, MVT::v8i16); 7917 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); 7918 7919 // Multiply the odd 8-bit parts, producing 16-bit sums. 7920 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, 7921 LHS, RHS, DAG, dl, MVT::v8i16); 7922 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); 7923 7924 // Merge the results together. Because vmuleub and vmuloub are 7925 // instructions with a big-endian bias, we must reverse the 7926 // element numbering and reverse the meaning of "odd" and "even" 7927 // when generating little endian code. 7928 int Ops[16]; 7929 for (unsigned i = 0; i != 8; ++i) { 7930 if (isLittleEndian) { 7931 Ops[i*2 ] = 2*i; 7932 Ops[i*2+1] = 2*i+16; 7933 } else { 7934 Ops[i*2 ] = 2*i+1; 7935 Ops[i*2+1] = 2*i+1+16; 7936 } 7937 } 7938 if (isLittleEndian) 7939 return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops); 7940 else 7941 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); 7942 } else { 7943 llvm_unreachable("Unknown mul to lower!"); 7944 } 7945 } 7946 7947 /// LowerOperation - Provide custom lowering hooks for some operations. 7948 /// 7949 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 7950 switch (Op.getOpcode()) { 7951 default: llvm_unreachable("Wasn't expecting to be able to lower this!"); 7952 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 7953 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 7954 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 7955 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 7956 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 7957 case ISD::SETCC: return LowerSETCC(Op, DAG); 7958 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); 7959 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); 7960 case ISD::VASTART: 7961 return LowerVASTART(Op, DAG, Subtarget); 7962 7963 case ISD::VAARG: 7964 return LowerVAARG(Op, DAG, Subtarget); 7965 7966 case ISD::VACOPY: 7967 return LowerVACOPY(Op, DAG, Subtarget); 7968 7969 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG, Subtarget); 7970 case ISD::DYNAMIC_STACKALLOC: 7971 return LowerDYNAMIC_STACKALLOC(Op, DAG, Subtarget); 7972 case ISD::GET_DYNAMIC_AREA_OFFSET: return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG, Subtarget); 7973 7974 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); 7975 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); 7976 7977 case ISD::LOAD: return LowerLOAD(Op, DAG); 7978 case ISD::STORE: return LowerSTORE(Op, DAG); 7979 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); 7980 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 7981 case ISD::FP_TO_UINT: 7982 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, 7983 SDLoc(Op)); 7984 case ISD::UINT_TO_FP: 7985 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 7986 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 7987 7988 // Lower 64-bit shifts. 7989 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); 7990 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); 7991 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); 7992 7993 // Vector-related lowering. 7994 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 7995 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 7996 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 7997 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 7998 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 7999 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 8000 case ISD::MUL: return LowerMUL(Op, DAG); 8001 8002 // For counter-based loop handling. 8003 case ISD::INTRINSIC_W_CHAIN: return SDValue(); 8004 8005 // Frame & Return address. 8006 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 8007 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 8008 } 8009 } 8010 8011 void PPCTargetLowering::ReplaceNodeResults(SDNode *N, 8012 SmallVectorImpl<SDValue>&Results, 8013 SelectionDAG &DAG) const { 8014 SDLoc dl(N); 8015 switch (N->getOpcode()) { 8016 default: 8017 llvm_unreachable("Do not know how to custom type legalize this operation!"); 8018 case ISD::READCYCLECOUNTER: { 8019 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 8020 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0)); 8021 8022 Results.push_back(RTB); 8023 Results.push_back(RTB.getValue(1)); 8024 Results.push_back(RTB.getValue(2)); 8025 break; 8026 } 8027 case ISD::INTRINSIC_W_CHAIN: { 8028 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 8029 Intrinsic::ppc_is_decremented_ctr_nonzero) 8030 break; 8031 8032 assert(N->getValueType(0) == MVT::i1 && 8033 "Unexpected result type for CTR decrement intrinsic"); 8034 EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 8035 N->getValueType(0)); 8036 SDVTList VTs = DAG.getVTList(SVT, MVT::Other); 8037 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), 8038 N->getOperand(1)); 8039 8040 Results.push_back(NewInt); 8041 Results.push_back(NewInt.getValue(1)); 8042 break; 8043 } 8044 case ISD::VAARG: { 8045 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64()) 8046 return; 8047 8048 EVT VT = N->getValueType(0); 8049 8050 if (VT == MVT::i64) { 8051 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG, Subtarget); 8052 8053 Results.push_back(NewNode); 8054 Results.push_back(NewNode.getValue(1)); 8055 } 8056 return; 8057 } 8058 case ISD::FP_ROUND_INREG: { 8059 assert(N->getValueType(0) == MVT::ppcf128); 8060 assert(N->getOperand(0).getValueType() == MVT::ppcf128); 8061 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 8062 MVT::f64, N->getOperand(0), 8063 DAG.getIntPtrConstant(0, dl)); 8064 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 8065 MVT::f64, N->getOperand(0), 8066 DAG.getIntPtrConstant(1, dl)); 8067 8068 // Add the two halves of the long double in round-to-zero mode. 8069 SDValue FPreg = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); 8070 8071 // We know the low half is about to be thrown away, so just use something 8072 // convenient. 8073 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::ppcf128, 8074 FPreg, FPreg)); 8075 return; 8076 } 8077 case ISD::FP_TO_SINT: 8078 case ISD::FP_TO_UINT: 8079 // LowerFP_TO_INT() can only handle f32 and f64. 8080 if (N->getOperand(0).getValueType() == MVT::ppcf128) 8081 return; 8082 Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); 8083 return; 8084 } 8085 } 8086 8087 //===----------------------------------------------------------------------===// 8088 // Other Lowering Code 8089 //===----------------------------------------------------------------------===// 8090 8091 static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { 8092 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 8093 Function *Func = Intrinsic::getDeclaration(M, Id); 8094 return Builder.CreateCall(Func, {}); 8095 } 8096 8097 // The mappings for emitLeading/TrailingFence is taken from 8098 // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html 8099 Instruction* PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, 8100 AtomicOrdering Ord, bool IsStore, 8101 bool IsLoad) const { 8102 if (Ord == SequentiallyConsistent) 8103 return callIntrinsic(Builder, Intrinsic::ppc_sync); 8104 if (isAtLeastRelease(Ord)) 8105 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 8106 return nullptr; 8107 } 8108 8109 Instruction* PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder, 8110 AtomicOrdering Ord, bool IsStore, 8111 bool IsLoad) const { 8112 if (IsLoad && isAtLeastAcquire(Ord)) 8113 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 8114 // FIXME: this is too conservative, a dependent branch + isync is enough. 8115 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and 8116 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html 8117 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification. 8118 return nullptr; 8119 } 8120 8121 MachineBasicBlock * 8122 PPCTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB, 8123 unsigned AtomicSize, 8124 unsigned BinOpcode) const { 8125 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 8126 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 8127 8128 auto LoadMnemonic = PPC::LDARX; 8129 auto StoreMnemonic = PPC::STDCX; 8130 switch (AtomicSize) { 8131 default: 8132 llvm_unreachable("Unexpected size of atomic entity"); 8133 case 1: 8134 LoadMnemonic = PPC::LBARX; 8135 StoreMnemonic = PPC::STBCX; 8136 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 8137 break; 8138 case 2: 8139 LoadMnemonic = PPC::LHARX; 8140 StoreMnemonic = PPC::STHCX; 8141 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 8142 break; 8143 case 4: 8144 LoadMnemonic = PPC::LWARX; 8145 StoreMnemonic = PPC::STWCX; 8146 break; 8147 case 8: 8148 LoadMnemonic = PPC::LDARX; 8149 StoreMnemonic = PPC::STDCX; 8150 break; 8151 } 8152 8153 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 8154 MachineFunction *F = BB->getParent(); 8155 MachineFunction::iterator It = ++BB->getIterator(); 8156 8157 unsigned dest = MI->getOperand(0).getReg(); 8158 unsigned ptrA = MI->getOperand(1).getReg(); 8159 unsigned ptrB = MI->getOperand(2).getReg(); 8160 unsigned incr = MI->getOperand(3).getReg(); 8161 DebugLoc dl = MI->getDebugLoc(); 8162 8163 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 8164 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 8165 F->insert(It, loopMBB); 8166 F->insert(It, exitMBB); 8167 exitMBB->splice(exitMBB->begin(), BB, 8168 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 8169 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 8170 8171 MachineRegisterInfo &RegInfo = F->getRegInfo(); 8172 unsigned TmpReg = (!BinOpcode) ? incr : 8173 RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass 8174 : &PPC::GPRCRegClass); 8175 8176 // thisMBB: 8177 // ... 8178 // fallthrough --> loopMBB 8179 BB->addSuccessor(loopMBB); 8180 8181 // loopMBB: 8182 // l[wd]arx dest, ptr 8183 // add r0, dest, incr 8184 // st[wd]cx. r0, ptr 8185 // bne- loopMBB 8186 // fallthrough --> exitMBB 8187 BB = loopMBB; 8188 BuildMI(BB, dl, TII->get(LoadMnemonic), dest) 8189 .addReg(ptrA).addReg(ptrB); 8190 if (BinOpcode) 8191 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); 8192 BuildMI(BB, dl, TII->get(StoreMnemonic)) 8193 .addReg(TmpReg).addReg(ptrA).addReg(ptrB); 8194 BuildMI(BB, dl, TII->get(PPC::BCC)) 8195 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 8196 BB->addSuccessor(loopMBB); 8197 BB->addSuccessor(exitMBB); 8198 8199 // exitMBB: 8200 // ... 8201 BB = exitMBB; 8202 return BB; 8203 } 8204 8205 MachineBasicBlock * 8206 PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr *MI, 8207 MachineBasicBlock *BB, 8208 bool is8bit, // operation 8209 unsigned BinOpcode) const { 8210 // If we support part-word atomic mnemonics, just use them 8211 if (Subtarget.hasPartwordAtomics()) 8212 return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode); 8213 8214 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 8215 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 8216 // In 64 bit mode we have to use 64 bits for addresses, even though the 8217 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address 8218 // registers without caring whether they're 32 or 64, but here we're 8219 // doing actual arithmetic on the addresses. 8220 bool is64bit = Subtarget.isPPC64(); 8221 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 8222 8223 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 8224 MachineFunction *F = BB->getParent(); 8225 MachineFunction::iterator It = ++BB->getIterator(); 8226 8227 unsigned dest = MI->getOperand(0).getReg(); 8228 unsigned ptrA = MI->getOperand(1).getReg(); 8229 unsigned ptrB = MI->getOperand(2).getReg(); 8230 unsigned incr = MI->getOperand(3).getReg(); 8231 DebugLoc dl = MI->getDebugLoc(); 8232 8233 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 8234 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 8235 F->insert(It, loopMBB); 8236 F->insert(It, exitMBB); 8237 exitMBB->splice(exitMBB->begin(), BB, 8238 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 8239 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 8240 8241 MachineRegisterInfo &RegInfo = F->getRegInfo(); 8242 const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass 8243 : &PPC::GPRCRegClass; 8244 unsigned PtrReg = RegInfo.createVirtualRegister(RC); 8245 unsigned Shift1Reg = RegInfo.createVirtualRegister(RC); 8246 unsigned ShiftReg = RegInfo.createVirtualRegister(RC); 8247 unsigned Incr2Reg = RegInfo.createVirtualRegister(RC); 8248 unsigned MaskReg = RegInfo.createVirtualRegister(RC); 8249 unsigned Mask2Reg = RegInfo.createVirtualRegister(RC); 8250 unsigned Mask3Reg = RegInfo.createVirtualRegister(RC); 8251 unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC); 8252 unsigned Tmp3Reg = RegInfo.createVirtualRegister(RC); 8253 unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC); 8254 unsigned TmpDestReg = RegInfo.createVirtualRegister(RC); 8255 unsigned Ptr1Reg; 8256 unsigned TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RC); 8257 8258 // thisMBB: 8259 // ... 8260 // fallthrough --> loopMBB 8261 BB->addSuccessor(loopMBB); 8262 8263 // The 4-byte load must be aligned, while a char or short may be 8264 // anywhere in the word. Hence all this nasty bookkeeping code. 8265 // add ptr1, ptrA, ptrB [copy if ptrA==0] 8266 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 8267 // xori shift, shift1, 24 [16] 8268 // rlwinm ptr, ptr1, 0, 0, 29 8269 // slw incr2, incr, shift 8270 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 8271 // slw mask, mask2, shift 8272 // loopMBB: 8273 // lwarx tmpDest, ptr 8274 // add tmp, tmpDest, incr2 8275 // andc tmp2, tmpDest, mask 8276 // and tmp3, tmp, mask 8277 // or tmp4, tmp3, tmp2 8278 // stwcx. tmp4, ptr 8279 // bne- loopMBB 8280 // fallthrough --> exitMBB 8281 // srw dest, tmpDest, shift 8282 if (ptrA != ZeroReg) { 8283 Ptr1Reg = RegInfo.createVirtualRegister(RC); 8284 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 8285 .addReg(ptrA).addReg(ptrB); 8286 } else { 8287 Ptr1Reg = ptrB; 8288 } 8289 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg) 8290 .addImm(3).addImm(27).addImm(is8bit ? 28 : 27); 8291 BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg) 8292 .addReg(Shift1Reg).addImm(is8bit ? 24 : 16); 8293 if (is64bit) 8294 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 8295 .addReg(Ptr1Reg).addImm(0).addImm(61); 8296 else 8297 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 8298 .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29); 8299 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg) 8300 .addReg(incr).addReg(ShiftReg); 8301 if (is8bit) 8302 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 8303 else { 8304 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 8305 BuildMI(BB, dl, TII->get(PPC::ORI),Mask2Reg).addReg(Mask3Reg).addImm(65535); 8306 } 8307 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 8308 .addReg(Mask2Reg).addReg(ShiftReg); 8309 8310 BB = loopMBB; 8311 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 8312 .addReg(ZeroReg).addReg(PtrReg); 8313 if (BinOpcode) 8314 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) 8315 .addReg(Incr2Reg).addReg(TmpDestReg); 8316 BuildMI(BB, dl, TII->get(is64bit ? PPC::ANDC8 : PPC::ANDC), Tmp2Reg) 8317 .addReg(TmpDestReg).addReg(MaskReg); 8318 BuildMI(BB, dl, TII->get(is64bit ? PPC::AND8 : PPC::AND), Tmp3Reg) 8319 .addReg(TmpReg).addReg(MaskReg); 8320 BuildMI(BB, dl, TII->get(is64bit ? PPC::OR8 : PPC::OR), Tmp4Reg) 8321 .addReg(Tmp3Reg).addReg(Tmp2Reg); 8322 BuildMI(BB, dl, TII->get(PPC::STWCX)) 8323 .addReg(Tmp4Reg).addReg(ZeroReg).addReg(PtrReg); 8324 BuildMI(BB, dl, TII->get(PPC::BCC)) 8325 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 8326 BB->addSuccessor(loopMBB); 8327 BB->addSuccessor(exitMBB); 8328 8329 // exitMBB: 8330 // ... 8331 BB = exitMBB; 8332 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest).addReg(TmpDestReg) 8333 .addReg(ShiftReg); 8334 return BB; 8335 } 8336 8337 llvm::MachineBasicBlock* 8338 PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr *MI, 8339 MachineBasicBlock *MBB) const { 8340 DebugLoc DL = MI->getDebugLoc(); 8341 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 8342 8343 MachineFunction *MF = MBB->getParent(); 8344 MachineRegisterInfo &MRI = MF->getRegInfo(); 8345 8346 const BasicBlock *BB = MBB->getBasicBlock(); 8347 MachineFunction::iterator I = ++MBB->getIterator(); 8348 8349 // Memory Reference 8350 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin(); 8351 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end(); 8352 8353 unsigned DstReg = MI->getOperand(0).getReg(); 8354 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 8355 assert(RC->hasType(MVT::i32) && "Invalid destination!"); 8356 unsigned mainDstReg = MRI.createVirtualRegister(RC); 8357 unsigned restoreDstReg = MRI.createVirtualRegister(RC); 8358 8359 MVT PVT = getPointerTy(MF->getDataLayout()); 8360 assert((PVT == MVT::i64 || PVT == MVT::i32) && 8361 "Invalid Pointer Size!"); 8362 // For v = setjmp(buf), we generate 8363 // 8364 // thisMBB: 8365 // SjLjSetup mainMBB 8366 // bl mainMBB 8367 // v_restore = 1 8368 // b sinkMBB 8369 // 8370 // mainMBB: 8371 // buf[LabelOffset] = LR 8372 // v_main = 0 8373 // 8374 // sinkMBB: 8375 // v = phi(main, restore) 8376 // 8377 8378 MachineBasicBlock *thisMBB = MBB; 8379 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); 8380 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); 8381 MF->insert(I, mainMBB); 8382 MF->insert(I, sinkMBB); 8383 8384 MachineInstrBuilder MIB; 8385 8386 // Transfer the remainder of BB and its successor edges to sinkMBB. 8387 sinkMBB->splice(sinkMBB->begin(), MBB, 8388 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 8389 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); 8390 8391 // Note that the structure of the jmp_buf used here is not compatible 8392 // with that used by libc, and is not designed to be. Specifically, it 8393 // stores only those 'reserved' registers that LLVM does not otherwise 8394 // understand how to spill. Also, by convention, by the time this 8395 // intrinsic is called, Clang has already stored the frame address in the 8396 // first slot of the buffer and stack address in the third. Following the 8397 // X86 target code, we'll store the jump address in the second slot. We also 8398 // need to save the TOC pointer (R2) to handle jumps between shared 8399 // libraries, and that will be stored in the fourth slot. The thread 8400 // identifier (R13) is not affected. 8401 8402 // thisMBB: 8403 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 8404 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 8405 const int64_t BPOffset = 4 * PVT.getStoreSize(); 8406 8407 // Prepare IP either in reg. 8408 const TargetRegisterClass *PtrRC = getRegClassFor(PVT); 8409 unsigned LabelReg = MRI.createVirtualRegister(PtrRC); 8410 unsigned BufReg = MI->getOperand(1).getReg(); 8411 8412 if (Subtarget.isPPC64() && Subtarget.isSVR4ABI()) { 8413 setUsesTOCBasePtr(*MBB->getParent()); 8414 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) 8415 .addReg(PPC::X2) 8416 .addImm(TOCOffset) 8417 .addReg(BufReg); 8418 MIB.setMemRefs(MMOBegin, MMOEnd); 8419 } 8420 8421 // Naked functions never have a base pointer, and so we use r1. For all 8422 // other functions, this decision must be delayed until during PEI. 8423 unsigned BaseReg; 8424 if (MF->getFunction()->hasFnAttribute(Attribute::Naked)) 8425 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; 8426 else 8427 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP; 8428 8429 MIB = BuildMI(*thisMBB, MI, DL, 8430 TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW)) 8431 .addReg(BaseReg) 8432 .addImm(BPOffset) 8433 .addReg(BufReg); 8434 MIB.setMemRefs(MMOBegin, MMOEnd); 8435 8436 // Setup 8437 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); 8438 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 8439 MIB.addRegMask(TRI->getNoPreservedMask()); 8440 8441 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); 8442 8443 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) 8444 .addMBB(mainMBB); 8445 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); 8446 8447 thisMBB->addSuccessor(mainMBB, BranchProbability::getZero()); 8448 thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne()); 8449 8450 // mainMBB: 8451 // mainDstReg = 0 8452 MIB = 8453 BuildMI(mainMBB, DL, 8454 TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); 8455 8456 // Store IP 8457 if (Subtarget.isPPC64()) { 8458 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) 8459 .addReg(LabelReg) 8460 .addImm(LabelOffset) 8461 .addReg(BufReg); 8462 } else { 8463 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) 8464 .addReg(LabelReg) 8465 .addImm(LabelOffset) 8466 .addReg(BufReg); 8467 } 8468 8469 MIB.setMemRefs(MMOBegin, MMOEnd); 8470 8471 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); 8472 mainMBB->addSuccessor(sinkMBB); 8473 8474 // sinkMBB: 8475 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 8476 TII->get(PPC::PHI), DstReg) 8477 .addReg(mainDstReg).addMBB(mainMBB) 8478 .addReg(restoreDstReg).addMBB(thisMBB); 8479 8480 MI->eraseFromParent(); 8481 return sinkMBB; 8482 } 8483 8484 MachineBasicBlock * 8485 PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr *MI, 8486 MachineBasicBlock *MBB) const { 8487 DebugLoc DL = MI->getDebugLoc(); 8488 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 8489 8490 MachineFunction *MF = MBB->getParent(); 8491 MachineRegisterInfo &MRI = MF->getRegInfo(); 8492 8493 // Memory Reference 8494 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin(); 8495 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end(); 8496 8497 MVT PVT = getPointerTy(MF->getDataLayout()); 8498 assert((PVT == MVT::i64 || PVT == MVT::i32) && 8499 "Invalid Pointer Size!"); 8500 8501 const TargetRegisterClass *RC = 8502 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 8503 unsigned Tmp = MRI.createVirtualRegister(RC); 8504 // Since FP is only updated here but NOT referenced, it's treated as GPR. 8505 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; 8506 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; 8507 unsigned BP = 8508 (PVT == MVT::i64) 8509 ? PPC::X30 8510 : (Subtarget.isSVR4ABI() && 8511 MF->getTarget().getRelocationModel() == Reloc::PIC_ 8512 ? PPC::R29 8513 : PPC::R30); 8514 8515 MachineInstrBuilder MIB; 8516 8517 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 8518 const int64_t SPOffset = 2 * PVT.getStoreSize(); 8519 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 8520 const int64_t BPOffset = 4 * PVT.getStoreSize(); 8521 8522 unsigned BufReg = MI->getOperand(0).getReg(); 8523 8524 // Reload FP (the jumped-to function may not have had a 8525 // frame pointer, and if so, then its r31 will be restored 8526 // as necessary). 8527 if (PVT == MVT::i64) { 8528 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) 8529 .addImm(0) 8530 .addReg(BufReg); 8531 } else { 8532 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) 8533 .addImm(0) 8534 .addReg(BufReg); 8535 } 8536 MIB.setMemRefs(MMOBegin, MMOEnd); 8537 8538 // Reload IP 8539 if (PVT == MVT::i64) { 8540 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) 8541 .addImm(LabelOffset) 8542 .addReg(BufReg); 8543 } else { 8544 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) 8545 .addImm(LabelOffset) 8546 .addReg(BufReg); 8547 } 8548 MIB.setMemRefs(MMOBegin, MMOEnd); 8549 8550 // Reload SP 8551 if (PVT == MVT::i64) { 8552 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) 8553 .addImm(SPOffset) 8554 .addReg(BufReg); 8555 } else { 8556 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) 8557 .addImm(SPOffset) 8558 .addReg(BufReg); 8559 } 8560 MIB.setMemRefs(MMOBegin, MMOEnd); 8561 8562 // Reload BP 8563 if (PVT == MVT::i64) { 8564 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) 8565 .addImm(BPOffset) 8566 .addReg(BufReg); 8567 } else { 8568 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) 8569 .addImm(BPOffset) 8570 .addReg(BufReg); 8571 } 8572 MIB.setMemRefs(MMOBegin, MMOEnd); 8573 8574 // Reload TOC 8575 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) { 8576 setUsesTOCBasePtr(*MBB->getParent()); 8577 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) 8578 .addImm(TOCOffset) 8579 .addReg(BufReg); 8580 8581 MIB.setMemRefs(MMOBegin, MMOEnd); 8582 } 8583 8584 // Jump 8585 BuildMI(*MBB, MI, DL, 8586 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); 8587 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); 8588 8589 MI->eraseFromParent(); 8590 return MBB; 8591 } 8592 8593 MachineBasicBlock * 8594 PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 8595 MachineBasicBlock *BB) const { 8596 if (MI->getOpcode() == TargetOpcode::STACKMAP || 8597 MI->getOpcode() == TargetOpcode::PATCHPOINT) { 8598 if (Subtarget.isPPC64() && Subtarget.isSVR4ABI() && 8599 MI->getOpcode() == TargetOpcode::PATCHPOINT) { 8600 // Call lowering should have added an r2 operand to indicate a dependence 8601 // on the TOC base pointer value. It can't however, because there is no 8602 // way to mark the dependence as implicit there, and so the stackmap code 8603 // will confuse it with a regular operand. Instead, add the dependence 8604 // here. 8605 setUsesTOCBasePtr(*BB->getParent()); 8606 MI->addOperand(MachineOperand::CreateReg(PPC::X2, false, true)); 8607 } 8608 8609 return emitPatchPoint(MI, BB); 8610 } 8611 8612 if (MI->getOpcode() == PPC::EH_SjLj_SetJmp32 || 8613 MI->getOpcode() == PPC::EH_SjLj_SetJmp64) { 8614 return emitEHSjLjSetJmp(MI, BB); 8615 } else if (MI->getOpcode() == PPC::EH_SjLj_LongJmp32 || 8616 MI->getOpcode() == PPC::EH_SjLj_LongJmp64) { 8617 return emitEHSjLjLongJmp(MI, BB); 8618 } 8619 8620 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 8621 8622 // To "insert" these instructions we actually have to insert their 8623 // control-flow patterns. 8624 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 8625 MachineFunction::iterator It = ++BB->getIterator(); 8626 8627 MachineFunction *F = BB->getParent(); 8628 8629 if (Subtarget.hasISEL() && (MI->getOpcode() == PPC::SELECT_CC_I4 || 8630 MI->getOpcode() == PPC::SELECT_CC_I8 || 8631 MI->getOpcode() == PPC::SELECT_I4 || 8632 MI->getOpcode() == PPC::SELECT_I8)) { 8633 SmallVector<MachineOperand, 2> Cond; 8634 if (MI->getOpcode() == PPC::SELECT_CC_I4 || 8635 MI->getOpcode() == PPC::SELECT_CC_I8) 8636 Cond.push_back(MI->getOperand(4)); 8637 else 8638 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 8639 Cond.push_back(MI->getOperand(1)); 8640 8641 DebugLoc dl = MI->getDebugLoc(); 8642 TII->insertSelect(*BB, MI, dl, MI->getOperand(0).getReg(), 8643 Cond, MI->getOperand(2).getReg(), 8644 MI->getOperand(3).getReg()); 8645 } else if (MI->getOpcode() == PPC::SELECT_CC_I4 || 8646 MI->getOpcode() == PPC::SELECT_CC_I8 || 8647 MI->getOpcode() == PPC::SELECT_CC_F4 || 8648 MI->getOpcode() == PPC::SELECT_CC_F8 || 8649 MI->getOpcode() == PPC::SELECT_CC_QFRC || 8650 MI->getOpcode() == PPC::SELECT_CC_QSRC || 8651 MI->getOpcode() == PPC::SELECT_CC_QBRC || 8652 MI->getOpcode() == PPC::SELECT_CC_VRRC || 8653 MI->getOpcode() == PPC::SELECT_CC_VSFRC || 8654 MI->getOpcode() == PPC::SELECT_CC_VSSRC || 8655 MI->getOpcode() == PPC::SELECT_CC_VSRC || 8656 MI->getOpcode() == PPC::SELECT_I4 || 8657 MI->getOpcode() == PPC::SELECT_I8 || 8658 MI->getOpcode() == PPC::SELECT_F4 || 8659 MI->getOpcode() == PPC::SELECT_F8 || 8660 MI->getOpcode() == PPC::SELECT_QFRC || 8661 MI->getOpcode() == PPC::SELECT_QSRC || 8662 MI->getOpcode() == PPC::SELECT_QBRC || 8663 MI->getOpcode() == PPC::SELECT_VRRC || 8664 MI->getOpcode() == PPC::SELECT_VSFRC || 8665 MI->getOpcode() == PPC::SELECT_VSSRC || 8666 MI->getOpcode() == PPC::SELECT_VSRC) { 8667 // The incoming instruction knows the destination vreg to set, the 8668 // condition code register to branch on, the true/false values to 8669 // select between, and a branch opcode to use. 8670 8671 // thisMBB: 8672 // ... 8673 // TrueVal = ... 8674 // cmpTY ccX, r1, r2 8675 // bCC copy1MBB 8676 // fallthrough --> copy0MBB 8677 MachineBasicBlock *thisMBB = BB; 8678 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 8679 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 8680 DebugLoc dl = MI->getDebugLoc(); 8681 F->insert(It, copy0MBB); 8682 F->insert(It, sinkMBB); 8683 8684 // Transfer the remainder of BB and its successor edges to sinkMBB. 8685 sinkMBB->splice(sinkMBB->begin(), BB, 8686 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 8687 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 8688 8689 // Next, add the true and fallthrough blocks as its successors. 8690 BB->addSuccessor(copy0MBB); 8691 BB->addSuccessor(sinkMBB); 8692 8693 if (MI->getOpcode() == PPC::SELECT_I4 || 8694 MI->getOpcode() == PPC::SELECT_I8 || 8695 MI->getOpcode() == PPC::SELECT_F4 || 8696 MI->getOpcode() == PPC::SELECT_F8 || 8697 MI->getOpcode() == PPC::SELECT_QFRC || 8698 MI->getOpcode() == PPC::SELECT_QSRC || 8699 MI->getOpcode() == PPC::SELECT_QBRC || 8700 MI->getOpcode() == PPC::SELECT_VRRC || 8701 MI->getOpcode() == PPC::SELECT_VSFRC || 8702 MI->getOpcode() == PPC::SELECT_VSSRC || 8703 MI->getOpcode() == PPC::SELECT_VSRC) { 8704 BuildMI(BB, dl, TII->get(PPC::BC)) 8705 .addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB); 8706 } else { 8707 unsigned SelectPred = MI->getOperand(4).getImm(); 8708 BuildMI(BB, dl, TII->get(PPC::BCC)) 8709 .addImm(SelectPred).addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB); 8710 } 8711 8712 // copy0MBB: 8713 // %FalseValue = ... 8714 // # fallthrough to sinkMBB 8715 BB = copy0MBB; 8716 8717 // Update machine-CFG edges 8718 BB->addSuccessor(sinkMBB); 8719 8720 // sinkMBB: 8721 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 8722 // ... 8723 BB = sinkMBB; 8724 BuildMI(*BB, BB->begin(), dl, 8725 TII->get(PPC::PHI), MI->getOperand(0).getReg()) 8726 .addReg(MI->getOperand(3).getReg()).addMBB(copy0MBB) 8727 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 8728 } else if (MI->getOpcode() == PPC::ReadTB) { 8729 // To read the 64-bit time-base register on a 32-bit target, we read the 8730 // two halves. Should the counter have wrapped while it was being read, we 8731 // need to try again. 8732 // ... 8733 // readLoop: 8734 // mfspr Rx,TBU # load from TBU 8735 // mfspr Ry,TB # load from TB 8736 // mfspr Rz,TBU # load from TBU 8737 // cmpw crX,Rx,Rz # check if 'old'='new' 8738 // bne readLoop # branch if they're not equal 8739 // ... 8740 8741 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB); 8742 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 8743 DebugLoc dl = MI->getDebugLoc(); 8744 F->insert(It, readMBB); 8745 F->insert(It, sinkMBB); 8746 8747 // Transfer the remainder of BB and its successor edges to sinkMBB. 8748 sinkMBB->splice(sinkMBB->begin(), BB, 8749 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 8750 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 8751 8752 BB->addSuccessor(readMBB); 8753 BB = readMBB; 8754 8755 MachineRegisterInfo &RegInfo = F->getRegInfo(); 8756 unsigned ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 8757 unsigned LoReg = MI->getOperand(0).getReg(); 8758 unsigned HiReg = MI->getOperand(1).getReg(); 8759 8760 BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269); 8761 BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268); 8762 BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269); 8763 8764 unsigned CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 8765 8766 BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg) 8767 .addReg(HiReg).addReg(ReadAgainReg); 8768 BuildMI(BB, dl, TII->get(PPC::BCC)) 8769 .addImm(PPC::PRED_NE).addReg(CmpReg).addMBB(readMBB); 8770 8771 BB->addSuccessor(readMBB); 8772 BB->addSuccessor(sinkMBB); 8773 } 8774 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) 8775 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); 8776 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) 8777 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); 8778 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) 8779 BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4); 8780 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) 8781 BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8); 8782 8783 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I8) 8784 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); 8785 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I16) 8786 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); 8787 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I32) 8788 BB = EmitAtomicBinary(MI, BB, 4, PPC::AND); 8789 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I64) 8790 BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8); 8791 8792 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I8) 8793 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); 8794 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I16) 8795 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); 8796 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I32) 8797 BB = EmitAtomicBinary(MI, BB, 4, PPC::OR); 8798 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I64) 8799 BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8); 8800 8801 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) 8802 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); 8803 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) 8804 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); 8805 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) 8806 BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR); 8807 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) 8808 BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8); 8809 8810 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) 8811 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND); 8812 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) 8813 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND); 8814 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) 8815 BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND); 8816 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) 8817 BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8); 8818 8819 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) 8820 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); 8821 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) 8822 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); 8823 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) 8824 BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF); 8825 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) 8826 BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8); 8827 8828 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I8) 8829 BB = EmitPartwordAtomicBinary(MI, BB, true, 0); 8830 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I16) 8831 BB = EmitPartwordAtomicBinary(MI, BB, false, 0); 8832 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I32) 8833 BB = EmitAtomicBinary(MI, BB, 4, 0); 8834 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I64) 8835 BB = EmitAtomicBinary(MI, BB, 8, 0); 8836 8837 else if (MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || 8838 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 || 8839 (Subtarget.hasPartwordAtomics() && 8840 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) || 8841 (Subtarget.hasPartwordAtomics() && 8842 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) { 8843 bool is64bit = MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; 8844 8845 auto LoadMnemonic = PPC::LDARX; 8846 auto StoreMnemonic = PPC::STDCX; 8847 switch(MI->getOpcode()) { 8848 default: 8849 llvm_unreachable("Compare and swap of unknown size"); 8850 case PPC::ATOMIC_CMP_SWAP_I8: 8851 LoadMnemonic = PPC::LBARX; 8852 StoreMnemonic = PPC::STBCX; 8853 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 8854 break; 8855 case PPC::ATOMIC_CMP_SWAP_I16: 8856 LoadMnemonic = PPC::LHARX; 8857 StoreMnemonic = PPC::STHCX; 8858 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 8859 break; 8860 case PPC::ATOMIC_CMP_SWAP_I32: 8861 LoadMnemonic = PPC::LWARX; 8862 StoreMnemonic = PPC::STWCX; 8863 break; 8864 case PPC::ATOMIC_CMP_SWAP_I64: 8865 LoadMnemonic = PPC::LDARX; 8866 StoreMnemonic = PPC::STDCX; 8867 break; 8868 } 8869 unsigned dest = MI->getOperand(0).getReg(); 8870 unsigned ptrA = MI->getOperand(1).getReg(); 8871 unsigned ptrB = MI->getOperand(2).getReg(); 8872 unsigned oldval = MI->getOperand(3).getReg(); 8873 unsigned newval = MI->getOperand(4).getReg(); 8874 DebugLoc dl = MI->getDebugLoc(); 8875 8876 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 8877 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 8878 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 8879 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 8880 F->insert(It, loop1MBB); 8881 F->insert(It, loop2MBB); 8882 F->insert(It, midMBB); 8883 F->insert(It, exitMBB); 8884 exitMBB->splice(exitMBB->begin(), BB, 8885 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 8886 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 8887 8888 // thisMBB: 8889 // ... 8890 // fallthrough --> loopMBB 8891 BB->addSuccessor(loop1MBB); 8892 8893 // loop1MBB: 8894 // l[bhwd]arx dest, ptr 8895 // cmp[wd] dest, oldval 8896 // bne- midMBB 8897 // loop2MBB: 8898 // st[bhwd]cx. newval, ptr 8899 // bne- loopMBB 8900 // b exitBB 8901 // midMBB: 8902 // st[bhwd]cx. dest, ptr 8903 // exitBB: 8904 BB = loop1MBB; 8905 BuildMI(BB, dl, TII->get(LoadMnemonic), dest) 8906 .addReg(ptrA).addReg(ptrB); 8907 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) 8908 .addReg(oldval).addReg(dest); 8909 BuildMI(BB, dl, TII->get(PPC::BCC)) 8910 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB); 8911 BB->addSuccessor(loop2MBB); 8912 BB->addSuccessor(midMBB); 8913 8914 BB = loop2MBB; 8915 BuildMI(BB, dl, TII->get(StoreMnemonic)) 8916 .addReg(newval).addReg(ptrA).addReg(ptrB); 8917 BuildMI(BB, dl, TII->get(PPC::BCC)) 8918 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB); 8919 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 8920 BB->addSuccessor(loop1MBB); 8921 BB->addSuccessor(exitMBB); 8922 8923 BB = midMBB; 8924 BuildMI(BB, dl, TII->get(StoreMnemonic)) 8925 .addReg(dest).addReg(ptrA).addReg(ptrB); 8926 BB->addSuccessor(exitMBB); 8927 8928 // exitMBB: 8929 // ... 8930 BB = exitMBB; 8931 } else if (MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || 8932 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { 8933 // We must use 64-bit registers for addresses when targeting 64-bit, 8934 // since we're actually doing arithmetic on them. Other registers 8935 // can be 32-bit. 8936 bool is64bit = Subtarget.isPPC64(); 8937 bool is8bit = MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; 8938 8939 unsigned dest = MI->getOperand(0).getReg(); 8940 unsigned ptrA = MI->getOperand(1).getReg(); 8941 unsigned ptrB = MI->getOperand(2).getReg(); 8942 unsigned oldval = MI->getOperand(3).getReg(); 8943 unsigned newval = MI->getOperand(4).getReg(); 8944 DebugLoc dl = MI->getDebugLoc(); 8945 8946 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 8947 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 8948 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 8949 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 8950 F->insert(It, loop1MBB); 8951 F->insert(It, loop2MBB); 8952 F->insert(It, midMBB); 8953 F->insert(It, exitMBB); 8954 exitMBB->splice(exitMBB->begin(), BB, 8955 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 8956 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 8957 8958 MachineRegisterInfo &RegInfo = F->getRegInfo(); 8959 const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass 8960 : &PPC::GPRCRegClass; 8961 unsigned PtrReg = RegInfo.createVirtualRegister(RC); 8962 unsigned Shift1Reg = RegInfo.createVirtualRegister(RC); 8963 unsigned ShiftReg = RegInfo.createVirtualRegister(RC); 8964 unsigned NewVal2Reg = RegInfo.createVirtualRegister(RC); 8965 unsigned NewVal3Reg = RegInfo.createVirtualRegister(RC); 8966 unsigned OldVal2Reg = RegInfo.createVirtualRegister(RC); 8967 unsigned OldVal3Reg = RegInfo.createVirtualRegister(RC); 8968 unsigned MaskReg = RegInfo.createVirtualRegister(RC); 8969 unsigned Mask2Reg = RegInfo.createVirtualRegister(RC); 8970 unsigned Mask3Reg = RegInfo.createVirtualRegister(RC); 8971 unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC); 8972 unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC); 8973 unsigned TmpDestReg = RegInfo.createVirtualRegister(RC); 8974 unsigned Ptr1Reg; 8975 unsigned TmpReg = RegInfo.createVirtualRegister(RC); 8976 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 8977 // thisMBB: 8978 // ... 8979 // fallthrough --> loopMBB 8980 BB->addSuccessor(loop1MBB); 8981 8982 // The 4-byte load must be aligned, while a char or short may be 8983 // anywhere in the word. Hence all this nasty bookkeeping code. 8984 // add ptr1, ptrA, ptrB [copy if ptrA==0] 8985 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 8986 // xori shift, shift1, 24 [16] 8987 // rlwinm ptr, ptr1, 0, 0, 29 8988 // slw newval2, newval, shift 8989 // slw oldval2, oldval,shift 8990 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 8991 // slw mask, mask2, shift 8992 // and newval3, newval2, mask 8993 // and oldval3, oldval2, mask 8994 // loop1MBB: 8995 // lwarx tmpDest, ptr 8996 // and tmp, tmpDest, mask 8997 // cmpw tmp, oldval3 8998 // bne- midMBB 8999 // loop2MBB: 9000 // andc tmp2, tmpDest, mask 9001 // or tmp4, tmp2, newval3 9002 // stwcx. tmp4, ptr 9003 // bne- loop1MBB 9004 // b exitBB 9005 // midMBB: 9006 // stwcx. tmpDest, ptr 9007 // exitBB: 9008 // srw dest, tmpDest, shift 9009 if (ptrA != ZeroReg) { 9010 Ptr1Reg = RegInfo.createVirtualRegister(RC); 9011 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 9012 .addReg(ptrA).addReg(ptrB); 9013 } else { 9014 Ptr1Reg = ptrB; 9015 } 9016 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg) 9017 .addImm(3).addImm(27).addImm(is8bit ? 28 : 27); 9018 BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg) 9019 .addReg(Shift1Reg).addImm(is8bit ? 24 : 16); 9020 if (is64bit) 9021 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 9022 .addReg(Ptr1Reg).addImm(0).addImm(61); 9023 else 9024 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 9025 .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29); 9026 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) 9027 .addReg(newval).addReg(ShiftReg); 9028 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) 9029 .addReg(oldval).addReg(ShiftReg); 9030 if (is8bit) 9031 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 9032 else { 9033 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 9034 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 9035 .addReg(Mask3Reg).addImm(65535); 9036 } 9037 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 9038 .addReg(Mask2Reg).addReg(ShiftReg); 9039 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) 9040 .addReg(NewVal2Reg).addReg(MaskReg); 9041 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) 9042 .addReg(OldVal2Reg).addReg(MaskReg); 9043 9044 BB = loop1MBB; 9045 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 9046 .addReg(ZeroReg).addReg(PtrReg); 9047 BuildMI(BB, dl, TII->get(PPC::AND),TmpReg) 9048 .addReg(TmpDestReg).addReg(MaskReg); 9049 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) 9050 .addReg(TmpReg).addReg(OldVal3Reg); 9051 BuildMI(BB, dl, TII->get(PPC::BCC)) 9052 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB); 9053 BB->addSuccessor(loop2MBB); 9054 BB->addSuccessor(midMBB); 9055 9056 BB = loop2MBB; 9057 BuildMI(BB, dl, TII->get(PPC::ANDC),Tmp2Reg) 9058 .addReg(TmpDestReg).addReg(MaskReg); 9059 BuildMI(BB, dl, TII->get(PPC::OR),Tmp4Reg) 9060 .addReg(Tmp2Reg).addReg(NewVal3Reg); 9061 BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(Tmp4Reg) 9062 .addReg(ZeroReg).addReg(PtrReg); 9063 BuildMI(BB, dl, TII->get(PPC::BCC)) 9064 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB); 9065 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 9066 BB->addSuccessor(loop1MBB); 9067 BB->addSuccessor(exitMBB); 9068 9069 BB = midMBB; 9070 BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(TmpDestReg) 9071 .addReg(ZeroReg).addReg(PtrReg); 9072 BB->addSuccessor(exitMBB); 9073 9074 // exitMBB: 9075 // ... 9076 BB = exitMBB; 9077 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW),dest).addReg(TmpReg) 9078 .addReg(ShiftReg); 9079 } else if (MI->getOpcode() == PPC::FADDrtz) { 9080 // This pseudo performs an FADD with rounding mode temporarily forced 9081 // to round-to-zero. We emit this via custom inserter since the FPSCR 9082 // is not modeled at the SelectionDAG level. 9083 unsigned Dest = MI->getOperand(0).getReg(); 9084 unsigned Src1 = MI->getOperand(1).getReg(); 9085 unsigned Src2 = MI->getOperand(2).getReg(); 9086 DebugLoc dl = MI->getDebugLoc(); 9087 9088 MachineRegisterInfo &RegInfo = F->getRegInfo(); 9089 unsigned MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 9090 9091 // Save FPSCR value. 9092 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); 9093 9094 // Set rounding mode to round-to-zero. 9095 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); 9096 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); 9097 9098 // Perform addition. 9099 BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); 9100 9101 // Restore FPSCR value. 9102 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); 9103 } else if (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT || 9104 MI->getOpcode() == PPC::ANDIo_1_GT_BIT || 9105 MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8 || 9106 MI->getOpcode() == PPC::ANDIo_1_GT_BIT8) { 9107 unsigned Opcode = (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8 || 9108 MI->getOpcode() == PPC::ANDIo_1_GT_BIT8) ? 9109 PPC::ANDIo8 : PPC::ANDIo; 9110 bool isEQ = (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT || 9111 MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8); 9112 9113 MachineRegisterInfo &RegInfo = F->getRegInfo(); 9114 unsigned Dest = RegInfo.createVirtualRegister(Opcode == PPC::ANDIo ? 9115 &PPC::GPRCRegClass : 9116 &PPC::G8RCRegClass); 9117 9118 DebugLoc dl = MI->getDebugLoc(); 9119 BuildMI(*BB, MI, dl, TII->get(Opcode), Dest) 9120 .addReg(MI->getOperand(1).getReg()).addImm(1); 9121 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), 9122 MI->getOperand(0).getReg()) 9123 .addReg(isEQ ? PPC::CR0EQ : PPC::CR0GT); 9124 } else if (MI->getOpcode() == PPC::TCHECK_RET) { 9125 DebugLoc Dl = MI->getDebugLoc(); 9126 MachineRegisterInfo &RegInfo = F->getRegInfo(); 9127 unsigned CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 9128 BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg); 9129 return BB; 9130 } else { 9131 llvm_unreachable("Unexpected instr type to insert"); 9132 } 9133 9134 MI->eraseFromParent(); // The pseudo instruction is gone now. 9135 return BB; 9136 } 9137 9138 //===----------------------------------------------------------------------===// 9139 // Target Optimization Hooks 9140 //===----------------------------------------------------------------------===// 9141 9142 static std::string getRecipOp(const char *Base, EVT VT) { 9143 std::string RecipOp(Base); 9144 if (VT.getScalarType() == MVT::f64) 9145 RecipOp += "d"; 9146 else 9147 RecipOp += "f"; 9148 9149 if (VT.isVector()) 9150 RecipOp = "vec-" + RecipOp; 9151 9152 return RecipOp; 9153 } 9154 9155 SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand, 9156 DAGCombinerInfo &DCI, 9157 unsigned &RefinementSteps, 9158 bool &UseOneConstNR) const { 9159 EVT VT = Operand.getValueType(); 9160 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || 9161 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || 9162 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 9163 (VT == MVT::v2f64 && Subtarget.hasVSX()) || 9164 (VT == MVT::v4f32 && Subtarget.hasQPX()) || 9165 (VT == MVT::v4f64 && Subtarget.hasQPX())) { 9166 TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals; 9167 std::string RecipOp = getRecipOp("sqrt", VT); 9168 if (!Recips.isEnabled(RecipOp)) 9169 return SDValue(); 9170 9171 RefinementSteps = Recips.getRefinementSteps(RecipOp); 9172 UseOneConstNR = true; 9173 return DCI.DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand); 9174 } 9175 return SDValue(); 9176 } 9177 9178 SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, 9179 DAGCombinerInfo &DCI, 9180 unsigned &RefinementSteps) const { 9181 EVT VT = Operand.getValueType(); 9182 if ((VT == MVT::f32 && Subtarget.hasFRES()) || 9183 (VT == MVT::f64 && Subtarget.hasFRE()) || 9184 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 9185 (VT == MVT::v2f64 && Subtarget.hasVSX()) || 9186 (VT == MVT::v4f32 && Subtarget.hasQPX()) || 9187 (VT == MVT::v4f64 && Subtarget.hasQPX())) { 9188 TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals; 9189 std::string RecipOp = getRecipOp("div", VT); 9190 if (!Recips.isEnabled(RecipOp)) 9191 return SDValue(); 9192 9193 RefinementSteps = Recips.getRefinementSteps(RecipOp); 9194 return DCI.DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); 9195 } 9196 return SDValue(); 9197 } 9198 9199 unsigned PPCTargetLowering::combineRepeatedFPDivisors() const { 9200 // Note: This functionality is used only when unsafe-fp-math is enabled, and 9201 // on cores with reciprocal estimates (which are used when unsafe-fp-math is 9202 // enabled for division), this functionality is redundant with the default 9203 // combiner logic (once the division -> reciprocal/multiply transformation 9204 // has taken place). As a result, this matters more for older cores than for 9205 // newer ones. 9206 9207 // Combine multiple FDIVs with the same divisor into multiple FMULs by the 9208 // reciprocal if there are two or more FDIVs (for embedded cores with only 9209 // one FP pipeline) for three or more FDIVs (for generic OOO cores). 9210 switch (Subtarget.getDarwinDirective()) { 9211 default: 9212 return 3; 9213 case PPC::DIR_440: 9214 case PPC::DIR_A2: 9215 case PPC::DIR_E500mc: 9216 case PPC::DIR_E5500: 9217 return 2; 9218 } 9219 } 9220 9221 // isConsecutiveLSLoc needs to work even if all adds have not yet been 9222 // collapsed, and so we need to look through chains of them. 9223 static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base, 9224 int64_t& Offset, SelectionDAG &DAG) { 9225 if (DAG.isBaseWithConstantOffset(Loc)) { 9226 Base = Loc.getOperand(0); 9227 Offset += cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue(); 9228 9229 // The base might itself be a base plus an offset, and if so, accumulate 9230 // that as well. 9231 getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG); 9232 } 9233 } 9234 9235 static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base, 9236 unsigned Bytes, int Dist, 9237 SelectionDAG &DAG) { 9238 if (VT.getSizeInBits() / 8 != Bytes) 9239 return false; 9240 9241 SDValue BaseLoc = Base->getBasePtr(); 9242 if (Loc.getOpcode() == ISD::FrameIndex) { 9243 if (BaseLoc.getOpcode() != ISD::FrameIndex) 9244 return false; 9245 const MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 9246 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 9247 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 9248 int FS = MFI->getObjectSize(FI); 9249 int BFS = MFI->getObjectSize(BFI); 9250 if (FS != BFS || FS != (int)Bytes) return false; 9251 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 9252 } 9253 9254 SDValue Base1 = Loc, Base2 = BaseLoc; 9255 int64_t Offset1 = 0, Offset2 = 0; 9256 getBaseWithConstantOffset(Loc, Base1, Offset1, DAG); 9257 getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG); 9258 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes)) 9259 return true; 9260 9261 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9262 const GlobalValue *GV1 = nullptr; 9263 const GlobalValue *GV2 = nullptr; 9264 Offset1 = 0; 9265 Offset2 = 0; 9266 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 9267 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 9268 if (isGA1 && isGA2 && GV1 == GV2) 9269 return Offset1 == (Offset2 + Dist*Bytes); 9270 return false; 9271 } 9272 9273 // Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does 9274 // not enforce equality of the chain operands. 9275 static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, 9276 unsigned Bytes, int Dist, 9277 SelectionDAG &DAG) { 9278 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N)) { 9279 EVT VT = LS->getMemoryVT(); 9280 SDValue Loc = LS->getBasePtr(); 9281 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG); 9282 } 9283 9284 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 9285 EVT VT; 9286 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 9287 default: return false; 9288 case Intrinsic::ppc_qpx_qvlfd: 9289 case Intrinsic::ppc_qpx_qvlfda: 9290 VT = MVT::v4f64; 9291 break; 9292 case Intrinsic::ppc_qpx_qvlfs: 9293 case Intrinsic::ppc_qpx_qvlfsa: 9294 VT = MVT::v4f32; 9295 break; 9296 case Intrinsic::ppc_qpx_qvlfcd: 9297 case Intrinsic::ppc_qpx_qvlfcda: 9298 VT = MVT::v2f64; 9299 break; 9300 case Intrinsic::ppc_qpx_qvlfcs: 9301 case Intrinsic::ppc_qpx_qvlfcsa: 9302 VT = MVT::v2f32; 9303 break; 9304 case Intrinsic::ppc_qpx_qvlfiwa: 9305 case Intrinsic::ppc_qpx_qvlfiwz: 9306 case Intrinsic::ppc_altivec_lvx: 9307 case Intrinsic::ppc_altivec_lvxl: 9308 case Intrinsic::ppc_vsx_lxvw4x: 9309 VT = MVT::v4i32; 9310 break; 9311 case Intrinsic::ppc_vsx_lxvd2x: 9312 VT = MVT::v2f64; 9313 break; 9314 case Intrinsic::ppc_altivec_lvebx: 9315 VT = MVT::i8; 9316 break; 9317 case Intrinsic::ppc_altivec_lvehx: 9318 VT = MVT::i16; 9319 break; 9320 case Intrinsic::ppc_altivec_lvewx: 9321 VT = MVT::i32; 9322 break; 9323 } 9324 9325 return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG); 9326 } 9327 9328 if (N->getOpcode() == ISD::INTRINSIC_VOID) { 9329 EVT VT; 9330 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 9331 default: return false; 9332 case Intrinsic::ppc_qpx_qvstfd: 9333 case Intrinsic::ppc_qpx_qvstfda: 9334 VT = MVT::v4f64; 9335 break; 9336 case Intrinsic::ppc_qpx_qvstfs: 9337 case Intrinsic::ppc_qpx_qvstfsa: 9338 VT = MVT::v4f32; 9339 break; 9340 case Intrinsic::ppc_qpx_qvstfcd: 9341 case Intrinsic::ppc_qpx_qvstfcda: 9342 VT = MVT::v2f64; 9343 break; 9344 case Intrinsic::ppc_qpx_qvstfcs: 9345 case Intrinsic::ppc_qpx_qvstfcsa: 9346 VT = MVT::v2f32; 9347 break; 9348 case Intrinsic::ppc_qpx_qvstfiw: 9349 case Intrinsic::ppc_qpx_qvstfiwa: 9350 case Intrinsic::ppc_altivec_stvx: 9351 case Intrinsic::ppc_altivec_stvxl: 9352 case Intrinsic::ppc_vsx_stxvw4x: 9353 VT = MVT::v4i32; 9354 break; 9355 case Intrinsic::ppc_vsx_stxvd2x: 9356 VT = MVT::v2f64; 9357 break; 9358 case Intrinsic::ppc_altivec_stvebx: 9359 VT = MVT::i8; 9360 break; 9361 case Intrinsic::ppc_altivec_stvehx: 9362 VT = MVT::i16; 9363 break; 9364 case Intrinsic::ppc_altivec_stvewx: 9365 VT = MVT::i32; 9366 break; 9367 } 9368 9369 return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG); 9370 } 9371 9372 return false; 9373 } 9374 9375 // Return true is there is a nearyby consecutive load to the one provided 9376 // (regardless of alignment). We search up and down the chain, looking though 9377 // token factors and other loads (but nothing else). As a result, a true result 9378 // indicates that it is safe to create a new consecutive load adjacent to the 9379 // load provided. 9380 static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { 9381 SDValue Chain = LD->getChain(); 9382 EVT VT = LD->getMemoryVT(); 9383 9384 SmallSet<SDNode *, 16> LoadRoots; 9385 SmallVector<SDNode *, 8> Queue(1, Chain.getNode()); 9386 SmallSet<SDNode *, 16> Visited; 9387 9388 // First, search up the chain, branching to follow all token-factor operands. 9389 // If we find a consecutive load, then we're done, otherwise, record all 9390 // nodes just above the top-level loads and token factors. 9391 while (!Queue.empty()) { 9392 SDNode *ChainNext = Queue.pop_back_val(); 9393 if (!Visited.insert(ChainNext).second) 9394 continue; 9395 9396 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(ChainNext)) { 9397 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 9398 return true; 9399 9400 if (!Visited.count(ChainLD->getChain().getNode())) 9401 Queue.push_back(ChainLD->getChain().getNode()); 9402 } else if (ChainNext->getOpcode() == ISD::TokenFactor) { 9403 for (const SDUse &O : ChainNext->ops()) 9404 if (!Visited.count(O.getNode())) 9405 Queue.push_back(O.getNode()); 9406 } else 9407 LoadRoots.insert(ChainNext); 9408 } 9409 9410 // Second, search down the chain, starting from the top-level nodes recorded 9411 // in the first phase. These top-level nodes are the nodes just above all 9412 // loads and token factors. Starting with their uses, recursively look though 9413 // all loads (just the chain uses) and token factors to find a consecutive 9414 // load. 9415 Visited.clear(); 9416 Queue.clear(); 9417 9418 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(), 9419 IE = LoadRoots.end(); I != IE; ++I) { 9420 Queue.push_back(*I); 9421 9422 while (!Queue.empty()) { 9423 SDNode *LoadRoot = Queue.pop_back_val(); 9424 if (!Visited.insert(LoadRoot).second) 9425 continue; 9426 9427 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(LoadRoot)) 9428 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 9429 return true; 9430 9431 for (SDNode::use_iterator UI = LoadRoot->use_begin(), 9432 UE = LoadRoot->use_end(); UI != UE; ++UI) 9433 if (((isa<MemSDNode>(*UI) && 9434 cast<MemSDNode>(*UI)->getChain().getNode() == LoadRoot) || 9435 UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI)) 9436 Queue.push_back(*UI); 9437 } 9438 } 9439 9440 return false; 9441 } 9442 9443 SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, 9444 DAGCombinerInfo &DCI) const { 9445 SelectionDAG &DAG = DCI.DAG; 9446 SDLoc dl(N); 9447 9448 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits"); 9449 // If we're tracking CR bits, we need to be careful that we don't have: 9450 // trunc(binary-ops(zext(x), zext(y))) 9451 // or 9452 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) 9453 // such that we're unnecessarily moving things into GPRs when it would be 9454 // better to keep them in CR bits. 9455 9456 // Note that trunc here can be an actual i1 trunc, or can be the effective 9457 // truncation that comes from a setcc or select_cc. 9458 if (N->getOpcode() == ISD::TRUNCATE && 9459 N->getValueType(0) != MVT::i1) 9460 return SDValue(); 9461 9462 if (N->getOperand(0).getValueType() != MVT::i32 && 9463 N->getOperand(0).getValueType() != MVT::i64) 9464 return SDValue(); 9465 9466 if (N->getOpcode() == ISD::SETCC || 9467 N->getOpcode() == ISD::SELECT_CC) { 9468 // If we're looking at a comparison, then we need to make sure that the 9469 // high bits (all except for the first) don't matter the result. 9470 ISD::CondCode CC = 9471 cast<CondCodeSDNode>(N->getOperand( 9472 N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); 9473 unsigned OpBits = N->getOperand(0).getValueSizeInBits(); 9474 9475 if (ISD::isSignedIntSetCC(CC)) { 9476 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || 9477 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) 9478 return SDValue(); 9479 } else if (ISD::isUnsignedIntSetCC(CC)) { 9480 if (!DAG.MaskedValueIsZero(N->getOperand(0), 9481 APInt::getHighBitsSet(OpBits, OpBits-1)) || 9482 !DAG.MaskedValueIsZero(N->getOperand(1), 9483 APInt::getHighBitsSet(OpBits, OpBits-1))) 9484 return SDValue(); 9485 } else { 9486 // This is neither a signed nor an unsigned comparison, just make sure 9487 // that the high bits are equal. 9488 APInt Op1Zero, Op1One; 9489 APInt Op2Zero, Op2One; 9490 DAG.computeKnownBits(N->getOperand(0), Op1Zero, Op1One); 9491 DAG.computeKnownBits(N->getOperand(1), Op2Zero, Op2One); 9492 9493 // We don't really care about what is known about the first bit (if 9494 // anything), so clear it in all masks prior to comparing them. 9495 Op1Zero.clearBit(0); Op1One.clearBit(0); 9496 Op2Zero.clearBit(0); Op2One.clearBit(0); 9497 9498 if (Op1Zero != Op2Zero || Op1One != Op2One) 9499 return SDValue(); 9500 } 9501 } 9502 9503 // We now know that the higher-order bits are irrelevant, we just need to 9504 // make sure that all of the intermediate operations are bit operations, and 9505 // all inputs are extensions. 9506 if (N->getOperand(0).getOpcode() != ISD::AND && 9507 N->getOperand(0).getOpcode() != ISD::OR && 9508 N->getOperand(0).getOpcode() != ISD::XOR && 9509 N->getOperand(0).getOpcode() != ISD::SELECT && 9510 N->getOperand(0).getOpcode() != ISD::SELECT_CC && 9511 N->getOperand(0).getOpcode() != ISD::TRUNCATE && 9512 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && 9513 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && 9514 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) 9515 return SDValue(); 9516 9517 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && 9518 N->getOperand(1).getOpcode() != ISD::AND && 9519 N->getOperand(1).getOpcode() != ISD::OR && 9520 N->getOperand(1).getOpcode() != ISD::XOR && 9521 N->getOperand(1).getOpcode() != ISD::SELECT && 9522 N->getOperand(1).getOpcode() != ISD::SELECT_CC && 9523 N->getOperand(1).getOpcode() != ISD::TRUNCATE && 9524 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && 9525 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && 9526 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) 9527 return SDValue(); 9528 9529 SmallVector<SDValue, 4> Inputs; 9530 SmallVector<SDValue, 8> BinOps, PromOps; 9531 SmallPtrSet<SDNode *, 16> Visited; 9532 9533 for (unsigned i = 0; i < 2; ++i) { 9534 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 9535 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 9536 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 9537 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || 9538 isa<ConstantSDNode>(N->getOperand(i))) 9539 Inputs.push_back(N->getOperand(i)); 9540 else 9541 BinOps.push_back(N->getOperand(i)); 9542 9543 if (N->getOpcode() == ISD::TRUNCATE) 9544 break; 9545 } 9546 9547 // Visit all inputs, collect all binary operations (and, or, xor and 9548 // select) that are all fed by extensions. 9549 while (!BinOps.empty()) { 9550 SDValue BinOp = BinOps.back(); 9551 BinOps.pop_back(); 9552 9553 if (!Visited.insert(BinOp.getNode()).second) 9554 continue; 9555 9556 PromOps.push_back(BinOp); 9557 9558 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 9559 // The condition of the select is not promoted. 9560 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 9561 continue; 9562 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 9563 continue; 9564 9565 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 9566 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 9567 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 9568 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || 9569 isa<ConstantSDNode>(BinOp.getOperand(i))) { 9570 Inputs.push_back(BinOp.getOperand(i)); 9571 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 9572 BinOp.getOperand(i).getOpcode() == ISD::OR || 9573 BinOp.getOperand(i).getOpcode() == ISD::XOR || 9574 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 9575 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || 9576 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 9577 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 9578 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 9579 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { 9580 BinOps.push_back(BinOp.getOperand(i)); 9581 } else { 9582 // We have an input that is not an extension or another binary 9583 // operation; we'll abort this transformation. 9584 return SDValue(); 9585 } 9586 } 9587 } 9588 9589 // Make sure that this is a self-contained cluster of operations (which 9590 // is not quite the same thing as saying that everything has only one 9591 // use). 9592 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 9593 if (isa<ConstantSDNode>(Inputs[i])) 9594 continue; 9595 9596 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 9597 UE = Inputs[i].getNode()->use_end(); 9598 UI != UE; ++UI) { 9599 SDNode *User = *UI; 9600 if (User != N && !Visited.count(User)) 9601 return SDValue(); 9602 9603 // Make sure that we're not going to promote the non-output-value 9604 // operand(s) or SELECT or SELECT_CC. 9605 // FIXME: Although we could sometimes handle this, and it does occur in 9606 // practice that one of the condition inputs to the select is also one of 9607 // the outputs, we currently can't deal with this. 9608 if (User->getOpcode() == ISD::SELECT) { 9609 if (User->getOperand(0) == Inputs[i]) 9610 return SDValue(); 9611 } else if (User->getOpcode() == ISD::SELECT_CC) { 9612 if (User->getOperand(0) == Inputs[i] || 9613 User->getOperand(1) == Inputs[i]) 9614 return SDValue(); 9615 } 9616 } 9617 } 9618 9619 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 9620 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 9621 UE = PromOps[i].getNode()->use_end(); 9622 UI != UE; ++UI) { 9623 SDNode *User = *UI; 9624 if (User != N && !Visited.count(User)) 9625 return SDValue(); 9626 9627 // Make sure that we're not going to promote the non-output-value 9628 // operand(s) or SELECT or SELECT_CC. 9629 // FIXME: Although we could sometimes handle this, and it does occur in 9630 // practice that one of the condition inputs to the select is also one of 9631 // the outputs, we currently can't deal with this. 9632 if (User->getOpcode() == ISD::SELECT) { 9633 if (User->getOperand(0) == PromOps[i]) 9634 return SDValue(); 9635 } else if (User->getOpcode() == ISD::SELECT_CC) { 9636 if (User->getOperand(0) == PromOps[i] || 9637 User->getOperand(1) == PromOps[i]) 9638 return SDValue(); 9639 } 9640 } 9641 } 9642 9643 // Replace all inputs with the extension operand. 9644 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 9645 // Constants may have users outside the cluster of to-be-promoted nodes, 9646 // and so we need to replace those as we do the promotions. 9647 if (isa<ConstantSDNode>(Inputs[i])) 9648 continue; 9649 else 9650 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); 9651 } 9652 9653 // Replace all operations (these are all the same, but have a different 9654 // (i1) return type). DAG.getNode will validate that the types of 9655 // a binary operator match, so go through the list in reverse so that 9656 // we've likely promoted both operands first. Any intermediate truncations or 9657 // extensions disappear. 9658 while (!PromOps.empty()) { 9659 SDValue PromOp = PromOps.back(); 9660 PromOps.pop_back(); 9661 9662 if (PromOp.getOpcode() == ISD::TRUNCATE || 9663 PromOp.getOpcode() == ISD::SIGN_EXTEND || 9664 PromOp.getOpcode() == ISD::ZERO_EXTEND || 9665 PromOp.getOpcode() == ISD::ANY_EXTEND) { 9666 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) && 9667 PromOp.getOperand(0).getValueType() != MVT::i1) { 9668 // The operand is not yet ready (see comment below). 9669 PromOps.insert(PromOps.begin(), PromOp); 9670 continue; 9671 } 9672 9673 SDValue RepValue = PromOp.getOperand(0); 9674 if (isa<ConstantSDNode>(RepValue)) 9675 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); 9676 9677 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); 9678 continue; 9679 } 9680 9681 unsigned C; 9682 switch (PromOp.getOpcode()) { 9683 default: C = 0; break; 9684 case ISD::SELECT: C = 1; break; 9685 case ISD::SELECT_CC: C = 2; break; 9686 } 9687 9688 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 9689 PromOp.getOperand(C).getValueType() != MVT::i1) || 9690 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 9691 PromOp.getOperand(C+1).getValueType() != MVT::i1)) { 9692 // The to-be-promoted operands of this node have not yet been 9693 // promoted (this should be rare because we're going through the 9694 // list backward, but if one of the operands has several users in 9695 // this cluster of to-be-promoted nodes, it is possible). 9696 PromOps.insert(PromOps.begin(), PromOp); 9697 continue; 9698 } 9699 9700 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 9701 PromOp.getNode()->op_end()); 9702 9703 // If there are any constant inputs, make sure they're replaced now. 9704 for (unsigned i = 0; i < 2; ++i) 9705 if (isa<ConstantSDNode>(Ops[C+i])) 9706 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); 9707 9708 DAG.ReplaceAllUsesOfValueWith(PromOp, 9709 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); 9710 } 9711 9712 // Now we're left with the initial truncation itself. 9713 if (N->getOpcode() == ISD::TRUNCATE) 9714 return N->getOperand(0); 9715 9716 // Otherwise, this is a comparison. The operands to be compared have just 9717 // changed type (to i1), but everything else is the same. 9718 return SDValue(N, 0); 9719 } 9720 9721 SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, 9722 DAGCombinerInfo &DCI) const { 9723 SelectionDAG &DAG = DCI.DAG; 9724 SDLoc dl(N); 9725 9726 // If we're tracking CR bits, we need to be careful that we don't have: 9727 // zext(binary-ops(trunc(x), trunc(y))) 9728 // or 9729 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) 9730 // such that we're unnecessarily moving things into CR bits that can more 9731 // efficiently stay in GPRs. Note that if we're not certain that the high 9732 // bits are set as required by the final extension, we still may need to do 9733 // some masking to get the proper behavior. 9734 9735 // This same functionality is important on PPC64 when dealing with 9736 // 32-to-64-bit extensions; these occur often when 32-bit values are used as 9737 // the return values of functions. Because it is so similar, it is handled 9738 // here as well. 9739 9740 if (N->getValueType(0) != MVT::i32 && 9741 N->getValueType(0) != MVT::i64) 9742 return SDValue(); 9743 9744 if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) || 9745 (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64()))) 9746 return SDValue(); 9747 9748 if (N->getOperand(0).getOpcode() != ISD::AND && 9749 N->getOperand(0).getOpcode() != ISD::OR && 9750 N->getOperand(0).getOpcode() != ISD::XOR && 9751 N->getOperand(0).getOpcode() != ISD::SELECT && 9752 N->getOperand(0).getOpcode() != ISD::SELECT_CC) 9753 return SDValue(); 9754 9755 SmallVector<SDValue, 4> Inputs; 9756 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps; 9757 SmallPtrSet<SDNode *, 16> Visited; 9758 9759 // Visit all inputs, collect all binary operations (and, or, xor and 9760 // select) that are all fed by truncations. 9761 while (!BinOps.empty()) { 9762 SDValue BinOp = BinOps.back(); 9763 BinOps.pop_back(); 9764 9765 if (!Visited.insert(BinOp.getNode()).second) 9766 continue; 9767 9768 PromOps.push_back(BinOp); 9769 9770 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 9771 // The condition of the select is not promoted. 9772 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 9773 continue; 9774 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 9775 continue; 9776 9777 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 9778 isa<ConstantSDNode>(BinOp.getOperand(i))) { 9779 Inputs.push_back(BinOp.getOperand(i)); 9780 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 9781 BinOp.getOperand(i).getOpcode() == ISD::OR || 9782 BinOp.getOperand(i).getOpcode() == ISD::XOR || 9783 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 9784 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { 9785 BinOps.push_back(BinOp.getOperand(i)); 9786 } else { 9787 // We have an input that is not a truncation or another binary 9788 // operation; we'll abort this transformation. 9789 return SDValue(); 9790 } 9791 } 9792 } 9793 9794 // The operands of a select that must be truncated when the select is 9795 // promoted because the operand is actually part of the to-be-promoted set. 9796 DenseMap<SDNode *, EVT> SelectTruncOp[2]; 9797 9798 // Make sure that this is a self-contained cluster of operations (which 9799 // is not quite the same thing as saying that everything has only one 9800 // use). 9801 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 9802 if (isa<ConstantSDNode>(Inputs[i])) 9803 continue; 9804 9805 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 9806 UE = Inputs[i].getNode()->use_end(); 9807 UI != UE; ++UI) { 9808 SDNode *User = *UI; 9809 if (User != N && !Visited.count(User)) 9810 return SDValue(); 9811 9812 // If we're going to promote the non-output-value operand(s) or SELECT or 9813 // SELECT_CC, record them for truncation. 9814 if (User->getOpcode() == ISD::SELECT) { 9815 if (User->getOperand(0) == Inputs[i]) 9816 SelectTruncOp[0].insert(std::make_pair(User, 9817 User->getOperand(0).getValueType())); 9818 } else if (User->getOpcode() == ISD::SELECT_CC) { 9819 if (User->getOperand(0) == Inputs[i]) 9820 SelectTruncOp[0].insert(std::make_pair(User, 9821 User->getOperand(0).getValueType())); 9822 if (User->getOperand(1) == Inputs[i]) 9823 SelectTruncOp[1].insert(std::make_pair(User, 9824 User->getOperand(1).getValueType())); 9825 } 9826 } 9827 } 9828 9829 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 9830 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 9831 UE = PromOps[i].getNode()->use_end(); 9832 UI != UE; ++UI) { 9833 SDNode *User = *UI; 9834 if (User != N && !Visited.count(User)) 9835 return SDValue(); 9836 9837 // If we're going to promote the non-output-value operand(s) or SELECT or 9838 // SELECT_CC, record them for truncation. 9839 if (User->getOpcode() == ISD::SELECT) { 9840 if (User->getOperand(0) == PromOps[i]) 9841 SelectTruncOp[0].insert(std::make_pair(User, 9842 User->getOperand(0).getValueType())); 9843 } else if (User->getOpcode() == ISD::SELECT_CC) { 9844 if (User->getOperand(0) == PromOps[i]) 9845 SelectTruncOp[0].insert(std::make_pair(User, 9846 User->getOperand(0).getValueType())); 9847 if (User->getOperand(1) == PromOps[i]) 9848 SelectTruncOp[1].insert(std::make_pair(User, 9849 User->getOperand(1).getValueType())); 9850 } 9851 } 9852 } 9853 9854 unsigned PromBits = N->getOperand(0).getValueSizeInBits(); 9855 bool ReallyNeedsExt = false; 9856 if (N->getOpcode() != ISD::ANY_EXTEND) { 9857 // If all of the inputs are not already sign/zero extended, then 9858 // we'll still need to do that at the end. 9859 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 9860 if (isa<ConstantSDNode>(Inputs[i])) 9861 continue; 9862 9863 unsigned OpBits = 9864 Inputs[i].getOperand(0).getValueSizeInBits(); 9865 assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); 9866 9867 if ((N->getOpcode() == ISD::ZERO_EXTEND && 9868 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), 9869 APInt::getHighBitsSet(OpBits, 9870 OpBits-PromBits))) || 9871 (N->getOpcode() == ISD::SIGN_EXTEND && 9872 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < 9873 (OpBits-(PromBits-1)))) { 9874 ReallyNeedsExt = true; 9875 break; 9876 } 9877 } 9878 } 9879 9880 // Replace all inputs, either with the truncation operand, or a 9881 // truncation or extension to the final output type. 9882 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 9883 // Constant inputs need to be replaced with the to-be-promoted nodes that 9884 // use them because they might have users outside of the cluster of 9885 // promoted nodes. 9886 if (isa<ConstantSDNode>(Inputs[i])) 9887 continue; 9888 9889 SDValue InSrc = Inputs[i].getOperand(0); 9890 if (Inputs[i].getValueType() == N->getValueType(0)) 9891 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); 9892 else if (N->getOpcode() == ISD::SIGN_EXTEND) 9893 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 9894 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); 9895 else if (N->getOpcode() == ISD::ZERO_EXTEND) 9896 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 9897 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); 9898 else 9899 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 9900 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); 9901 } 9902 9903 // Replace all operations (these are all the same, but have a different 9904 // (promoted) return type). DAG.getNode will validate that the types of 9905 // a binary operator match, so go through the list in reverse so that 9906 // we've likely promoted both operands first. 9907 while (!PromOps.empty()) { 9908 SDValue PromOp = PromOps.back(); 9909 PromOps.pop_back(); 9910 9911 unsigned C; 9912 switch (PromOp.getOpcode()) { 9913 default: C = 0; break; 9914 case ISD::SELECT: C = 1; break; 9915 case ISD::SELECT_CC: C = 2; break; 9916 } 9917 9918 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 9919 PromOp.getOperand(C).getValueType() != N->getValueType(0)) || 9920 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 9921 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { 9922 // The to-be-promoted operands of this node have not yet been 9923 // promoted (this should be rare because we're going through the 9924 // list backward, but if one of the operands has several users in 9925 // this cluster of to-be-promoted nodes, it is possible). 9926 PromOps.insert(PromOps.begin(), PromOp); 9927 continue; 9928 } 9929 9930 // For SELECT and SELECT_CC nodes, we do a similar check for any 9931 // to-be-promoted comparison inputs. 9932 if (PromOp.getOpcode() == ISD::SELECT || 9933 PromOp.getOpcode() == ISD::SELECT_CC) { 9934 if ((SelectTruncOp[0].count(PromOp.getNode()) && 9935 PromOp.getOperand(0).getValueType() != N->getValueType(0)) || 9936 (SelectTruncOp[1].count(PromOp.getNode()) && 9937 PromOp.getOperand(1).getValueType() != N->getValueType(0))) { 9938 PromOps.insert(PromOps.begin(), PromOp); 9939 continue; 9940 } 9941 } 9942 9943 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 9944 PromOp.getNode()->op_end()); 9945 9946 // If this node has constant inputs, then they'll need to be promoted here. 9947 for (unsigned i = 0; i < 2; ++i) { 9948 if (!isa<ConstantSDNode>(Ops[C+i])) 9949 continue; 9950 if (Ops[C+i].getValueType() == N->getValueType(0)) 9951 continue; 9952 9953 if (N->getOpcode() == ISD::SIGN_EXTEND) 9954 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 9955 else if (N->getOpcode() == ISD::ZERO_EXTEND) 9956 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 9957 else 9958 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 9959 } 9960 9961 // If we've promoted the comparison inputs of a SELECT or SELECT_CC, 9962 // truncate them again to the original value type. 9963 if (PromOp.getOpcode() == ISD::SELECT || 9964 PromOp.getOpcode() == ISD::SELECT_CC) { 9965 auto SI0 = SelectTruncOp[0].find(PromOp.getNode()); 9966 if (SI0 != SelectTruncOp[0].end()) 9967 Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]); 9968 auto SI1 = SelectTruncOp[1].find(PromOp.getNode()); 9969 if (SI1 != SelectTruncOp[1].end()) 9970 Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]); 9971 } 9972 9973 DAG.ReplaceAllUsesOfValueWith(PromOp, 9974 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); 9975 } 9976 9977 // Now we're left with the initial extension itself. 9978 if (!ReallyNeedsExt) 9979 return N->getOperand(0); 9980 9981 // To zero extend, just mask off everything except for the first bit (in the 9982 // i1 case). 9983 if (N->getOpcode() == ISD::ZERO_EXTEND) 9984 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), 9985 DAG.getConstant(APInt::getLowBitsSet( 9986 N->getValueSizeInBits(0), PromBits), 9987 dl, N->getValueType(0))); 9988 9989 assert(N->getOpcode() == ISD::SIGN_EXTEND && 9990 "Invalid extension type"); 9991 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout()); 9992 SDValue ShiftCst = 9993 DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy); 9994 return DAG.getNode( 9995 ISD::SRA, dl, N->getValueType(0), 9996 DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst), 9997 ShiftCst); 9998 } 9999 10000 SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, 10001 DAGCombinerInfo &DCI) const { 10002 assert((N->getOpcode() == ISD::SINT_TO_FP || 10003 N->getOpcode() == ISD::UINT_TO_FP) && 10004 "Need an int -> FP conversion node here"); 10005 10006 if (!Subtarget.has64BitSupport()) 10007 return SDValue(); 10008 10009 SelectionDAG &DAG = DCI.DAG; 10010 SDLoc dl(N); 10011 SDValue Op(N, 0); 10012 10013 // Don't handle ppc_fp128 here or i1 conversions. 10014 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 10015 return SDValue(); 10016 if (Op.getOperand(0).getValueType() == MVT::i1) 10017 return SDValue(); 10018 10019 // For i32 intermediate values, unfortunately, the conversion functions 10020 // leave the upper 32 bits of the value are undefined. Within the set of 10021 // scalar instructions, we have no method for zero- or sign-extending the 10022 // value. Thus, we cannot handle i32 intermediate values here. 10023 if (Op.getOperand(0).getValueType() == MVT::i32) 10024 return SDValue(); 10025 10026 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 10027 "UINT_TO_FP is supported only with FPCVT"); 10028 10029 // If we have FCFIDS, then use it when converting to single-precision. 10030 // Otherwise, convert to double-precision and then round. 10031 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 10032 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 10033 : PPCISD::FCFIDS) 10034 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 10035 : PPCISD::FCFID); 10036 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 10037 ? MVT::f32 10038 : MVT::f64; 10039 10040 // If we're converting from a float, to an int, and back to a float again, 10041 // then we don't need the store/load pair at all. 10042 if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT && 10043 Subtarget.hasFPCVT()) || 10044 (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) { 10045 SDValue Src = Op.getOperand(0).getOperand(0); 10046 if (Src.getValueType() == MVT::f32) { 10047 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 10048 DCI.AddToWorklist(Src.getNode()); 10049 } else if (Src.getValueType() != MVT::f64) { 10050 // Make sure that we don't pick up a ppc_fp128 source value. 10051 return SDValue(); 10052 } 10053 10054 unsigned FCTOp = 10055 Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 10056 PPCISD::FCTIDUZ; 10057 10058 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src); 10059 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp); 10060 10061 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { 10062 FP = DAG.getNode(ISD::FP_ROUND, dl, 10063 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 10064 DCI.AddToWorklist(FP.getNode()); 10065 } 10066 10067 return FP; 10068 } 10069 10070 return SDValue(); 10071 } 10072 10073 // expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for 10074 // builtins) into loads with swaps. 10075 SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N, 10076 DAGCombinerInfo &DCI) const { 10077 SelectionDAG &DAG = DCI.DAG; 10078 SDLoc dl(N); 10079 SDValue Chain; 10080 SDValue Base; 10081 MachineMemOperand *MMO; 10082 10083 switch (N->getOpcode()) { 10084 default: 10085 llvm_unreachable("Unexpected opcode for little endian VSX load"); 10086 case ISD::LOAD: { 10087 LoadSDNode *LD = cast<LoadSDNode>(N); 10088 Chain = LD->getChain(); 10089 Base = LD->getBasePtr(); 10090 MMO = LD->getMemOperand(); 10091 // If the MMO suggests this isn't a load of a full vector, leave 10092 // things alone. For a built-in, we have to make the change for 10093 // correctness, so if there is a size problem that will be a bug. 10094 if (MMO->getSize() < 16) 10095 return SDValue(); 10096 break; 10097 } 10098 case ISD::INTRINSIC_W_CHAIN: { 10099 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 10100 Chain = Intrin->getChain(); 10101 // Similarly to the store case below, Intrin->getBasePtr() doesn't get 10102 // us what we want. Get operand 2 instead. 10103 Base = Intrin->getOperand(2); 10104 MMO = Intrin->getMemOperand(); 10105 break; 10106 } 10107 } 10108 10109 MVT VecTy = N->getValueType(0).getSimpleVT(); 10110 SDValue LoadOps[] = { Chain, Base }; 10111 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl, 10112 DAG.getVTList(VecTy, MVT::Other), 10113 LoadOps, VecTy, MMO); 10114 DCI.AddToWorklist(Load.getNode()); 10115 Chain = Load.getValue(1); 10116 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, 10117 DAG.getVTList(VecTy, MVT::Other), Chain, Load); 10118 DCI.AddToWorklist(Swap.getNode()); 10119 return Swap; 10120 } 10121 10122 // expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for 10123 // builtins) into stores with swaps. 10124 SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N, 10125 DAGCombinerInfo &DCI) const { 10126 SelectionDAG &DAG = DCI.DAG; 10127 SDLoc dl(N); 10128 SDValue Chain; 10129 SDValue Base; 10130 unsigned SrcOpnd; 10131 MachineMemOperand *MMO; 10132 10133 switch (N->getOpcode()) { 10134 default: 10135 llvm_unreachable("Unexpected opcode for little endian VSX store"); 10136 case ISD::STORE: { 10137 StoreSDNode *ST = cast<StoreSDNode>(N); 10138 Chain = ST->getChain(); 10139 Base = ST->getBasePtr(); 10140 MMO = ST->getMemOperand(); 10141 SrcOpnd = 1; 10142 // If the MMO suggests this isn't a store of a full vector, leave 10143 // things alone. For a built-in, we have to make the change for 10144 // correctness, so if there is a size problem that will be a bug. 10145 if (MMO->getSize() < 16) 10146 return SDValue(); 10147 break; 10148 } 10149 case ISD::INTRINSIC_VOID: { 10150 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 10151 Chain = Intrin->getChain(); 10152 // Intrin->getBasePtr() oddly does not get what we want. 10153 Base = Intrin->getOperand(3); 10154 MMO = Intrin->getMemOperand(); 10155 SrcOpnd = 2; 10156 break; 10157 } 10158 } 10159 10160 SDValue Src = N->getOperand(SrcOpnd); 10161 MVT VecTy = Src.getValueType().getSimpleVT(); 10162 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, 10163 DAG.getVTList(VecTy, MVT::Other), Chain, Src); 10164 DCI.AddToWorklist(Swap.getNode()); 10165 Chain = Swap.getValue(1); 10166 SDValue StoreOps[] = { Chain, Swap, Base }; 10167 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl, 10168 DAG.getVTList(MVT::Other), 10169 StoreOps, VecTy, MMO); 10170 DCI.AddToWorklist(Store.getNode()); 10171 return Store; 10172 } 10173 10174 SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, 10175 DAGCombinerInfo &DCI) const { 10176 SelectionDAG &DAG = DCI.DAG; 10177 SDLoc dl(N); 10178 switch (N->getOpcode()) { 10179 default: break; 10180 case PPCISD::SHL: 10181 if (isNullConstant(N->getOperand(0))) // 0 << V -> 0. 10182 return N->getOperand(0); 10183 break; 10184 case PPCISD::SRL: 10185 if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0. 10186 return N->getOperand(0); 10187 break; 10188 case PPCISD::SRA: 10189 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 10190 if (C->isNullValue() || // 0 >>s V -> 0. 10191 C->isAllOnesValue()) // -1 >>s V -> -1. 10192 return N->getOperand(0); 10193 } 10194 break; 10195 case ISD::SIGN_EXTEND: 10196 case ISD::ZERO_EXTEND: 10197 case ISD::ANY_EXTEND: 10198 return DAGCombineExtBoolTrunc(N, DCI); 10199 case ISD::TRUNCATE: 10200 case ISD::SETCC: 10201 case ISD::SELECT_CC: 10202 return DAGCombineTruncBoolExt(N, DCI); 10203 case ISD::SINT_TO_FP: 10204 case ISD::UINT_TO_FP: 10205 return combineFPToIntToFP(N, DCI); 10206 case ISD::STORE: { 10207 // Turn STORE (FP_TO_SINT F) -> STFIWX(FCTIWZ(F)). 10208 if (Subtarget.hasSTFIWX() && !cast<StoreSDNode>(N)->isTruncatingStore() && 10209 N->getOperand(1).getOpcode() == ISD::FP_TO_SINT && 10210 N->getOperand(1).getValueType() == MVT::i32 && 10211 N->getOperand(1).getOperand(0).getValueType() != MVT::ppcf128) { 10212 SDValue Val = N->getOperand(1).getOperand(0); 10213 if (Val.getValueType() == MVT::f32) { 10214 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); 10215 DCI.AddToWorklist(Val.getNode()); 10216 } 10217 Val = DAG.getNode(PPCISD::FCTIWZ, dl, MVT::f64, Val); 10218 DCI.AddToWorklist(Val.getNode()); 10219 10220 SDValue Ops[] = { 10221 N->getOperand(0), Val, N->getOperand(2), 10222 DAG.getValueType(N->getOperand(1).getValueType()) 10223 }; 10224 10225 Val = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 10226 DAG.getVTList(MVT::Other), Ops, 10227 cast<StoreSDNode>(N)->getMemoryVT(), 10228 cast<StoreSDNode>(N)->getMemOperand()); 10229 DCI.AddToWorklist(Val.getNode()); 10230 return Val; 10231 } 10232 10233 // Turn STORE (BSWAP) -> sthbrx/stwbrx. 10234 if (cast<StoreSDNode>(N)->isUnindexed() && 10235 N->getOperand(1).getOpcode() == ISD::BSWAP && 10236 N->getOperand(1).getNode()->hasOneUse() && 10237 (N->getOperand(1).getValueType() == MVT::i32 || 10238 N->getOperand(1).getValueType() == MVT::i16 || 10239 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && 10240 N->getOperand(1).getValueType() == MVT::i64))) { 10241 SDValue BSwapOp = N->getOperand(1).getOperand(0); 10242 // Do an any-extend to 32-bits if this is a half-word input. 10243 if (BSwapOp.getValueType() == MVT::i16) 10244 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); 10245 10246 SDValue Ops[] = { 10247 N->getOperand(0), BSwapOp, N->getOperand(2), 10248 DAG.getValueType(N->getOperand(1).getValueType()) 10249 }; 10250 return 10251 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), 10252 Ops, cast<StoreSDNode>(N)->getMemoryVT(), 10253 cast<StoreSDNode>(N)->getMemOperand()); 10254 } 10255 10256 // For little endian, VSX stores require generating xxswapd/lxvd2x. 10257 EVT VT = N->getOperand(1).getValueType(); 10258 if (VT.isSimple()) { 10259 MVT StoreVT = VT.getSimpleVT(); 10260 if (Subtarget.hasVSX() && Subtarget.isLittleEndian() && 10261 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 || 10262 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32)) 10263 return expandVSXStoreForLE(N, DCI); 10264 } 10265 break; 10266 } 10267 case ISD::LOAD: { 10268 LoadSDNode *LD = cast<LoadSDNode>(N); 10269 EVT VT = LD->getValueType(0); 10270 10271 // For little endian, VSX loads require generating lxvd2x/xxswapd. 10272 if (VT.isSimple()) { 10273 MVT LoadVT = VT.getSimpleVT(); 10274 if (Subtarget.hasVSX() && Subtarget.isLittleEndian() && 10275 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 || 10276 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32)) 10277 return expandVSXLoadForLE(N, DCI); 10278 } 10279 10280 EVT MemVT = LD->getMemoryVT(); 10281 Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); 10282 unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty); 10283 Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext()); 10284 unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy); 10285 if (LD->isUnindexed() && VT.isVector() && 10286 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && 10287 // P8 and later hardware should just use LOAD. 10288 !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 || 10289 VT == MVT::v4i32 || VT == MVT::v4f32)) || 10290 (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) && 10291 LD->getAlignment() >= ScalarABIAlignment)) && 10292 LD->getAlignment() < ABIAlignment) { 10293 // This is a type-legal unaligned Altivec or QPX load. 10294 SDValue Chain = LD->getChain(); 10295 SDValue Ptr = LD->getBasePtr(); 10296 bool isLittleEndian = Subtarget.isLittleEndian(); 10297 10298 // This implements the loading of unaligned vectors as described in 10299 // the venerable Apple Velocity Engine overview. Specifically: 10300 // https://developer.apple.com/hardwaredrivers/ve/alignment.html 10301 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html 10302 // 10303 // The general idea is to expand a sequence of one or more unaligned 10304 // loads into an alignment-based permutation-control instruction (lvsl 10305 // or lvsr), a series of regular vector loads (which always truncate 10306 // their input address to an aligned address), and a series of 10307 // permutations. The results of these permutations are the requested 10308 // loaded values. The trick is that the last "extra" load is not taken 10309 // from the address you might suspect (sizeof(vector) bytes after the 10310 // last requested load), but rather sizeof(vector) - 1 bytes after the 10311 // last requested vector. The point of this is to avoid a page fault if 10312 // the base address happened to be aligned. This works because if the 10313 // base address is aligned, then adding less than a full vector length 10314 // will cause the last vector in the sequence to be (re)loaded. 10315 // Otherwise, the next vector will be fetched as you might suspect was 10316 // necessary. 10317 10318 // We might be able to reuse the permutation generation from 10319 // a different base address offset from this one by an aligned amount. 10320 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this 10321 // optimization later. 10322 Intrinsic::ID Intr, IntrLD, IntrPerm; 10323 MVT PermCntlTy, PermTy, LDTy; 10324 if (Subtarget.hasAltivec()) { 10325 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr : 10326 Intrinsic::ppc_altivec_lvsl; 10327 IntrLD = Intrinsic::ppc_altivec_lvx; 10328 IntrPerm = Intrinsic::ppc_altivec_vperm; 10329 PermCntlTy = MVT::v16i8; 10330 PermTy = MVT::v4i32; 10331 LDTy = MVT::v4i32; 10332 } else { 10333 Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld : 10334 Intrinsic::ppc_qpx_qvlpcls; 10335 IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd : 10336 Intrinsic::ppc_qpx_qvlfs; 10337 IntrPerm = Intrinsic::ppc_qpx_qvfperm; 10338 PermCntlTy = MVT::v4f64; 10339 PermTy = MVT::v4f64; 10340 LDTy = MemVT.getSimpleVT(); 10341 } 10342 10343 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); 10344 10345 // Create the new MMO for the new base load. It is like the original MMO, 10346 // but represents an area in memory almost twice the vector size centered 10347 // on the original address. If the address is unaligned, we might start 10348 // reading up to (sizeof(vector)-1) bytes below the address of the 10349 // original unaligned load. 10350 MachineFunction &MF = DAG.getMachineFunction(); 10351 MachineMemOperand *BaseMMO = 10352 MF.getMachineMemOperand(LD->getMemOperand(), 10353 -(long)MemVT.getStoreSize()+1, 10354 2*MemVT.getStoreSize()-1); 10355 10356 // Create the new base load. 10357 SDValue LDXIntID = 10358 DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout())); 10359 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr }; 10360 SDValue BaseLoad = 10361 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 10362 DAG.getVTList(PermTy, MVT::Other), 10363 BaseLoadOps, LDTy, BaseMMO); 10364 10365 // Note that the value of IncOffset (which is provided to the next 10366 // load's pointer info offset value, and thus used to calculate the 10367 // alignment), and the value of IncValue (which is actually used to 10368 // increment the pointer value) are different! This is because we 10369 // require the next load to appear to be aligned, even though it 10370 // is actually offset from the base pointer by a lesser amount. 10371 int IncOffset = VT.getSizeInBits() / 8; 10372 int IncValue = IncOffset; 10373 10374 // Walk (both up and down) the chain looking for another load at the real 10375 // (aligned) offset (the alignment of the other load does not matter in 10376 // this case). If found, then do not use the offset reduction trick, as 10377 // that will prevent the loads from being later combined (as they would 10378 // otherwise be duplicates). 10379 if (!findConsecutiveLoad(LD, DAG)) 10380 --IncValue; 10381 10382 SDValue Increment = 10383 DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout())); 10384 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 10385 10386 MachineMemOperand *ExtraMMO = 10387 MF.getMachineMemOperand(LD->getMemOperand(), 10388 1, 2*MemVT.getStoreSize()-1); 10389 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr }; 10390 SDValue ExtraLoad = 10391 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 10392 DAG.getVTList(PermTy, MVT::Other), 10393 ExtraLoadOps, LDTy, ExtraMMO); 10394 10395 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 10396 BaseLoad.getValue(1), ExtraLoad.getValue(1)); 10397 10398 // Because vperm has a big-endian bias, we must reverse the order 10399 // of the input vectors and complement the permute control vector 10400 // when generating little endian code. We have already handled the 10401 // latter by using lvsr instead of lvsl, so just reverse BaseLoad 10402 // and ExtraLoad here. 10403 SDValue Perm; 10404 if (isLittleEndian) 10405 Perm = BuildIntrinsicOp(IntrPerm, 10406 ExtraLoad, BaseLoad, PermCntl, DAG, dl); 10407 else 10408 Perm = BuildIntrinsicOp(IntrPerm, 10409 BaseLoad, ExtraLoad, PermCntl, DAG, dl); 10410 10411 if (VT != PermTy) 10412 Perm = Subtarget.hasAltivec() ? 10413 DAG.getNode(ISD::BITCAST, dl, VT, Perm) : 10414 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX 10415 DAG.getTargetConstant(1, dl, MVT::i64)); 10416 // second argument is 1 because this rounding 10417 // is always exact. 10418 10419 // The output of the permutation is our loaded result, the TokenFactor is 10420 // our new chain. 10421 DCI.CombineTo(N, Perm, TF); 10422 return SDValue(N, 0); 10423 } 10424 } 10425 break; 10426 case ISD::INTRINSIC_WO_CHAIN: { 10427 bool isLittleEndian = Subtarget.isLittleEndian(); 10428 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 10429 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr 10430 : Intrinsic::ppc_altivec_lvsl); 10431 if ((IID == Intr || 10432 IID == Intrinsic::ppc_qpx_qvlpcld || 10433 IID == Intrinsic::ppc_qpx_qvlpcls) && 10434 N->getOperand(1)->getOpcode() == ISD::ADD) { 10435 SDValue Add = N->getOperand(1); 10436 10437 int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ? 10438 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */; 10439 10440 if (DAG.MaskedValueIsZero( 10441 Add->getOperand(1), 10442 APInt::getAllOnesValue(Bits /* alignment */) 10443 .zext( 10444 Add.getValueType().getScalarType().getSizeInBits()))) { 10445 SDNode *BasePtr = Add->getOperand(0).getNode(); 10446 for (SDNode::use_iterator UI = BasePtr->use_begin(), 10447 UE = BasePtr->use_end(); 10448 UI != UE; ++UI) { 10449 if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 10450 cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) { 10451 // We've found another LVSL/LVSR, and this address is an aligned 10452 // multiple of that one. The results will be the same, so use the 10453 // one we've just found instead. 10454 10455 return SDValue(*UI, 0); 10456 } 10457 } 10458 } 10459 10460 if (isa<ConstantSDNode>(Add->getOperand(1))) { 10461 SDNode *BasePtr = Add->getOperand(0).getNode(); 10462 for (SDNode::use_iterator UI = BasePtr->use_begin(), 10463 UE = BasePtr->use_end(); UI != UE; ++UI) { 10464 if (UI->getOpcode() == ISD::ADD && 10465 isa<ConstantSDNode>(UI->getOperand(1)) && 10466 (cast<ConstantSDNode>(Add->getOperand(1))->getZExtValue() - 10467 cast<ConstantSDNode>(UI->getOperand(1))->getZExtValue()) % 10468 (1ULL << Bits) == 0) { 10469 SDNode *OtherAdd = *UI; 10470 for (SDNode::use_iterator VI = OtherAdd->use_begin(), 10471 VE = OtherAdd->use_end(); VI != VE; ++VI) { 10472 if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 10473 cast<ConstantSDNode>(VI->getOperand(0))->getZExtValue() == IID) { 10474 return SDValue(*VI, 0); 10475 } 10476 } 10477 } 10478 } 10479 } 10480 } 10481 } 10482 10483 break; 10484 case ISD::INTRINSIC_W_CHAIN: { 10485 // For little endian, VSX loads require generating lxvd2x/xxswapd. 10486 if (Subtarget.hasVSX() && Subtarget.isLittleEndian()) { 10487 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 10488 default: 10489 break; 10490 case Intrinsic::ppc_vsx_lxvw4x: 10491 case Intrinsic::ppc_vsx_lxvd2x: 10492 return expandVSXLoadForLE(N, DCI); 10493 } 10494 } 10495 break; 10496 } 10497 case ISD::INTRINSIC_VOID: { 10498 // For little endian, VSX stores require generating xxswapd/stxvd2x. 10499 if (Subtarget.hasVSX() && Subtarget.isLittleEndian()) { 10500 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 10501 default: 10502 break; 10503 case Intrinsic::ppc_vsx_stxvw4x: 10504 case Intrinsic::ppc_vsx_stxvd2x: 10505 return expandVSXStoreForLE(N, DCI); 10506 } 10507 } 10508 break; 10509 } 10510 case ISD::BSWAP: 10511 // Turn BSWAP (LOAD) -> lhbrx/lwbrx. 10512 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && 10513 N->getOperand(0).hasOneUse() && 10514 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || 10515 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && 10516 N->getValueType(0) == MVT::i64))) { 10517 SDValue Load = N->getOperand(0); 10518 LoadSDNode *LD = cast<LoadSDNode>(Load); 10519 // Create the byte-swapping load. 10520 SDValue Ops[] = { 10521 LD->getChain(), // Chain 10522 LD->getBasePtr(), // Ptr 10523 DAG.getValueType(N->getValueType(0)) // VT 10524 }; 10525 SDValue BSLoad = 10526 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, 10527 DAG.getVTList(N->getValueType(0) == MVT::i64 ? 10528 MVT::i64 : MVT::i32, MVT::Other), 10529 Ops, LD->getMemoryVT(), LD->getMemOperand()); 10530 10531 // If this is an i16 load, insert the truncate. 10532 SDValue ResVal = BSLoad; 10533 if (N->getValueType(0) == MVT::i16) 10534 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); 10535 10536 // First, combine the bswap away. This makes the value produced by the 10537 // load dead. 10538 DCI.CombineTo(N, ResVal); 10539 10540 // Next, combine the load away, we give it a bogus result value but a real 10541 // chain result. The result value is dead because the bswap is dead. 10542 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); 10543 10544 // Return N so it doesn't get rechecked! 10545 return SDValue(N, 0); 10546 } 10547 10548 break; 10549 case PPCISD::VCMP: { 10550 // If a VCMPo node already exists with exactly the same operands as this 10551 // node, use its result instead of this node (VCMPo computes both a CR6 and 10552 // a normal output). 10553 // 10554 if (!N->getOperand(0).hasOneUse() && 10555 !N->getOperand(1).hasOneUse() && 10556 !N->getOperand(2).hasOneUse()) { 10557 10558 // Scan all of the users of the LHS, looking for VCMPo's that match. 10559 SDNode *VCMPoNode = nullptr; 10560 10561 SDNode *LHSN = N->getOperand(0).getNode(); 10562 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); 10563 UI != E; ++UI) 10564 if (UI->getOpcode() == PPCISD::VCMPo && 10565 UI->getOperand(1) == N->getOperand(1) && 10566 UI->getOperand(2) == N->getOperand(2) && 10567 UI->getOperand(0) == N->getOperand(0)) { 10568 VCMPoNode = *UI; 10569 break; 10570 } 10571 10572 // If there is no VCMPo node, or if the flag value has a single use, don't 10573 // transform this. 10574 if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) 10575 break; 10576 10577 // Look at the (necessarily single) use of the flag value. If it has a 10578 // chain, this transformation is more complex. Note that multiple things 10579 // could use the value result, which we should ignore. 10580 SDNode *FlagUser = nullptr; 10581 for (SDNode::use_iterator UI = VCMPoNode->use_begin(); 10582 FlagUser == nullptr; ++UI) { 10583 assert(UI != VCMPoNode->use_end() && "Didn't find user!"); 10584 SDNode *User = *UI; 10585 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 10586 if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { 10587 FlagUser = User; 10588 break; 10589 } 10590 } 10591 } 10592 10593 // If the user is a MFOCRF instruction, we know this is safe. 10594 // Otherwise we give up for right now. 10595 if (FlagUser->getOpcode() == PPCISD::MFOCRF) 10596 return SDValue(VCMPoNode, 0); 10597 } 10598 break; 10599 } 10600 case ISD::BRCOND: { 10601 SDValue Cond = N->getOperand(1); 10602 SDValue Target = N->getOperand(2); 10603 10604 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && 10605 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() == 10606 Intrinsic::ppc_is_decremented_ctr_nonzero) { 10607 10608 // We now need to make the intrinsic dead (it cannot be instruction 10609 // selected). 10610 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); 10611 assert(Cond.getNode()->hasOneUse() && 10612 "Counter decrement has more than one use"); 10613 10614 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, 10615 N->getOperand(0), Target); 10616 } 10617 } 10618 break; 10619 case ISD::BR_CC: { 10620 // If this is a branch on an altivec predicate comparison, lower this so 10621 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This 10622 // lowering is done pre-legalize, because the legalizer lowers the predicate 10623 // compare down to code that is difficult to reassemble. 10624 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 10625 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); 10626 10627 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero 10628 // value. If so, pass-through the AND to get to the intrinsic. 10629 if (LHS.getOpcode() == ISD::AND && 10630 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && 10631 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() == 10632 Intrinsic::ppc_is_decremented_ctr_nonzero && 10633 isa<ConstantSDNode>(LHS.getOperand(1)) && 10634 !isNullConstant(LHS.getOperand(1))) 10635 LHS = LHS.getOperand(0); 10636 10637 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && 10638 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() == 10639 Intrinsic::ppc_is_decremented_ctr_nonzero && 10640 isa<ConstantSDNode>(RHS)) { 10641 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && 10642 "Counter decrement comparison is not EQ or NE"); 10643 10644 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 10645 bool isBDNZ = (CC == ISD::SETEQ && Val) || 10646 (CC == ISD::SETNE && !Val); 10647 10648 // We now need to make the intrinsic dead (it cannot be instruction 10649 // selected). 10650 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); 10651 assert(LHS.getNode()->hasOneUse() && 10652 "Counter decrement has more than one use"); 10653 10654 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, 10655 N->getOperand(0), N->getOperand(4)); 10656 } 10657 10658 int CompareOpc; 10659 bool isDot; 10660 10661 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 10662 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && 10663 getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) { 10664 assert(isDot && "Can't compare against a vector result!"); 10665 10666 // If this is a comparison against something other than 0/1, then we know 10667 // that the condition is never/always true. 10668 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 10669 if (Val != 0 && Val != 1) { 10670 if (CC == ISD::SETEQ) // Cond never true, remove branch. 10671 return N->getOperand(0); 10672 // Always !=, turn it into an unconditional branch. 10673 return DAG.getNode(ISD::BR, dl, MVT::Other, 10674 N->getOperand(0), N->getOperand(4)); 10675 } 10676 10677 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); 10678 10679 // Create the PPCISD altivec 'dot' comparison node. 10680 SDValue Ops[] = { 10681 LHS.getOperand(2), // LHS of compare 10682 LHS.getOperand(3), // RHS of compare 10683 DAG.getConstant(CompareOpc, dl, MVT::i32) 10684 }; 10685 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; 10686 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 10687 10688 // Unpack the result based on how the target uses it. 10689 PPC::Predicate CompOpc; 10690 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) { 10691 default: // Can't happen, don't crash on invalid number though. 10692 case 0: // Branch on the value of the EQ bit of CR6. 10693 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; 10694 break; 10695 case 1: // Branch on the inverted value of the EQ bit of CR6. 10696 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; 10697 break; 10698 case 2: // Branch on the value of the LT bit of CR6. 10699 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; 10700 break; 10701 case 3: // Branch on the inverted value of the LT bit of CR6. 10702 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; 10703 break; 10704 } 10705 10706 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), 10707 DAG.getConstant(CompOpc, dl, MVT::i32), 10708 DAG.getRegister(PPC::CR6, MVT::i32), 10709 N->getOperand(4), CompNode.getValue(1)); 10710 } 10711 break; 10712 } 10713 } 10714 10715 return SDValue(); 10716 } 10717 10718 SDValue 10719 PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, 10720 SelectionDAG &DAG, 10721 std::vector<SDNode *> *Created) const { 10722 // fold (sdiv X, pow2) 10723 EVT VT = N->getValueType(0); 10724 if (VT == MVT::i64 && !Subtarget.isPPC64()) 10725 return SDValue(); 10726 if ((VT != MVT::i32 && VT != MVT::i64) || 10727 !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2())) 10728 return SDValue(); 10729 10730 SDLoc DL(N); 10731 SDValue N0 = N->getOperand(0); 10732 10733 bool IsNegPow2 = (-Divisor).isPowerOf2(); 10734 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros(); 10735 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT); 10736 10737 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt); 10738 if (Created) 10739 Created->push_back(Op.getNode()); 10740 10741 if (IsNegPow2) { 10742 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); 10743 if (Created) 10744 Created->push_back(Op.getNode()); 10745 } 10746 10747 return Op; 10748 } 10749 10750 //===----------------------------------------------------------------------===// 10751 // Inline Assembly Support 10752 //===----------------------------------------------------------------------===// 10753 10754 void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, 10755 APInt &KnownZero, 10756 APInt &KnownOne, 10757 const SelectionDAG &DAG, 10758 unsigned Depth) const { 10759 KnownZero = KnownOne = APInt(KnownZero.getBitWidth(), 0); 10760 switch (Op.getOpcode()) { 10761 default: break; 10762 case PPCISD::LBRX: { 10763 // lhbrx is known to have the top bits cleared out. 10764 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16) 10765 KnownZero = 0xFFFF0000; 10766 break; 10767 } 10768 case ISD::INTRINSIC_WO_CHAIN: { 10769 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 10770 default: break; 10771 case Intrinsic::ppc_altivec_vcmpbfp_p: 10772 case Intrinsic::ppc_altivec_vcmpeqfp_p: 10773 case Intrinsic::ppc_altivec_vcmpequb_p: 10774 case Intrinsic::ppc_altivec_vcmpequh_p: 10775 case Intrinsic::ppc_altivec_vcmpequw_p: 10776 case Intrinsic::ppc_altivec_vcmpequd_p: 10777 case Intrinsic::ppc_altivec_vcmpgefp_p: 10778 case Intrinsic::ppc_altivec_vcmpgtfp_p: 10779 case Intrinsic::ppc_altivec_vcmpgtsb_p: 10780 case Intrinsic::ppc_altivec_vcmpgtsh_p: 10781 case Intrinsic::ppc_altivec_vcmpgtsw_p: 10782 case Intrinsic::ppc_altivec_vcmpgtsd_p: 10783 case Intrinsic::ppc_altivec_vcmpgtub_p: 10784 case Intrinsic::ppc_altivec_vcmpgtuh_p: 10785 case Intrinsic::ppc_altivec_vcmpgtuw_p: 10786 case Intrinsic::ppc_altivec_vcmpgtud_p: 10787 KnownZero = ~1U; // All bits but the low one are known to be zero. 10788 break; 10789 } 10790 } 10791 } 10792 } 10793 10794 unsigned PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { 10795 switch (Subtarget.getDarwinDirective()) { 10796 default: break; 10797 case PPC::DIR_970: 10798 case PPC::DIR_PWR4: 10799 case PPC::DIR_PWR5: 10800 case PPC::DIR_PWR5X: 10801 case PPC::DIR_PWR6: 10802 case PPC::DIR_PWR6X: 10803 case PPC::DIR_PWR7: 10804 case PPC::DIR_PWR8: { 10805 if (!ML) 10806 break; 10807 10808 const PPCInstrInfo *TII = Subtarget.getInstrInfo(); 10809 10810 // For small loops (between 5 and 8 instructions), align to a 32-byte 10811 // boundary so that the entire loop fits in one instruction-cache line. 10812 uint64_t LoopSize = 0; 10813 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I) 10814 for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { 10815 LoopSize += TII->GetInstSizeInBytes(J); 10816 if (LoopSize > 32) 10817 break; 10818 } 10819 10820 if (LoopSize > 16 && LoopSize <= 32) 10821 return 5; 10822 10823 break; 10824 } 10825 } 10826 10827 return TargetLowering::getPrefLoopAlignment(ML); 10828 } 10829 10830 /// getConstraintType - Given a constraint, return the type of 10831 /// constraint it is for this target. 10832 PPCTargetLowering::ConstraintType 10833 PPCTargetLowering::getConstraintType(StringRef Constraint) const { 10834 if (Constraint.size() == 1) { 10835 switch (Constraint[0]) { 10836 default: break; 10837 case 'b': 10838 case 'r': 10839 case 'f': 10840 case 'v': 10841 case 'y': 10842 return C_RegisterClass; 10843 case 'Z': 10844 // FIXME: While Z does indicate a memory constraint, it specifically 10845 // indicates an r+r address (used in conjunction with the 'y' modifier 10846 // in the replacement string). Currently, we're forcing the base 10847 // register to be r0 in the asm printer (which is interpreted as zero) 10848 // and forming the complete address in the second register. This is 10849 // suboptimal. 10850 return C_Memory; 10851 } 10852 } else if (Constraint == "wc") { // individual CR bits. 10853 return C_RegisterClass; 10854 } else if (Constraint == "wa" || Constraint == "wd" || 10855 Constraint == "wf" || Constraint == "ws") { 10856 return C_RegisterClass; // VSX registers. 10857 } 10858 return TargetLowering::getConstraintType(Constraint); 10859 } 10860 10861 /// Examine constraint type and operand type and determine a weight value. 10862 /// This object must already have been set up with the operand type 10863 /// and the current alternative constraint selected. 10864 TargetLowering::ConstraintWeight 10865 PPCTargetLowering::getSingleConstraintMatchWeight( 10866 AsmOperandInfo &info, const char *constraint) const { 10867 ConstraintWeight weight = CW_Invalid; 10868 Value *CallOperandVal = info.CallOperandVal; 10869 // If we don't have a value, we can't do a match, 10870 // but allow it at the lowest weight. 10871 if (!CallOperandVal) 10872 return CW_Default; 10873 Type *type = CallOperandVal->getType(); 10874 10875 // Look at the constraint type. 10876 if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) 10877 return CW_Register; // an individual CR bit. 10878 else if ((StringRef(constraint) == "wa" || 10879 StringRef(constraint) == "wd" || 10880 StringRef(constraint) == "wf") && 10881 type->isVectorTy()) 10882 return CW_Register; 10883 else if (StringRef(constraint) == "ws" && type->isDoubleTy()) 10884 return CW_Register; 10885 10886 switch (*constraint) { 10887 default: 10888 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 10889 break; 10890 case 'b': 10891 if (type->isIntegerTy()) 10892 weight = CW_Register; 10893 break; 10894 case 'f': 10895 if (type->isFloatTy()) 10896 weight = CW_Register; 10897 break; 10898 case 'd': 10899 if (type->isDoubleTy()) 10900 weight = CW_Register; 10901 break; 10902 case 'v': 10903 if (type->isVectorTy()) 10904 weight = CW_Register; 10905 break; 10906 case 'y': 10907 weight = CW_Register; 10908 break; 10909 case 'Z': 10910 weight = CW_Memory; 10911 break; 10912 } 10913 return weight; 10914 } 10915 10916 std::pair<unsigned, const TargetRegisterClass *> 10917 PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 10918 StringRef Constraint, 10919 MVT VT) const { 10920 if (Constraint.size() == 1) { 10921 // GCC RS6000 Constraint Letters 10922 switch (Constraint[0]) { 10923 case 'b': // R1-R31 10924 if (VT == MVT::i64 && Subtarget.isPPC64()) 10925 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); 10926 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); 10927 case 'r': // R0-R31 10928 if (VT == MVT::i64 && Subtarget.isPPC64()) 10929 return std::make_pair(0U, &PPC::G8RCRegClass); 10930 return std::make_pair(0U, &PPC::GPRCRegClass); 10931 case 'f': 10932 if (VT == MVT::f32 || VT == MVT::i32) 10933 return std::make_pair(0U, &PPC::F4RCRegClass); 10934 if (VT == MVT::f64 || VT == MVT::i64) 10935 return std::make_pair(0U, &PPC::F8RCRegClass); 10936 if (VT == MVT::v4f64 && Subtarget.hasQPX()) 10937 return std::make_pair(0U, &PPC::QFRCRegClass); 10938 if (VT == MVT::v4f32 && Subtarget.hasQPX()) 10939 return std::make_pair(0U, &PPC::QSRCRegClass); 10940 break; 10941 case 'v': 10942 if (VT == MVT::v4f64 && Subtarget.hasQPX()) 10943 return std::make_pair(0U, &PPC::QFRCRegClass); 10944 if (VT == MVT::v4f32 && Subtarget.hasQPX()) 10945 return std::make_pair(0U, &PPC::QSRCRegClass); 10946 if (Subtarget.hasAltivec()) 10947 return std::make_pair(0U, &PPC::VRRCRegClass); 10948 case 'y': // crrc 10949 return std::make_pair(0U, &PPC::CRRCRegClass); 10950 } 10951 } else if (Constraint == "wc" && Subtarget.useCRBits()) { 10952 // An individual CR bit. 10953 return std::make_pair(0U, &PPC::CRBITRCRegClass); 10954 } else if ((Constraint == "wa" || Constraint == "wd" || 10955 Constraint == "wf") && Subtarget.hasVSX()) { 10956 return std::make_pair(0U, &PPC::VSRCRegClass); 10957 } else if (Constraint == "ws" && Subtarget.hasVSX()) { 10958 if (VT == MVT::f32 && Subtarget.hasP8Vector()) 10959 return std::make_pair(0U, &PPC::VSSRCRegClass); 10960 else 10961 return std::make_pair(0U, &PPC::VSFRCRegClass); 10962 } 10963 10964 std::pair<unsigned, const TargetRegisterClass *> R = 10965 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 10966 10967 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers 10968 // (which we call X[0-9]+). If a 64-bit value has been requested, and a 10969 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent 10970 // register. 10971 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use 10972 // the AsmName field from *RegisterInfo.td, then this would not be necessary. 10973 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() && 10974 PPC::GPRCRegClass.contains(R.first)) 10975 return std::make_pair(TRI->getMatchingSuperReg(R.first, 10976 PPC::sub_32, &PPC::G8RCRegClass), 10977 &PPC::G8RCRegClass); 10978 10979 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. 10980 if (!R.second && StringRef("{cc}").equals_lower(Constraint)) { 10981 R.first = PPC::CR0; 10982 R.second = &PPC::CRRCRegClass; 10983 } 10984 10985 return R; 10986 } 10987 10988 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 10989 /// vector. If it is invalid, don't add anything to Ops. 10990 void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, 10991 std::string &Constraint, 10992 std::vector<SDValue>&Ops, 10993 SelectionDAG &DAG) const { 10994 SDValue Result; 10995 10996 // Only support length 1 constraints. 10997 if (Constraint.length() > 1) return; 10998 10999 char Letter = Constraint[0]; 11000 switch (Letter) { 11001 default: break; 11002 case 'I': 11003 case 'J': 11004 case 'K': 11005 case 'L': 11006 case 'M': 11007 case 'N': 11008 case 'O': 11009 case 'P': { 11010 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op); 11011 if (!CST) return; // Must be an immediate to match. 11012 SDLoc dl(Op); 11013 int64_t Value = CST->getSExtValue(); 11014 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative 11015 // numbers are printed as such. 11016 switch (Letter) { 11017 default: llvm_unreachable("Unknown constraint letter!"); 11018 case 'I': // "I" is a signed 16-bit constant. 11019 if (isInt<16>(Value)) 11020 Result = DAG.getTargetConstant(Value, dl, TCVT); 11021 break; 11022 case 'J': // "J" is a constant with only the high-order 16 bits nonzero. 11023 if (isShiftedUInt<16, 16>(Value)) 11024 Result = DAG.getTargetConstant(Value, dl, TCVT); 11025 break; 11026 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. 11027 if (isShiftedInt<16, 16>(Value)) 11028 Result = DAG.getTargetConstant(Value, dl, TCVT); 11029 break; 11030 case 'K': // "K" is a constant with only the low-order 16 bits nonzero. 11031 if (isUInt<16>(Value)) 11032 Result = DAG.getTargetConstant(Value, dl, TCVT); 11033 break; 11034 case 'M': // "M" is a constant that is greater than 31. 11035 if (Value > 31) 11036 Result = DAG.getTargetConstant(Value, dl, TCVT); 11037 break; 11038 case 'N': // "N" is a positive constant that is an exact power of two. 11039 if (Value > 0 && isPowerOf2_64(Value)) 11040 Result = DAG.getTargetConstant(Value, dl, TCVT); 11041 break; 11042 case 'O': // "O" is the constant zero. 11043 if (Value == 0) 11044 Result = DAG.getTargetConstant(Value, dl, TCVT); 11045 break; 11046 case 'P': // "P" is a constant whose negation is a signed 16-bit constant. 11047 if (isInt<16>(-Value)) 11048 Result = DAG.getTargetConstant(Value, dl, TCVT); 11049 break; 11050 } 11051 break; 11052 } 11053 } 11054 11055 if (Result.getNode()) { 11056 Ops.push_back(Result); 11057 return; 11058 } 11059 11060 // Handle standard constraint letters. 11061 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 11062 } 11063 11064 // isLegalAddressingMode - Return true if the addressing mode represented 11065 // by AM is legal for this target, for a load/store of the specified type. 11066 bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, 11067 const AddrMode &AM, Type *Ty, 11068 unsigned AS) const { 11069 // PPC does not allow r+i addressing modes for vectors! 11070 if (Ty->isVectorTy() && AM.BaseOffs != 0) 11071 return false; 11072 11073 // PPC allows a sign-extended 16-bit immediate field. 11074 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 11075 return false; 11076 11077 // No global is ever allowed as a base. 11078 if (AM.BaseGV) 11079 return false; 11080 11081 // PPC only support r+r, 11082 switch (AM.Scale) { 11083 case 0: // "r+i" or just "i", depending on HasBaseReg. 11084 break; 11085 case 1: 11086 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 11087 return false; 11088 // Otherwise we have r+r or r+i. 11089 break; 11090 case 2: 11091 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 11092 return false; 11093 // Allow 2*r as r+r. 11094 break; 11095 default: 11096 // No other scales are supported. 11097 return false; 11098 } 11099 11100 return true; 11101 } 11102 11103 SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, 11104 SelectionDAG &DAG) const { 11105 MachineFunction &MF = DAG.getMachineFunction(); 11106 MachineFrameInfo *MFI = MF.getFrameInfo(); 11107 MFI->setReturnAddressIsTaken(true); 11108 11109 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 11110 return SDValue(); 11111 11112 SDLoc dl(Op); 11113 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 11114 11115 // Make sure the function does not optimize away the store of the RA to 11116 // the stack. 11117 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 11118 FuncInfo->setLRStoreRequired(); 11119 bool isPPC64 = Subtarget.isPPC64(); 11120 auto PtrVT = getPointerTy(MF.getDataLayout()); 11121 11122 if (Depth > 0) { 11123 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 11124 SDValue Offset = 11125 DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl, 11126 isPPC64 ? MVT::i64 : MVT::i32); 11127 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), 11128 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), 11129 MachinePointerInfo(), false, false, false, 0); 11130 } 11131 11132 // Just load the return address off the stack. 11133 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); 11134 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, 11135 MachinePointerInfo(), false, false, false, 0); 11136 } 11137 11138 SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, 11139 SelectionDAG &DAG) const { 11140 SDLoc dl(Op); 11141 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 11142 11143 MachineFunction &MF = DAG.getMachineFunction(); 11144 MachineFrameInfo *MFI = MF.getFrameInfo(); 11145 MFI->setFrameAddressIsTaken(true); 11146 11147 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout()); 11148 bool isPPC64 = PtrVT == MVT::i64; 11149 11150 // Naked functions never have a frame pointer, and so we use r1. For all 11151 // other functions, this decision must be delayed until during PEI. 11152 unsigned FrameReg; 11153 if (MF.getFunction()->hasFnAttribute(Attribute::Naked)) 11154 FrameReg = isPPC64 ? PPC::X1 : PPC::R1; 11155 else 11156 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; 11157 11158 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, 11159 PtrVT); 11160 while (Depth--) 11161 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), 11162 FrameAddr, MachinePointerInfo(), false, false, 11163 false, 0); 11164 return FrameAddr; 11165 } 11166 11167 // FIXME? Maybe this could be a TableGen attribute on some registers and 11168 // this table could be generated automatically from RegInfo. 11169 unsigned PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT, 11170 SelectionDAG &DAG) const { 11171 bool isPPC64 = Subtarget.isPPC64(); 11172 bool isDarwinABI = Subtarget.isDarwinABI(); 11173 11174 if ((isPPC64 && VT != MVT::i64 && VT != MVT::i32) || 11175 (!isPPC64 && VT != MVT::i32)) 11176 report_fatal_error("Invalid register global variable type"); 11177 11178 bool is64Bit = isPPC64 && VT == MVT::i64; 11179 unsigned Reg = StringSwitch<unsigned>(RegName) 11180 .Case("r1", is64Bit ? PPC::X1 : PPC::R1) 11181 .Case("r2", (isDarwinABI || isPPC64) ? 0 : PPC::R2) 11182 .Case("r13", (!isPPC64 && isDarwinABI) ? 0 : 11183 (is64Bit ? PPC::X13 : PPC::R13)) 11184 .Default(0); 11185 11186 if (Reg) 11187 return Reg; 11188 report_fatal_error("Invalid register name global variable"); 11189 } 11190 11191 bool 11192 PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 11193 // The PowerPC target isn't yet aware of offsets. 11194 return false; 11195 } 11196 11197 bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 11198 const CallInst &I, 11199 unsigned Intrinsic) const { 11200 11201 switch (Intrinsic) { 11202 case Intrinsic::ppc_qpx_qvlfd: 11203 case Intrinsic::ppc_qpx_qvlfs: 11204 case Intrinsic::ppc_qpx_qvlfcd: 11205 case Intrinsic::ppc_qpx_qvlfcs: 11206 case Intrinsic::ppc_qpx_qvlfiwa: 11207 case Intrinsic::ppc_qpx_qvlfiwz: 11208 case Intrinsic::ppc_altivec_lvx: 11209 case Intrinsic::ppc_altivec_lvxl: 11210 case Intrinsic::ppc_altivec_lvebx: 11211 case Intrinsic::ppc_altivec_lvehx: 11212 case Intrinsic::ppc_altivec_lvewx: 11213 case Intrinsic::ppc_vsx_lxvd2x: 11214 case Intrinsic::ppc_vsx_lxvw4x: { 11215 EVT VT; 11216 switch (Intrinsic) { 11217 case Intrinsic::ppc_altivec_lvebx: 11218 VT = MVT::i8; 11219 break; 11220 case Intrinsic::ppc_altivec_lvehx: 11221 VT = MVT::i16; 11222 break; 11223 case Intrinsic::ppc_altivec_lvewx: 11224 VT = MVT::i32; 11225 break; 11226 case Intrinsic::ppc_vsx_lxvd2x: 11227 VT = MVT::v2f64; 11228 break; 11229 case Intrinsic::ppc_qpx_qvlfd: 11230 VT = MVT::v4f64; 11231 break; 11232 case Intrinsic::ppc_qpx_qvlfs: 11233 VT = MVT::v4f32; 11234 break; 11235 case Intrinsic::ppc_qpx_qvlfcd: 11236 VT = MVT::v2f64; 11237 break; 11238 case Intrinsic::ppc_qpx_qvlfcs: 11239 VT = MVT::v2f32; 11240 break; 11241 default: 11242 VT = MVT::v4i32; 11243 break; 11244 } 11245 11246 Info.opc = ISD::INTRINSIC_W_CHAIN; 11247 Info.memVT = VT; 11248 Info.ptrVal = I.getArgOperand(0); 11249 Info.offset = -VT.getStoreSize()+1; 11250 Info.size = 2*VT.getStoreSize()-1; 11251 Info.align = 1; 11252 Info.vol = false; 11253 Info.readMem = true; 11254 Info.writeMem = false; 11255 return true; 11256 } 11257 case Intrinsic::ppc_qpx_qvlfda: 11258 case Intrinsic::ppc_qpx_qvlfsa: 11259 case Intrinsic::ppc_qpx_qvlfcda: 11260 case Intrinsic::ppc_qpx_qvlfcsa: 11261 case Intrinsic::ppc_qpx_qvlfiwaa: 11262 case Intrinsic::ppc_qpx_qvlfiwza: { 11263 EVT VT; 11264 switch (Intrinsic) { 11265 case Intrinsic::ppc_qpx_qvlfda: 11266 VT = MVT::v4f64; 11267 break; 11268 case Intrinsic::ppc_qpx_qvlfsa: 11269 VT = MVT::v4f32; 11270 break; 11271 case Intrinsic::ppc_qpx_qvlfcda: 11272 VT = MVT::v2f64; 11273 break; 11274 case Intrinsic::ppc_qpx_qvlfcsa: 11275 VT = MVT::v2f32; 11276 break; 11277 default: 11278 VT = MVT::v4i32; 11279 break; 11280 } 11281 11282 Info.opc = ISD::INTRINSIC_W_CHAIN; 11283 Info.memVT = VT; 11284 Info.ptrVal = I.getArgOperand(0); 11285 Info.offset = 0; 11286 Info.size = VT.getStoreSize(); 11287 Info.align = 1; 11288 Info.vol = false; 11289 Info.readMem = true; 11290 Info.writeMem = false; 11291 return true; 11292 } 11293 case Intrinsic::ppc_qpx_qvstfd: 11294 case Intrinsic::ppc_qpx_qvstfs: 11295 case Intrinsic::ppc_qpx_qvstfcd: 11296 case Intrinsic::ppc_qpx_qvstfcs: 11297 case Intrinsic::ppc_qpx_qvstfiw: 11298 case Intrinsic::ppc_altivec_stvx: 11299 case Intrinsic::ppc_altivec_stvxl: 11300 case Intrinsic::ppc_altivec_stvebx: 11301 case Intrinsic::ppc_altivec_stvehx: 11302 case Intrinsic::ppc_altivec_stvewx: 11303 case Intrinsic::ppc_vsx_stxvd2x: 11304 case Intrinsic::ppc_vsx_stxvw4x: { 11305 EVT VT; 11306 switch (Intrinsic) { 11307 case Intrinsic::ppc_altivec_stvebx: 11308 VT = MVT::i8; 11309 break; 11310 case Intrinsic::ppc_altivec_stvehx: 11311 VT = MVT::i16; 11312 break; 11313 case Intrinsic::ppc_altivec_stvewx: 11314 VT = MVT::i32; 11315 break; 11316 case Intrinsic::ppc_vsx_stxvd2x: 11317 VT = MVT::v2f64; 11318 break; 11319 case Intrinsic::ppc_qpx_qvstfd: 11320 VT = MVT::v4f64; 11321 break; 11322 case Intrinsic::ppc_qpx_qvstfs: 11323 VT = MVT::v4f32; 11324 break; 11325 case Intrinsic::ppc_qpx_qvstfcd: 11326 VT = MVT::v2f64; 11327 break; 11328 case Intrinsic::ppc_qpx_qvstfcs: 11329 VT = MVT::v2f32; 11330 break; 11331 default: 11332 VT = MVT::v4i32; 11333 break; 11334 } 11335 11336 Info.opc = ISD::INTRINSIC_VOID; 11337 Info.memVT = VT; 11338 Info.ptrVal = I.getArgOperand(1); 11339 Info.offset = -VT.getStoreSize()+1; 11340 Info.size = 2*VT.getStoreSize()-1; 11341 Info.align = 1; 11342 Info.vol = false; 11343 Info.readMem = false; 11344 Info.writeMem = true; 11345 return true; 11346 } 11347 case Intrinsic::ppc_qpx_qvstfda: 11348 case Intrinsic::ppc_qpx_qvstfsa: 11349 case Intrinsic::ppc_qpx_qvstfcda: 11350 case Intrinsic::ppc_qpx_qvstfcsa: 11351 case Intrinsic::ppc_qpx_qvstfiwa: { 11352 EVT VT; 11353 switch (Intrinsic) { 11354 case Intrinsic::ppc_qpx_qvstfda: 11355 VT = MVT::v4f64; 11356 break; 11357 case Intrinsic::ppc_qpx_qvstfsa: 11358 VT = MVT::v4f32; 11359 break; 11360 case Intrinsic::ppc_qpx_qvstfcda: 11361 VT = MVT::v2f64; 11362 break; 11363 case Intrinsic::ppc_qpx_qvstfcsa: 11364 VT = MVT::v2f32; 11365 break; 11366 default: 11367 VT = MVT::v4i32; 11368 break; 11369 } 11370 11371 Info.opc = ISD::INTRINSIC_VOID; 11372 Info.memVT = VT; 11373 Info.ptrVal = I.getArgOperand(1); 11374 Info.offset = 0; 11375 Info.size = VT.getStoreSize(); 11376 Info.align = 1; 11377 Info.vol = false; 11378 Info.readMem = false; 11379 Info.writeMem = true; 11380 return true; 11381 } 11382 default: 11383 break; 11384 } 11385 11386 return false; 11387 } 11388 11389 /// getOptimalMemOpType - Returns the target specific optimal type for load 11390 /// and store operations as a result of memset, memcpy, and memmove 11391 /// lowering. If DstAlign is zero that means it's safe to destination 11392 /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 11393 /// means there isn't a need to check it against alignment requirement, 11394 /// probably because the source does not need to be loaded. If 'IsMemset' is 11395 /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that 11396 /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy 11397 /// source is constant so it does not need to be loaded. 11398 /// It returns EVT::Other if the type should be determined using generic 11399 /// target-independent logic. 11400 EVT PPCTargetLowering::getOptimalMemOpType(uint64_t Size, 11401 unsigned DstAlign, unsigned SrcAlign, 11402 bool IsMemset, bool ZeroMemset, 11403 bool MemcpyStrSrc, 11404 MachineFunction &MF) const { 11405 if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { 11406 const Function *F = MF.getFunction(); 11407 // When expanding a memset, require at least two QPX instructions to cover 11408 // the cost of loading the value to be stored from the constant pool. 11409 if (Subtarget.hasQPX() && Size >= 32 && (!IsMemset || Size >= 64) && 11410 (!SrcAlign || SrcAlign >= 32) && (!DstAlign || DstAlign >= 32) && 11411 !F->hasFnAttribute(Attribute::NoImplicitFloat)) { 11412 return MVT::v4f64; 11413 } 11414 11415 // We should use Altivec/VSX loads and stores when available. For unaligned 11416 // addresses, unaligned VSX loads are only fast starting with the P8. 11417 if (Subtarget.hasAltivec() && Size >= 16 && 11418 (((!SrcAlign || SrcAlign >= 16) && (!DstAlign || DstAlign >= 16)) || 11419 ((IsMemset && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) 11420 return MVT::v4i32; 11421 } 11422 11423 if (Subtarget.isPPC64()) { 11424 return MVT::i64; 11425 } 11426 11427 return MVT::i32; 11428 } 11429 11430 /// \brief Returns true if it is beneficial to convert a load of a constant 11431 /// to just the constant itself. 11432 bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 11433 Type *Ty) const { 11434 assert(Ty->isIntegerTy()); 11435 11436 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 11437 if (BitSize == 0 || BitSize > 64) 11438 return false; 11439 return true; 11440 } 11441 11442 bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 11443 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 11444 return false; 11445 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 11446 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 11447 return NumBits1 == 64 && NumBits2 == 32; 11448 } 11449 11450 bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 11451 if (!VT1.isInteger() || !VT2.isInteger()) 11452 return false; 11453 unsigned NumBits1 = VT1.getSizeInBits(); 11454 unsigned NumBits2 = VT2.getSizeInBits(); 11455 return NumBits1 == 64 && NumBits2 == 32; 11456 } 11457 11458 bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 11459 // Generally speaking, zexts are not free, but they are free when they can be 11460 // folded with other operations. 11461 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) { 11462 EVT MemVT = LD->getMemoryVT(); 11463 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 || 11464 (Subtarget.isPPC64() && MemVT == MVT::i32)) && 11465 (LD->getExtensionType() == ISD::NON_EXTLOAD || 11466 LD->getExtensionType() == ISD::ZEXTLOAD)) 11467 return true; 11468 } 11469 11470 // FIXME: Add other cases... 11471 // - 32-bit shifts with a zext to i64 11472 // - zext after ctlz, bswap, etc. 11473 // - zext after and by a constant mask 11474 11475 return TargetLowering::isZExtFree(Val, VT2); 11476 } 11477 11478 bool PPCTargetLowering::isFPExtFree(EVT VT) const { 11479 assert(VT.isFloatingPoint()); 11480 return true; 11481 } 11482 11483 bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 11484 return isInt<16>(Imm) || isUInt<16>(Imm); 11485 } 11486 11487 bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { 11488 return isInt<16>(Imm) || isUInt<16>(Imm); 11489 } 11490 11491 bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 11492 unsigned, 11493 unsigned, 11494 bool *Fast) const { 11495 if (DisablePPCUnaligned) 11496 return false; 11497 11498 // PowerPC supports unaligned memory access for simple non-vector types. 11499 // Although accessing unaligned addresses is not as efficient as accessing 11500 // aligned addresses, it is generally more efficient than manual expansion, 11501 // and generally only traps for software emulation when crossing page 11502 // boundaries. 11503 11504 if (!VT.isSimple()) 11505 return false; 11506 11507 if (VT.getSimpleVT().isVector()) { 11508 if (Subtarget.hasVSX()) { 11509 if (VT != MVT::v2f64 && VT != MVT::v2i64 && 11510 VT != MVT::v4f32 && VT != MVT::v4i32) 11511 return false; 11512 } else { 11513 return false; 11514 } 11515 } 11516 11517 if (VT == MVT::ppcf128) 11518 return false; 11519 11520 if (Fast) 11521 *Fast = true; 11522 11523 return true; 11524 } 11525 11526 bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 11527 VT = VT.getScalarType(); 11528 11529 if (!VT.isSimple()) 11530 return false; 11531 11532 switch (VT.getSimpleVT().SimpleTy) { 11533 case MVT::f32: 11534 case MVT::f64: 11535 return true; 11536 default: 11537 break; 11538 } 11539 11540 return false; 11541 } 11542 11543 const MCPhysReg * 11544 PPCTargetLowering::getScratchRegisters(CallingConv::ID) const { 11545 // LR is a callee-save register, but we must treat it as clobbered by any call 11546 // site. Hence we include LR in the scratch registers, which are in turn added 11547 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies 11548 // to CTR, which is used by any indirect call. 11549 static const MCPhysReg ScratchRegs[] = { 11550 PPC::X12, PPC::LR8, PPC::CTR8, 0 11551 }; 11552 11553 return ScratchRegs; 11554 } 11555 11556 unsigned PPCTargetLowering::getExceptionPointerRegister( 11557 const Constant *PersonalityFn) const { 11558 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3; 11559 } 11560 11561 unsigned PPCTargetLowering::getExceptionSelectorRegister( 11562 const Constant *PersonalityFn) const { 11563 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4; 11564 } 11565 11566 bool 11567 PPCTargetLowering::shouldExpandBuildVectorWithShuffles( 11568 EVT VT , unsigned DefinedValues) const { 11569 if (VT == MVT::v2i64) 11570 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves 11571 11572 if (Subtarget.hasQPX()) { 11573 if (VT == MVT::v4f32 || VT == MVT::v4f64 || VT == MVT::v4i1) 11574 return true; 11575 } 11576 11577 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); 11578 } 11579 11580 Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { 11581 if (DisableILPPref || Subtarget.enableMachineScheduler()) 11582 return TargetLowering::getSchedulingPreference(N); 11583 11584 return Sched::ILP; 11585 } 11586 11587 // Create a fast isel object. 11588 FastISel * 11589 PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, 11590 const TargetLibraryInfo *LibInfo) const { 11591 return PPC::createFastISel(FuncInfo, LibInfo); 11592 } 11593