1 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines the interfaces that ARM uses to lower LLVM code into a 11 // selection DAG. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "ARMISelLowering.h" 16 #include "ARMBaseInstrInfo.h" 17 #include "ARMBaseRegisterInfo.h" 18 #include "ARMCallingConv.h" 19 #include "ARMConstantPoolValue.h" 20 #include "ARMMachineFunctionInfo.h" 21 #include "ARMPerfectShuffle.h" 22 #include "ARMRegisterInfo.h" 23 #include "ARMSelectionDAGInfo.h" 24 #include "ARMSubtarget.h" 25 #include "MCTargetDesc/ARMAddressingModes.h" 26 #include "MCTargetDesc/ARMBaseInfo.h" 27 #include "Utils/ARMBaseInfo.h" 28 #include "llvm/ADT/APFloat.h" 29 #include "llvm/ADT/APInt.h" 30 #include "llvm/ADT/ArrayRef.h" 31 #include "llvm/ADT/BitVector.h" 32 #include "llvm/ADT/DenseMap.h" 33 #include "llvm/ADT/STLExtras.h" 34 #include "llvm/ADT/SmallPtrSet.h" 35 #include "llvm/ADT/SmallVector.h" 36 #include "llvm/ADT/Statistic.h" 37 #include "llvm/ADT/StringExtras.h" 38 #include "llvm/ADT/StringRef.h" 39 #include "llvm/ADT/StringSwitch.h" 40 #include "llvm/ADT/Triple.h" 41 #include "llvm/ADT/Twine.h" 42 #include "llvm/Analysis/VectorUtils.h" 43 #include "llvm/CodeGen/CallingConvLower.h" 44 #include "llvm/CodeGen/ISDOpcodes.h" 45 #include "llvm/CodeGen/IntrinsicLowering.h" 46 #include "llvm/CodeGen/MachineBasicBlock.h" 47 #include "llvm/CodeGen/MachineConstantPool.h" 48 #include "llvm/CodeGen/MachineFrameInfo.h" 49 #include "llvm/CodeGen/MachineFunction.h" 50 #include "llvm/CodeGen/MachineInstr.h" 51 #include "llvm/CodeGen/MachineInstrBuilder.h" 52 #include "llvm/CodeGen/MachineJumpTableInfo.h" 53 #include "llvm/CodeGen/MachineMemOperand.h" 54 #include "llvm/CodeGen/MachineOperand.h" 55 #include "llvm/CodeGen/MachineRegisterInfo.h" 56 #include "llvm/CodeGen/RuntimeLibcalls.h" 57 #include "llvm/CodeGen/SelectionDAG.h" 58 #include "llvm/CodeGen/SelectionDAGNodes.h" 59 #include "llvm/CodeGen/TargetInstrInfo.h" 60 #include "llvm/CodeGen/TargetLowering.h" 61 #include "llvm/CodeGen/TargetOpcodes.h" 62 #include "llvm/CodeGen/TargetRegisterInfo.h" 63 #include "llvm/CodeGen/TargetSubtargetInfo.h" 64 #include "llvm/CodeGen/ValueTypes.h" 65 #include "llvm/IR/Attributes.h" 66 #include "llvm/IR/CallingConv.h" 67 #include "llvm/IR/Constant.h" 68 #include "llvm/IR/Constants.h" 69 #include "llvm/IR/DataLayout.h" 70 #include "llvm/IR/DebugLoc.h" 71 #include "llvm/IR/DerivedTypes.h" 72 #include "llvm/IR/Function.h" 73 #include "llvm/IR/GlobalAlias.h" 74 #include "llvm/IR/GlobalValue.h" 75 #include "llvm/IR/GlobalVariable.h" 76 #include "llvm/IR/IRBuilder.h" 77 #include "llvm/IR/InlineAsm.h" 78 #include "llvm/IR/Instruction.h" 79 #include "llvm/IR/Instructions.h" 80 #include "llvm/IR/IntrinsicInst.h" 81 #include "llvm/IR/Intrinsics.h" 82 #include "llvm/IR/Module.h" 83 #include "llvm/IR/Type.h" 84 #include "llvm/IR/User.h" 85 #include "llvm/IR/Value.h" 86 #include "llvm/MC/MCInstrDesc.h" 87 #include "llvm/MC/MCInstrItineraries.h" 88 #include "llvm/MC/MCRegisterInfo.h" 89 #include "llvm/MC/MCSchedule.h" 90 #include "llvm/Support/AtomicOrdering.h" 91 #include "llvm/Support/BranchProbability.h" 92 #include "llvm/Support/Casting.h" 93 #include "llvm/Support/CodeGen.h" 94 #include "llvm/Support/CommandLine.h" 95 #include "llvm/Support/Compiler.h" 96 #include "llvm/Support/Debug.h" 97 #include "llvm/Support/ErrorHandling.h" 98 #include "llvm/Support/KnownBits.h" 99 #include "llvm/Support/MachineValueType.h" 100 #include "llvm/Support/MathExtras.h" 101 #include "llvm/Support/raw_ostream.h" 102 #include "llvm/Target/TargetMachine.h" 103 #include "llvm/Target/TargetOptions.h" 104 #include <algorithm> 105 #include <cassert> 106 #include <cstdint> 107 #include <cstdlib> 108 #include <iterator> 109 #include <limits> 110 #include <string> 111 #include <tuple> 112 #include <utility> 113 #include <vector> 114 115 using namespace llvm; 116 117 #define DEBUG_TYPE "arm-isel" 118 119 STATISTIC(NumTailCalls, "Number of tail calls"); 120 STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt"); 121 STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments"); 122 STATISTIC(NumConstpoolPromoted, 123 "Number of constants with their storage promoted into constant pools"); 124 125 static cl::opt<bool> 126 ARMInterworking("arm-interworking", cl::Hidden, 127 cl::desc("Enable / disable ARM interworking (for debugging only)"), 128 cl::init(true)); 129 130 static cl::opt<bool> EnableConstpoolPromotion( 131 "arm-promote-constant", cl::Hidden, 132 cl::desc("Enable / disable promotion of unnamed_addr constants into " 133 "constant pools"), 134 cl::init(false)); // FIXME: set to true by default once PR32780 is fixed 135 static cl::opt<unsigned> ConstpoolPromotionMaxSize( 136 "arm-promote-constant-max-size", cl::Hidden, 137 cl::desc("Maximum size of constant to promote into a constant pool"), 138 cl::init(64)); 139 static cl::opt<unsigned> ConstpoolPromotionMaxTotal( 140 "arm-promote-constant-max-total", cl::Hidden, 141 cl::desc("Maximum size of ALL constants to promote into a constant pool"), 142 cl::init(128)); 143 144 // The APCS parameter registers. 145 static const MCPhysReg GPRArgRegs[] = { 146 ARM::R0, ARM::R1, ARM::R2, ARM::R3 147 }; 148 149 void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT, 150 MVT PromotedBitwiseVT) { 151 if (VT != PromotedLdStVT) { 152 setOperationAction(ISD::LOAD, VT, Promote); 153 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT); 154 155 setOperationAction(ISD::STORE, VT, Promote); 156 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT); 157 } 158 159 MVT ElemTy = VT.getVectorElementType(); 160 if (ElemTy != MVT::f64) 161 setOperationAction(ISD::SETCC, VT, Custom); 162 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); 163 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 164 if (ElemTy == MVT::i32) { 165 setOperationAction(ISD::SINT_TO_FP, VT, Custom); 166 setOperationAction(ISD::UINT_TO_FP, VT, Custom); 167 setOperationAction(ISD::FP_TO_SINT, VT, Custom); 168 setOperationAction(ISD::FP_TO_UINT, VT, Custom); 169 } else { 170 setOperationAction(ISD::SINT_TO_FP, VT, Expand); 171 setOperationAction(ISD::UINT_TO_FP, VT, Expand); 172 setOperationAction(ISD::FP_TO_SINT, VT, Expand); 173 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 174 } 175 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 176 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 177 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); 178 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); 179 setOperationAction(ISD::SELECT, VT, Expand); 180 setOperationAction(ISD::SELECT_CC, VT, Expand); 181 setOperationAction(ISD::VSELECT, VT, Expand); 182 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 183 if (VT.isInteger()) { 184 setOperationAction(ISD::SHL, VT, Custom); 185 setOperationAction(ISD::SRA, VT, Custom); 186 setOperationAction(ISD::SRL, VT, Custom); 187 } 188 189 // Promote all bit-wise operations. 190 if (VT.isInteger() && VT != PromotedBitwiseVT) { 191 setOperationAction(ISD::AND, VT, Promote); 192 AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT); 193 setOperationAction(ISD::OR, VT, Promote); 194 AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT); 195 setOperationAction(ISD::XOR, VT, Promote); 196 AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT); 197 } 198 199 // Neon does not support vector divide/remainder operations. 200 setOperationAction(ISD::SDIV, VT, Expand); 201 setOperationAction(ISD::UDIV, VT, Expand); 202 setOperationAction(ISD::FDIV, VT, Expand); 203 setOperationAction(ISD::SREM, VT, Expand); 204 setOperationAction(ISD::UREM, VT, Expand); 205 setOperationAction(ISD::FREM, VT, Expand); 206 207 if (!VT.isFloatingPoint() && 208 VT != MVT::v2i64 && VT != MVT::v1i64) 209 for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) 210 setOperationAction(Opcode, VT, Legal); 211 } 212 213 void ARMTargetLowering::addDRTypeForNEON(MVT VT) { 214 addRegisterClass(VT, &ARM::DPRRegClass); 215 addTypeForNEON(VT, MVT::f64, MVT::v2i32); 216 } 217 218 void ARMTargetLowering::addQRTypeForNEON(MVT VT) { 219 addRegisterClass(VT, &ARM::DPairRegClass); 220 addTypeForNEON(VT, MVT::v2f64, MVT::v4i32); 221 } 222 223 ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, 224 const ARMSubtarget &STI) 225 : TargetLowering(TM), Subtarget(&STI) { 226 RegInfo = Subtarget->getRegisterInfo(); 227 Itins = Subtarget->getInstrItineraryData(); 228 229 setBooleanContents(ZeroOrOneBooleanContent); 230 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 231 232 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() && 233 !Subtarget->isTargetWatchOS()) { 234 bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard; 235 for (int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID) 236 setLibcallCallingConv(static_cast<RTLIB::Libcall>(LCID), 237 IsHFTarget ? CallingConv::ARM_AAPCS_VFP 238 : CallingConv::ARM_AAPCS); 239 } 240 241 if (Subtarget->isTargetMachO()) { 242 // Uses VFP for Thumb libfuncs if available. 243 if (Subtarget->isThumb() && Subtarget->hasVFP2() && 244 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) { 245 static const struct { 246 const RTLIB::Libcall Op; 247 const char * const Name; 248 const ISD::CondCode Cond; 249 } LibraryCalls[] = { 250 // Single-precision floating-point arithmetic. 251 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID }, 252 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID }, 253 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID }, 254 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID }, 255 256 // Double-precision floating-point arithmetic. 257 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID }, 258 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID }, 259 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID }, 260 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID }, 261 262 // Single-precision comparisons. 263 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE }, 264 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE }, 265 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE }, 266 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE }, 267 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE }, 268 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE }, 269 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE }, 270 { RTLIB::O_F32, "__unordsf2vfp", ISD::SETEQ }, 271 272 // Double-precision comparisons. 273 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE }, 274 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE }, 275 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE }, 276 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE }, 277 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE }, 278 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE }, 279 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE }, 280 { RTLIB::O_F64, "__unorddf2vfp", ISD::SETEQ }, 281 282 // Floating-point to integer conversions. 283 // i64 conversions are done via library routines even when generating VFP 284 // instructions, so use the same ones. 285 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID }, 286 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID }, 287 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID }, 288 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID }, 289 290 // Conversions between floating types. 291 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID }, 292 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID }, 293 294 // Integer to floating-point conversions. 295 // i64 conversions are done via library routines even when generating VFP 296 // instructions, so use the same ones. 297 // FIXME: There appears to be some naming inconsistency in ARM libgcc: 298 // e.g., __floatunsidf vs. __floatunssidfvfp. 299 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID }, 300 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID }, 301 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID }, 302 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID }, 303 }; 304 305 for (const auto &LC : LibraryCalls) { 306 setLibcallName(LC.Op, LC.Name); 307 if (LC.Cond != ISD::SETCC_INVALID) 308 setCmpLibcallCC(LC.Op, LC.Cond); 309 } 310 } 311 } 312 313 // These libcalls are not available in 32-bit. 314 setLibcallName(RTLIB::SHL_I128, nullptr); 315 setLibcallName(RTLIB::SRL_I128, nullptr); 316 setLibcallName(RTLIB::SRA_I128, nullptr); 317 318 // RTLIB 319 if (Subtarget->isAAPCS_ABI() && 320 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() || 321 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) { 322 static const struct { 323 const RTLIB::Libcall Op; 324 const char * const Name; 325 const CallingConv::ID CC; 326 const ISD::CondCode Cond; 327 } LibraryCalls[] = { 328 // Double-precision floating-point arithmetic helper functions 329 // RTABI chapter 4.1.2, Table 2 330 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 331 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 332 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 333 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 334 335 // Double-precision floating-point comparison helper functions 336 // RTABI chapter 4.1.2, Table 3 337 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE }, 338 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ }, 339 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE }, 340 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE }, 341 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE }, 342 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE }, 343 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE }, 344 { RTLIB::O_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ }, 345 346 // Single-precision floating-point arithmetic helper functions 347 // RTABI chapter 4.1.2, Table 4 348 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 349 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 350 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 351 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 352 353 // Single-precision floating-point comparison helper functions 354 // RTABI chapter 4.1.2, Table 5 355 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE }, 356 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ }, 357 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE }, 358 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE }, 359 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE }, 360 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE }, 361 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE }, 362 { RTLIB::O_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ }, 363 364 // Floating-point to integer conversions. 365 // RTABI chapter 4.1.2, Table 6 366 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 367 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 368 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 369 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 370 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 371 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 372 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 373 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 374 375 // Conversions between floating types. 376 // RTABI chapter 4.1.2, Table 7 377 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 378 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 379 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 380 381 // Integer to floating-point conversions. 382 // RTABI chapter 4.1.2, Table 8 383 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 384 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 385 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 386 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 387 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 388 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 389 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 390 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 391 392 // Long long helper functions 393 // RTABI chapter 4.2, Table 9 394 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 395 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 396 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 397 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 398 399 // Integer division functions 400 // RTABI chapter 4.3.1 401 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 402 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 403 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 404 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 405 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 406 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 407 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 408 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 409 }; 410 411 for (const auto &LC : LibraryCalls) { 412 setLibcallName(LC.Op, LC.Name); 413 setLibcallCallingConv(LC.Op, LC.CC); 414 if (LC.Cond != ISD::SETCC_INVALID) 415 setCmpLibcallCC(LC.Op, LC.Cond); 416 } 417 418 // EABI dependent RTLIB 419 if (TM.Options.EABIVersion == EABI::EABI4 || 420 TM.Options.EABIVersion == EABI::EABI5) { 421 static const struct { 422 const RTLIB::Libcall Op; 423 const char *const Name; 424 const CallingConv::ID CC; 425 const ISD::CondCode Cond; 426 } MemOpsLibraryCalls[] = { 427 // Memory operations 428 // RTABI chapter 4.3.4 429 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 430 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 431 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 432 }; 433 434 for (const auto &LC : MemOpsLibraryCalls) { 435 setLibcallName(LC.Op, LC.Name); 436 setLibcallCallingConv(LC.Op, LC.CC); 437 if (LC.Cond != ISD::SETCC_INVALID) 438 setCmpLibcallCC(LC.Op, LC.Cond); 439 } 440 } 441 } 442 443 if (Subtarget->isTargetWindows()) { 444 static const struct { 445 const RTLIB::Libcall Op; 446 const char * const Name; 447 const CallingConv::ID CC; 448 } LibraryCalls[] = { 449 { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP }, 450 { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP }, 451 { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP }, 452 { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP }, 453 { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP }, 454 { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP }, 455 { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP }, 456 { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP }, 457 }; 458 459 for (const auto &LC : LibraryCalls) { 460 setLibcallName(LC.Op, LC.Name); 461 setLibcallCallingConv(LC.Op, LC.CC); 462 } 463 } 464 465 // Use divmod compiler-rt calls for iOS 5.0 and later. 466 if (Subtarget->isTargetMachO() && 467 !(Subtarget->isTargetIOS() && 468 Subtarget->getTargetTriple().isOSVersionLT(5, 0))) { 469 setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4"); 470 setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4"); 471 } 472 473 // The half <-> float conversion functions are always soft-float on 474 // non-watchos platforms, but are needed for some targets which use a 475 // hard-float calling convention by default. 476 if (!Subtarget->isTargetWatchABI()) { 477 if (Subtarget->isAAPCS_ABI()) { 478 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS); 479 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS); 480 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS); 481 } else { 482 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS); 483 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS); 484 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS); 485 } 486 } 487 488 // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have 489 // a __gnu_ prefix (which is the default). 490 if (Subtarget->isTargetAEABI()) { 491 static const struct { 492 const RTLIB::Libcall Op; 493 const char * const Name; 494 const CallingConv::ID CC; 495 } LibraryCalls[] = { 496 { RTLIB::FPROUND_F32_F16, "__aeabi_f2h", CallingConv::ARM_AAPCS }, 497 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS }, 498 { RTLIB::FPEXT_F16_F32, "__aeabi_h2f", CallingConv::ARM_AAPCS }, 499 }; 500 501 for (const auto &LC : LibraryCalls) { 502 setLibcallName(LC.Op, LC.Name); 503 setLibcallCallingConv(LC.Op, LC.CC); 504 } 505 } 506 507 if (Subtarget->isThumb1Only()) 508 addRegisterClass(MVT::i32, &ARM::tGPRRegClass); 509 else 510 addRegisterClass(MVT::i32, &ARM::GPRRegClass); 511 512 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && 513 !Subtarget->isThumb1Only()) { 514 addRegisterClass(MVT::f32, &ARM::SPRRegClass); 515 addRegisterClass(MVT::f64, &ARM::DPRRegClass); 516 } 517 518 if (Subtarget->hasFullFP16()) { 519 addRegisterClass(MVT::f16, &ARM::HPRRegClass); 520 setOperationAction(ISD::BITCAST, MVT::i16, Custom); 521 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 522 setOperationAction(ISD::BITCAST, MVT::f16, Custom); 523 524 setOperationAction(ISD::FMINNUM, MVT::f16, Legal); 525 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal); 526 } 527 528 for (MVT VT : MVT::vector_valuetypes()) { 529 for (MVT InnerVT : MVT::vector_valuetypes()) { 530 setTruncStoreAction(VT, InnerVT, Expand); 531 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 532 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 533 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 534 } 535 536 setOperationAction(ISD::MULHS, VT, Expand); 537 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 538 setOperationAction(ISD::MULHU, VT, Expand); 539 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 540 541 setOperationAction(ISD::BSWAP, VT, Expand); 542 } 543 544 setOperationAction(ISD::ConstantFP, MVT::f32, Custom); 545 setOperationAction(ISD::ConstantFP, MVT::f64, Custom); 546 547 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom); 548 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom); 549 550 if (Subtarget->hasNEON()) { 551 addDRTypeForNEON(MVT::v2f32); 552 addDRTypeForNEON(MVT::v8i8); 553 addDRTypeForNEON(MVT::v4i16); 554 addDRTypeForNEON(MVT::v2i32); 555 addDRTypeForNEON(MVT::v1i64); 556 557 addQRTypeForNEON(MVT::v4f32); 558 addQRTypeForNEON(MVT::v2f64); 559 addQRTypeForNEON(MVT::v16i8); 560 addQRTypeForNEON(MVT::v8i16); 561 addQRTypeForNEON(MVT::v4i32); 562 addQRTypeForNEON(MVT::v2i64); 563 564 if (Subtarget->hasFullFP16()) { 565 addQRTypeForNEON(MVT::v8f16); 566 addDRTypeForNEON(MVT::v4f16); 567 } 568 569 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but 570 // neither Neon nor VFP support any arithmetic operations on it. 571 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively 572 // supported for v4f32. 573 setOperationAction(ISD::FADD, MVT::v2f64, Expand); 574 setOperationAction(ISD::FSUB, MVT::v2f64, Expand); 575 setOperationAction(ISD::FMUL, MVT::v2f64, Expand); 576 // FIXME: Code duplication: FDIV and FREM are expanded always, see 577 // ARMTargetLowering::addTypeForNEON method for details. 578 setOperationAction(ISD::FDIV, MVT::v2f64, Expand); 579 setOperationAction(ISD::FREM, MVT::v2f64, Expand); 580 // FIXME: Create unittest. 581 // In another words, find a way when "copysign" appears in DAG with vector 582 // operands. 583 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand); 584 // FIXME: Code duplication: SETCC has custom operation action, see 585 // ARMTargetLowering::addTypeForNEON method for details. 586 setOperationAction(ISD::SETCC, MVT::v2f64, Expand); 587 // FIXME: Create unittest for FNEG and for FABS. 588 setOperationAction(ISD::FNEG, MVT::v2f64, Expand); 589 setOperationAction(ISD::FABS, MVT::v2f64, Expand); 590 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand); 591 setOperationAction(ISD::FSIN, MVT::v2f64, Expand); 592 setOperationAction(ISD::FCOS, MVT::v2f64, Expand); 593 setOperationAction(ISD::FPOW, MVT::v2f64, Expand); 594 setOperationAction(ISD::FLOG, MVT::v2f64, Expand); 595 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand); 596 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand); 597 setOperationAction(ISD::FEXP, MVT::v2f64, Expand); 598 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand); 599 // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR. 600 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand); 601 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand); 602 setOperationAction(ISD::FRINT, MVT::v2f64, Expand); 603 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand); 604 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand); 605 setOperationAction(ISD::FMA, MVT::v2f64, Expand); 606 607 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); 608 setOperationAction(ISD::FSIN, MVT::v4f32, Expand); 609 setOperationAction(ISD::FCOS, MVT::v4f32, Expand); 610 setOperationAction(ISD::FPOW, MVT::v4f32, Expand); 611 setOperationAction(ISD::FLOG, MVT::v4f32, Expand); 612 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand); 613 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand); 614 setOperationAction(ISD::FEXP, MVT::v4f32, Expand); 615 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand); 616 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand); 617 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand); 618 setOperationAction(ISD::FRINT, MVT::v4f32, Expand); 619 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); 620 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand); 621 622 // Mark v2f32 intrinsics. 623 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand); 624 setOperationAction(ISD::FSIN, MVT::v2f32, Expand); 625 setOperationAction(ISD::FCOS, MVT::v2f32, Expand); 626 setOperationAction(ISD::FPOW, MVT::v2f32, Expand); 627 setOperationAction(ISD::FLOG, MVT::v2f32, Expand); 628 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand); 629 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand); 630 setOperationAction(ISD::FEXP, MVT::v2f32, Expand); 631 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand); 632 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand); 633 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand); 634 setOperationAction(ISD::FRINT, MVT::v2f32, Expand); 635 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand); 636 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand); 637 638 // Neon does not support some operations on v1i64 and v2i64 types. 639 setOperationAction(ISD::MUL, MVT::v1i64, Expand); 640 // Custom handling for some quad-vector types to detect VMULL. 641 setOperationAction(ISD::MUL, MVT::v8i16, Custom); 642 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 643 setOperationAction(ISD::MUL, MVT::v2i64, Custom); 644 // Custom handling for some vector types to avoid expensive expansions 645 setOperationAction(ISD::SDIV, MVT::v4i16, Custom); 646 setOperationAction(ISD::SDIV, MVT::v8i8, Custom); 647 setOperationAction(ISD::UDIV, MVT::v4i16, Custom); 648 setOperationAction(ISD::UDIV, MVT::v8i8, Custom); 649 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with 650 // a destination type that is wider than the source, and nor does 651 // it have a FP_TO_[SU]INT instruction with a narrower destination than 652 // source. 653 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); 654 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); 655 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom); 656 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom); 657 658 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); 659 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand); 660 661 // NEON does not have single instruction CTPOP for vectors with element 662 // types wider than 8-bits. However, custom lowering can leverage the 663 // v8i8/v16i8 vcnt instruction. 664 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom); 665 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); 666 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom); 667 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom); 668 setOperationAction(ISD::CTPOP, MVT::v1i64, Expand); 669 setOperationAction(ISD::CTPOP, MVT::v2i64, Expand); 670 671 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); 672 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); 673 674 // NEON does not have single instruction CTTZ for vectors. 675 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom); 676 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom); 677 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom); 678 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom); 679 680 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom); 681 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom); 682 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom); 683 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom); 684 685 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom); 686 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom); 687 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom); 688 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom); 689 690 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom); 691 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom); 692 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); 693 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom); 694 695 // NEON only has FMA instructions as of VFP4. 696 if (!Subtarget->hasVFP4()) { 697 setOperationAction(ISD::FMA, MVT::v2f32, Expand); 698 setOperationAction(ISD::FMA, MVT::v4f32, Expand); 699 } 700 701 setTargetDAGCombine(ISD::INTRINSIC_VOID); 702 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 703 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 704 setTargetDAGCombine(ISD::SHL); 705 setTargetDAGCombine(ISD::SRL); 706 setTargetDAGCombine(ISD::SRA); 707 setTargetDAGCombine(ISD::SIGN_EXTEND); 708 setTargetDAGCombine(ISD::ZERO_EXTEND); 709 setTargetDAGCombine(ISD::ANY_EXTEND); 710 setTargetDAGCombine(ISD::BUILD_VECTOR); 711 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 712 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 713 setTargetDAGCombine(ISD::STORE); 714 setTargetDAGCombine(ISD::FP_TO_SINT); 715 setTargetDAGCombine(ISD::FP_TO_UINT); 716 setTargetDAGCombine(ISD::FDIV); 717 setTargetDAGCombine(ISD::LOAD); 718 719 // It is legal to extload from v4i8 to v4i16 or v4i32. 720 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16, 721 MVT::v2i32}) { 722 for (MVT VT : MVT::integer_vector_valuetypes()) { 723 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal); 724 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal); 725 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal); 726 } 727 } 728 } 729 730 if (Subtarget->isFPOnlySP()) { 731 // When targeting a floating-point unit with only single-precision 732 // operations, f64 is legal for the few double-precision instructions which 733 // are present However, no double-precision operations other than moves, 734 // loads and stores are provided by the hardware. 735 setOperationAction(ISD::FADD, MVT::f64, Expand); 736 setOperationAction(ISD::FSUB, MVT::f64, Expand); 737 setOperationAction(ISD::FMUL, MVT::f64, Expand); 738 setOperationAction(ISD::FMA, MVT::f64, Expand); 739 setOperationAction(ISD::FDIV, MVT::f64, Expand); 740 setOperationAction(ISD::FREM, MVT::f64, Expand); 741 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 742 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand); 743 setOperationAction(ISD::FNEG, MVT::f64, Expand); 744 setOperationAction(ISD::FABS, MVT::f64, Expand); 745 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 746 setOperationAction(ISD::FSIN, MVT::f64, Expand); 747 setOperationAction(ISD::FCOS, MVT::f64, Expand); 748 setOperationAction(ISD::FPOW, MVT::f64, Expand); 749 setOperationAction(ISD::FLOG, MVT::f64, Expand); 750 setOperationAction(ISD::FLOG2, MVT::f64, Expand); 751 setOperationAction(ISD::FLOG10, MVT::f64, Expand); 752 setOperationAction(ISD::FEXP, MVT::f64, Expand); 753 setOperationAction(ISD::FEXP2, MVT::f64, Expand); 754 setOperationAction(ISD::FCEIL, MVT::f64, Expand); 755 setOperationAction(ISD::FTRUNC, MVT::f64, Expand); 756 setOperationAction(ISD::FRINT, MVT::f64, Expand); 757 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); 758 setOperationAction(ISD::FFLOOR, MVT::f64, Expand); 759 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 760 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 761 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 762 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 763 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom); 764 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom); 765 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); 766 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom); 767 } 768 769 computeRegisterProperties(Subtarget->getRegisterInfo()); 770 771 // ARM does not have floating-point extending loads. 772 for (MVT VT : MVT::fp_valuetypes()) { 773 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); 774 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); 775 } 776 777 // ... or truncating stores 778 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 779 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 780 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 781 782 // ARM does not have i1 sign extending load. 783 for (MVT VT : MVT::integer_valuetypes()) 784 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 785 786 // ARM supports all 4 flavors of integer indexed load / store. 787 if (!Subtarget->isThumb1Only()) { 788 for (unsigned im = (unsigned)ISD::PRE_INC; 789 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { 790 setIndexedLoadAction(im, MVT::i1, Legal); 791 setIndexedLoadAction(im, MVT::i8, Legal); 792 setIndexedLoadAction(im, MVT::i16, Legal); 793 setIndexedLoadAction(im, MVT::i32, Legal); 794 setIndexedStoreAction(im, MVT::i1, Legal); 795 setIndexedStoreAction(im, MVT::i8, Legal); 796 setIndexedStoreAction(im, MVT::i16, Legal); 797 setIndexedStoreAction(im, MVT::i32, Legal); 798 } 799 } else { 800 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}. 801 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal); 802 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal); 803 } 804 805 setOperationAction(ISD::SADDO, MVT::i32, Custom); 806 setOperationAction(ISD::UADDO, MVT::i32, Custom); 807 setOperationAction(ISD::SSUBO, MVT::i32, Custom); 808 setOperationAction(ISD::USUBO, MVT::i32, Custom); 809 810 setOperationAction(ISD::ADDCARRY, MVT::i32, Custom); 811 setOperationAction(ISD::SUBCARRY, MVT::i32, Custom); 812 813 // i64 operation support. 814 setOperationAction(ISD::MUL, MVT::i64, Expand); 815 setOperationAction(ISD::MULHU, MVT::i32, Expand); 816 if (Subtarget->isThumb1Only()) { 817 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 818 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 819 } 820 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops() 821 || (Subtarget->isThumb2() && !Subtarget->hasDSP())) 822 setOperationAction(ISD::MULHS, MVT::i32, Expand); 823 824 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 825 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 826 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 827 setOperationAction(ISD::SRL, MVT::i64, Custom); 828 setOperationAction(ISD::SRA, MVT::i64, Custom); 829 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); 830 831 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1. 832 if (Subtarget->isThumb1Only()) { 833 setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand); 834 setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand); 835 setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand); 836 } 837 838 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) 839 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 840 841 // ARM does not have ROTL. 842 setOperationAction(ISD::ROTL, MVT::i32, Expand); 843 for (MVT VT : MVT::vector_valuetypes()) { 844 setOperationAction(ISD::ROTL, VT, Expand); 845 setOperationAction(ISD::ROTR, VT, Expand); 846 } 847 setOperationAction(ISD::CTTZ, MVT::i32, Custom); 848 setOperationAction(ISD::CTPOP, MVT::i32, Expand); 849 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) 850 setOperationAction(ISD::CTLZ, MVT::i32, Expand); 851 852 // @llvm.readcyclecounter requires the Performance Monitors extension. 853 // Default to the 0 expansion on unsupported platforms. 854 // FIXME: Technically there are older ARM CPUs that have 855 // implementation-specific ways of obtaining this information. 856 if (Subtarget->hasPerfMon()) 857 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); 858 859 // Only ARMv6 has BSWAP. 860 if (!Subtarget->hasV6Ops()) 861 setOperationAction(ISD::BSWAP, MVT::i32, Expand); 862 863 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode() 864 : Subtarget->hasDivideInARMMode(); 865 if (!hasDivide) { 866 // These are expanded into libcalls if the cpu doesn't have HW divider. 867 setOperationAction(ISD::SDIV, MVT::i32, LibCall); 868 setOperationAction(ISD::UDIV, MVT::i32, LibCall); 869 } 870 871 if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) { 872 setOperationAction(ISD::SDIV, MVT::i32, Custom); 873 setOperationAction(ISD::UDIV, MVT::i32, Custom); 874 875 setOperationAction(ISD::SDIV, MVT::i64, Custom); 876 setOperationAction(ISD::UDIV, MVT::i64, Custom); 877 } 878 879 setOperationAction(ISD::SREM, MVT::i32, Expand); 880 setOperationAction(ISD::UREM, MVT::i32, Expand); 881 882 // Register based DivRem for AEABI (RTABI 4.2) 883 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || 884 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || 885 Subtarget->isTargetWindows()) { 886 setOperationAction(ISD::SREM, MVT::i64, Custom); 887 setOperationAction(ISD::UREM, MVT::i64, Custom); 888 HasStandaloneRem = false; 889 890 if (Subtarget->isTargetWindows()) { 891 const struct { 892 const RTLIB::Libcall Op; 893 const char * const Name; 894 const CallingConv::ID CC; 895 } LibraryCalls[] = { 896 { RTLIB::SDIVREM_I8, "__rt_sdiv", CallingConv::ARM_AAPCS }, 897 { RTLIB::SDIVREM_I16, "__rt_sdiv", CallingConv::ARM_AAPCS }, 898 { RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS }, 899 { RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS }, 900 901 { RTLIB::UDIVREM_I8, "__rt_udiv", CallingConv::ARM_AAPCS }, 902 { RTLIB::UDIVREM_I16, "__rt_udiv", CallingConv::ARM_AAPCS }, 903 { RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS }, 904 { RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS }, 905 }; 906 907 for (const auto &LC : LibraryCalls) { 908 setLibcallName(LC.Op, LC.Name); 909 setLibcallCallingConv(LC.Op, LC.CC); 910 } 911 } else { 912 const struct { 913 const RTLIB::Libcall Op; 914 const char * const Name; 915 const CallingConv::ID CC; 916 } LibraryCalls[] = { 917 { RTLIB::SDIVREM_I8, "__aeabi_idivmod", CallingConv::ARM_AAPCS }, 918 { RTLIB::SDIVREM_I16, "__aeabi_idivmod", CallingConv::ARM_AAPCS }, 919 { RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS }, 920 { RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS }, 921 922 { RTLIB::UDIVREM_I8, "__aeabi_uidivmod", CallingConv::ARM_AAPCS }, 923 { RTLIB::UDIVREM_I16, "__aeabi_uidivmod", CallingConv::ARM_AAPCS }, 924 { RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS }, 925 { RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS }, 926 }; 927 928 for (const auto &LC : LibraryCalls) { 929 setLibcallName(LC.Op, LC.Name); 930 setLibcallCallingConv(LC.Op, LC.CC); 931 } 932 } 933 934 setOperationAction(ISD::SDIVREM, MVT::i32, Custom); 935 setOperationAction(ISD::UDIVREM, MVT::i32, Custom); 936 setOperationAction(ISD::SDIVREM, MVT::i64, Custom); 937 setOperationAction(ISD::UDIVREM, MVT::i64, Custom); 938 } else { 939 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 940 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 941 } 942 943 if (Subtarget->isTargetWindows() && Subtarget->getTargetTriple().isOSMSVCRT()) 944 for (auto &VT : {MVT::f32, MVT::f64}) 945 setOperationAction(ISD::FPOWI, VT, Custom); 946 947 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 948 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 949 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 950 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 951 952 setOperationAction(ISD::TRAP, MVT::Other, Legal); 953 954 // Use the default implementation. 955 setOperationAction(ISD::VASTART, MVT::Other, Custom); 956 setOperationAction(ISD::VAARG, MVT::Other, Expand); 957 setOperationAction(ISD::VACOPY, MVT::Other, Expand); 958 setOperationAction(ISD::VAEND, MVT::Other, Expand); 959 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 960 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 961 962 if (Subtarget->isTargetWindows()) 963 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 964 else 965 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 966 967 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use 968 // the default expansion. 969 InsertFencesForAtomic = false; 970 if (Subtarget->hasAnyDataBarrier() && 971 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) { 972 // ATOMIC_FENCE needs custom lowering; the others should have been expanded 973 // to ldrex/strex loops already. 974 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); 975 if (!Subtarget->isThumb() || !Subtarget->isMClass()) 976 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); 977 978 // On v8, we have particularly efficient implementations of atomic fences 979 // if they can be combined with nearby atomic loads and stores. 980 if (!Subtarget->hasV8Ops() || getTargetMachine().getOptLevel() == 0) { 981 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc. 982 InsertFencesForAtomic = true; 983 } 984 } else { 985 // If there's anything we can use as a barrier, go through custom lowering 986 // for ATOMIC_FENCE. 987 // If target has DMB in thumb, Fences can be inserted. 988 if (Subtarget->hasDataBarrier()) 989 InsertFencesForAtomic = true; 990 991 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, 992 Subtarget->hasAnyDataBarrier() ? Custom : Expand); 993 994 // Set them all for expansion, which will force libcalls. 995 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand); 996 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Expand); 997 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Expand); 998 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand); 999 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Expand); 1000 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Expand); 1001 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Expand); 1002 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand); 1003 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand); 1004 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand); 1005 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand); 1006 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand); 1007 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the 1008 // Unordered/Monotonic case. 1009 if (!InsertFencesForAtomic) { 1010 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom); 1011 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom); 1012 } 1013 } 1014 1015 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 1016 1017 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes. 1018 if (!Subtarget->hasV6Ops()) { 1019 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 1020 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 1021 } 1022 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 1023 1024 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && 1025 !Subtarget->isThumb1Only()) { 1026 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR 1027 // iff target supports vfp2. 1028 setOperationAction(ISD::BITCAST, MVT::i64, Custom); 1029 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 1030 } 1031 1032 // We want to custom lower some of our intrinsics. 1033 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 1034 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 1035 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 1036 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom); 1037 if (Subtarget->useSjLjEH()) 1038 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume"); 1039 1040 setOperationAction(ISD::SETCC, MVT::i32, Expand); 1041 setOperationAction(ISD::SETCC, MVT::f32, Expand); 1042 setOperationAction(ISD::SETCC, MVT::f64, Expand); 1043 setOperationAction(ISD::SELECT, MVT::i32, Custom); 1044 setOperationAction(ISD::SELECT, MVT::f32, Custom); 1045 setOperationAction(ISD::SELECT, MVT::f64, Custom); 1046 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 1047 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 1048 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 1049 if (Subtarget->hasFullFP16()) { 1050 setOperationAction(ISD::SETCC, MVT::f16, Expand); 1051 setOperationAction(ISD::SELECT, MVT::f16, Custom); 1052 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom); 1053 } 1054 1055 setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom); 1056 1057 setOperationAction(ISD::BRCOND, MVT::Other, Custom); 1058 setOperationAction(ISD::BR_CC, MVT::i32, Custom); 1059 if (Subtarget->hasFullFP16()) 1060 setOperationAction(ISD::BR_CC, MVT::f16, Custom); 1061 setOperationAction(ISD::BR_CC, MVT::f32, Custom); 1062 setOperationAction(ISD::BR_CC, MVT::f64, Custom); 1063 setOperationAction(ISD::BR_JT, MVT::Other, Custom); 1064 1065 // We don't support sin/cos/fmod/copysign/pow 1066 setOperationAction(ISD::FSIN, MVT::f64, Expand); 1067 setOperationAction(ISD::FSIN, MVT::f32, Expand); 1068 setOperationAction(ISD::FCOS, MVT::f32, Expand); 1069 setOperationAction(ISD::FCOS, MVT::f64, Expand); 1070 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 1071 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 1072 setOperationAction(ISD::FREM, MVT::f64, Expand); 1073 setOperationAction(ISD::FREM, MVT::f32, Expand); 1074 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && 1075 !Subtarget->isThumb1Only()) { 1076 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 1077 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 1078 } 1079 setOperationAction(ISD::FPOW, MVT::f64, Expand); 1080 setOperationAction(ISD::FPOW, MVT::f32, Expand); 1081 1082 if (!Subtarget->hasVFP4()) { 1083 setOperationAction(ISD::FMA, MVT::f64, Expand); 1084 setOperationAction(ISD::FMA, MVT::f32, Expand); 1085 } 1086 1087 // Various VFP goodness 1088 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) { 1089 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded. 1090 if (!Subtarget->hasFPARMv8() || Subtarget->isFPOnlySP()) { 1091 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); 1092 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); 1093 } 1094 1095 // fp16 is a special v7 extension that adds f16 <-> f32 conversions. 1096 if (!Subtarget->hasFP16()) { 1097 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); 1098 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); 1099 } 1100 } 1101 1102 // Use __sincos_stret if available. 1103 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr && 1104 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) { 1105 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); 1106 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); 1107 } 1108 1109 // FP-ARMv8 implements a lot of rounding-like FP operations. 1110 if (Subtarget->hasFPARMv8()) { 1111 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 1112 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 1113 setOperationAction(ISD::FROUND, MVT::f32, Legal); 1114 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 1115 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); 1116 setOperationAction(ISD::FRINT, MVT::f32, Legal); 1117 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); 1118 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); 1119 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal); 1120 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal); 1121 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); 1122 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); 1123 1124 if (!Subtarget->isFPOnlySP()) { 1125 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 1126 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 1127 setOperationAction(ISD::FROUND, MVT::f64, Legal); 1128 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 1129 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); 1130 setOperationAction(ISD::FRINT, MVT::f64, Legal); 1131 setOperationAction(ISD::FMINNUM, MVT::f64, Legal); 1132 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); 1133 } 1134 } 1135 1136 if (Subtarget->hasNEON()) { 1137 // vmin and vmax aren't available in a scalar form, so we use 1138 // a NEON instruction with an undef lane instead. 1139 setOperationAction(ISD::FMINNAN, MVT::f16, Legal); 1140 setOperationAction(ISD::FMAXNAN, MVT::f16, Legal); 1141 setOperationAction(ISD::FMINNAN, MVT::f32, Legal); 1142 setOperationAction(ISD::FMAXNAN, MVT::f32, Legal); 1143 setOperationAction(ISD::FMINNAN, MVT::v2f32, Legal); 1144 setOperationAction(ISD::FMAXNAN, MVT::v2f32, Legal); 1145 setOperationAction(ISD::FMINNAN, MVT::v4f32, Legal); 1146 setOperationAction(ISD::FMAXNAN, MVT::v4f32, Legal); 1147 } 1148 1149 // We have target-specific dag combine patterns for the following nodes: 1150 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine 1151 setTargetDAGCombine(ISD::ADD); 1152 setTargetDAGCombine(ISD::SUB); 1153 setTargetDAGCombine(ISD::MUL); 1154 setTargetDAGCombine(ISD::AND); 1155 setTargetDAGCombine(ISD::OR); 1156 setTargetDAGCombine(ISD::XOR); 1157 1158 if (Subtarget->hasV6Ops()) 1159 setTargetDAGCombine(ISD::SRL); 1160 1161 setStackPointerRegisterToSaveRestore(ARM::SP); 1162 1163 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() || 1164 !Subtarget->hasVFP2()) 1165 setSchedulingPreference(Sched::RegPressure); 1166 else 1167 setSchedulingPreference(Sched::Hybrid); 1168 1169 //// temporary - rewrite interface to use type 1170 MaxStoresPerMemset = 8; 1171 MaxStoresPerMemsetOptSize = 4; 1172 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores 1173 MaxStoresPerMemcpyOptSize = 2; 1174 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores 1175 MaxStoresPerMemmoveOptSize = 2; 1176 1177 // On ARM arguments smaller than 4 bytes are extended, so all arguments 1178 // are at least 4 bytes aligned. 1179 setMinStackArgumentAlignment(4); 1180 1181 // Prefer likely predicted branches to selects on out-of-order cores. 1182 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder(); 1183 1184 setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2); 1185 } 1186 1187 bool ARMTargetLowering::useSoftFloat() const { 1188 return Subtarget->useSoftFloat(); 1189 } 1190 1191 // FIXME: It might make sense to define the representative register class as the 1192 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is 1193 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently, 1194 // SPR's representative would be DPR_VFP2. This should work well if register 1195 // pressure tracking were modified such that a register use would increment the 1196 // pressure of the register class's representative and all of it's super 1197 // classes' representatives transitively. We have not implemented this because 1198 // of the difficulty prior to coalescing of modeling operand register classes 1199 // due to the common occurrence of cross class copies and subregister insertions 1200 // and extractions. 1201 std::pair<const TargetRegisterClass *, uint8_t> 1202 ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, 1203 MVT VT) const { 1204 const TargetRegisterClass *RRC = nullptr; 1205 uint8_t Cost = 1; 1206 switch (VT.SimpleTy) { 1207 default: 1208 return TargetLowering::findRepresentativeClass(TRI, VT); 1209 // Use DPR as representative register class for all floating point 1210 // and vector types. Since there are 32 SPR registers and 32 DPR registers so 1211 // the cost is 1 for both f32 and f64. 1212 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16: 1213 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32: 1214 RRC = &ARM::DPRRegClass; 1215 // When NEON is used for SP, only half of the register file is available 1216 // because operations that define both SP and DP results will be constrained 1217 // to the VFP2 class (D0-D15). We currently model this constraint prior to 1218 // coalescing by double-counting the SP regs. See the FIXME above. 1219 if (Subtarget->useNEONForSinglePrecisionFP()) 1220 Cost = 2; 1221 break; 1222 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: 1223 case MVT::v4f32: case MVT::v2f64: 1224 RRC = &ARM::DPRRegClass; 1225 Cost = 2; 1226 break; 1227 case MVT::v4i64: 1228 RRC = &ARM::DPRRegClass; 1229 Cost = 4; 1230 break; 1231 case MVT::v8i64: 1232 RRC = &ARM::DPRRegClass; 1233 Cost = 8; 1234 break; 1235 } 1236 return std::make_pair(RRC, Cost); 1237 } 1238 1239 const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const { 1240 switch ((ARMISD::NodeType)Opcode) { 1241 case ARMISD::FIRST_NUMBER: break; 1242 case ARMISD::Wrapper: return "ARMISD::Wrapper"; 1243 case ARMISD::WrapperPIC: return "ARMISD::WrapperPIC"; 1244 case ARMISD::WrapperJT: return "ARMISD::WrapperJT"; 1245 case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL"; 1246 case ARMISD::CALL: return "ARMISD::CALL"; 1247 case ARMISD::CALL_PRED: return "ARMISD::CALL_PRED"; 1248 case ARMISD::CALL_NOLINK: return "ARMISD::CALL_NOLINK"; 1249 case ARMISD::BRCOND: return "ARMISD::BRCOND"; 1250 case ARMISD::BR_JT: return "ARMISD::BR_JT"; 1251 case ARMISD::BR2_JT: return "ARMISD::BR2_JT"; 1252 case ARMISD::RET_FLAG: return "ARMISD::RET_FLAG"; 1253 case ARMISD::INTRET_FLAG: return "ARMISD::INTRET_FLAG"; 1254 case ARMISD::PIC_ADD: return "ARMISD::PIC_ADD"; 1255 case ARMISD::CMP: return "ARMISD::CMP"; 1256 case ARMISD::CMN: return "ARMISD::CMN"; 1257 case ARMISD::CMPZ: return "ARMISD::CMPZ"; 1258 case ARMISD::CMPFP: return "ARMISD::CMPFP"; 1259 case ARMISD::CMPFPw0: return "ARMISD::CMPFPw0"; 1260 case ARMISD::BCC_i64: return "ARMISD::BCC_i64"; 1261 case ARMISD::FMSTAT: return "ARMISD::FMSTAT"; 1262 1263 case ARMISD::CMOV: return "ARMISD::CMOV"; 1264 1265 case ARMISD::SSAT: return "ARMISD::SSAT"; 1266 case ARMISD::USAT: return "ARMISD::USAT"; 1267 1268 case ARMISD::SRL_FLAG: return "ARMISD::SRL_FLAG"; 1269 case ARMISD::SRA_FLAG: return "ARMISD::SRA_FLAG"; 1270 case ARMISD::RRX: return "ARMISD::RRX"; 1271 1272 case ARMISD::ADDC: return "ARMISD::ADDC"; 1273 case ARMISD::ADDE: return "ARMISD::ADDE"; 1274 case ARMISD::SUBC: return "ARMISD::SUBC"; 1275 case ARMISD::SUBE: return "ARMISD::SUBE"; 1276 1277 case ARMISD::VMOVRRD: return "ARMISD::VMOVRRD"; 1278 case ARMISD::VMOVDRR: return "ARMISD::VMOVDRR"; 1279 case ARMISD::VMOVhr: return "ARMISD::VMOVhr"; 1280 case ARMISD::VMOVrh: return "ARMISD::VMOVrh"; 1281 case ARMISD::VMOVSR: return "ARMISD::VMOVSR"; 1282 1283 case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP"; 1284 case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP"; 1285 case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH"; 1286 1287 case ARMISD::TC_RETURN: return "ARMISD::TC_RETURN"; 1288 1289 case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER"; 1290 1291 case ARMISD::DYN_ALLOC: return "ARMISD::DYN_ALLOC"; 1292 1293 case ARMISD::MEMBARRIER_MCR: return "ARMISD::MEMBARRIER_MCR"; 1294 1295 case ARMISD::PRELOAD: return "ARMISD::PRELOAD"; 1296 1297 case ARMISD::WIN__CHKSTK: return "ARMISD::WIN__CHKSTK"; 1298 case ARMISD::WIN__DBZCHK: return "ARMISD::WIN__DBZCHK"; 1299 1300 case ARMISD::VCEQ: return "ARMISD::VCEQ"; 1301 case ARMISD::VCEQZ: return "ARMISD::VCEQZ"; 1302 case ARMISD::VCGE: return "ARMISD::VCGE"; 1303 case ARMISD::VCGEZ: return "ARMISD::VCGEZ"; 1304 case ARMISD::VCLEZ: return "ARMISD::VCLEZ"; 1305 case ARMISD::VCGEU: return "ARMISD::VCGEU"; 1306 case ARMISD::VCGT: return "ARMISD::VCGT"; 1307 case ARMISD::VCGTZ: return "ARMISD::VCGTZ"; 1308 case ARMISD::VCLTZ: return "ARMISD::VCLTZ"; 1309 case ARMISD::VCGTU: return "ARMISD::VCGTU"; 1310 case ARMISD::VTST: return "ARMISD::VTST"; 1311 1312 case ARMISD::VSHL: return "ARMISD::VSHL"; 1313 case ARMISD::VSHRs: return "ARMISD::VSHRs"; 1314 case ARMISD::VSHRu: return "ARMISD::VSHRu"; 1315 case ARMISD::VRSHRs: return "ARMISD::VRSHRs"; 1316 case ARMISD::VRSHRu: return "ARMISD::VRSHRu"; 1317 case ARMISD::VRSHRN: return "ARMISD::VRSHRN"; 1318 case ARMISD::VQSHLs: return "ARMISD::VQSHLs"; 1319 case ARMISD::VQSHLu: return "ARMISD::VQSHLu"; 1320 case ARMISD::VQSHLsu: return "ARMISD::VQSHLsu"; 1321 case ARMISD::VQSHRNs: return "ARMISD::VQSHRNs"; 1322 case ARMISD::VQSHRNu: return "ARMISD::VQSHRNu"; 1323 case ARMISD::VQSHRNsu: return "ARMISD::VQSHRNsu"; 1324 case ARMISD::VQRSHRNs: return "ARMISD::VQRSHRNs"; 1325 case ARMISD::VQRSHRNu: return "ARMISD::VQRSHRNu"; 1326 case ARMISD::VQRSHRNsu: return "ARMISD::VQRSHRNsu"; 1327 case ARMISD::VSLI: return "ARMISD::VSLI"; 1328 case ARMISD::VSRI: return "ARMISD::VSRI"; 1329 case ARMISD::VGETLANEu: return "ARMISD::VGETLANEu"; 1330 case ARMISD::VGETLANEs: return "ARMISD::VGETLANEs"; 1331 case ARMISD::VMOVIMM: return "ARMISD::VMOVIMM"; 1332 case ARMISD::VMVNIMM: return "ARMISD::VMVNIMM"; 1333 case ARMISD::VMOVFPIMM: return "ARMISD::VMOVFPIMM"; 1334 case ARMISD::VDUP: return "ARMISD::VDUP"; 1335 case ARMISD::VDUPLANE: return "ARMISD::VDUPLANE"; 1336 case ARMISD::VEXT: return "ARMISD::VEXT"; 1337 case ARMISD::VREV64: return "ARMISD::VREV64"; 1338 case ARMISD::VREV32: return "ARMISD::VREV32"; 1339 case ARMISD::VREV16: return "ARMISD::VREV16"; 1340 case ARMISD::VZIP: return "ARMISD::VZIP"; 1341 case ARMISD::VUZP: return "ARMISD::VUZP"; 1342 case ARMISD::VTRN: return "ARMISD::VTRN"; 1343 case ARMISD::VTBL1: return "ARMISD::VTBL1"; 1344 case ARMISD::VTBL2: return "ARMISD::VTBL2"; 1345 case ARMISD::VMULLs: return "ARMISD::VMULLs"; 1346 case ARMISD::VMULLu: return "ARMISD::VMULLu"; 1347 case ARMISD::UMAAL: return "ARMISD::UMAAL"; 1348 case ARMISD::UMLAL: return "ARMISD::UMLAL"; 1349 case ARMISD::SMLAL: return "ARMISD::SMLAL"; 1350 case ARMISD::SMLALBB: return "ARMISD::SMLALBB"; 1351 case ARMISD::SMLALBT: return "ARMISD::SMLALBT"; 1352 case ARMISD::SMLALTB: return "ARMISD::SMLALTB"; 1353 case ARMISD::SMLALTT: return "ARMISD::SMLALTT"; 1354 case ARMISD::SMULWB: return "ARMISD::SMULWB"; 1355 case ARMISD::SMULWT: return "ARMISD::SMULWT"; 1356 case ARMISD::SMLALD: return "ARMISD::SMLALD"; 1357 case ARMISD::SMLALDX: return "ARMISD::SMLALDX"; 1358 case ARMISD::SMLSLD: return "ARMISD::SMLSLD"; 1359 case ARMISD::SMLSLDX: return "ARMISD::SMLSLDX"; 1360 case ARMISD::SMMLAR: return "ARMISD::SMMLAR"; 1361 case ARMISD::SMMLSR: return "ARMISD::SMMLSR"; 1362 case ARMISD::BUILD_VECTOR: return "ARMISD::BUILD_VECTOR"; 1363 case ARMISD::BFI: return "ARMISD::BFI"; 1364 case ARMISD::VORRIMM: return "ARMISD::VORRIMM"; 1365 case ARMISD::VBICIMM: return "ARMISD::VBICIMM"; 1366 case ARMISD::VBSL: return "ARMISD::VBSL"; 1367 case ARMISD::MEMCPY: return "ARMISD::MEMCPY"; 1368 case ARMISD::VLD1DUP: return "ARMISD::VLD1DUP"; 1369 case ARMISD::VLD2DUP: return "ARMISD::VLD2DUP"; 1370 case ARMISD::VLD3DUP: return "ARMISD::VLD3DUP"; 1371 case ARMISD::VLD4DUP: return "ARMISD::VLD4DUP"; 1372 case ARMISD::VLD1_UPD: return "ARMISD::VLD1_UPD"; 1373 case ARMISD::VLD2_UPD: return "ARMISD::VLD2_UPD"; 1374 case ARMISD::VLD3_UPD: return "ARMISD::VLD3_UPD"; 1375 case ARMISD::VLD4_UPD: return "ARMISD::VLD4_UPD"; 1376 case ARMISD::VLD2LN_UPD: return "ARMISD::VLD2LN_UPD"; 1377 case ARMISD::VLD3LN_UPD: return "ARMISD::VLD3LN_UPD"; 1378 case ARMISD::VLD4LN_UPD: return "ARMISD::VLD4LN_UPD"; 1379 case ARMISD::VLD1DUP_UPD: return "ARMISD::VLD1DUP_UPD"; 1380 case ARMISD::VLD2DUP_UPD: return "ARMISD::VLD2DUP_UPD"; 1381 case ARMISD::VLD3DUP_UPD: return "ARMISD::VLD3DUP_UPD"; 1382 case ARMISD::VLD4DUP_UPD: return "ARMISD::VLD4DUP_UPD"; 1383 case ARMISD::VST1_UPD: return "ARMISD::VST1_UPD"; 1384 case ARMISD::VST2_UPD: return "ARMISD::VST2_UPD"; 1385 case ARMISD::VST3_UPD: return "ARMISD::VST3_UPD"; 1386 case ARMISD::VST4_UPD: return "ARMISD::VST4_UPD"; 1387 case ARMISD::VST2LN_UPD: return "ARMISD::VST2LN_UPD"; 1388 case ARMISD::VST3LN_UPD: return "ARMISD::VST3LN_UPD"; 1389 case ARMISD::VST4LN_UPD: return "ARMISD::VST4LN_UPD"; 1390 } 1391 return nullptr; 1392 } 1393 1394 EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1395 EVT VT) const { 1396 if (!VT.isVector()) 1397 return getPointerTy(DL); 1398 return VT.changeVectorElementTypeToInteger(); 1399 } 1400 1401 /// getRegClassFor - Return the register class that should be used for the 1402 /// specified value type. 1403 const TargetRegisterClass *ARMTargetLowering::getRegClassFor(MVT VT) const { 1404 // Map v4i64 to QQ registers but do not make the type legal. Similarly map 1405 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to 1406 // load / store 4 to 8 consecutive D registers. 1407 if (Subtarget->hasNEON()) { 1408 if (VT == MVT::v4i64) 1409 return &ARM::QQPRRegClass; 1410 if (VT == MVT::v8i64) 1411 return &ARM::QQQQPRRegClass; 1412 } 1413 return TargetLowering::getRegClassFor(VT); 1414 } 1415 1416 // memcpy, and other memory intrinsics, typically tries to use LDM/STM if the 1417 // source/dest is aligned and the copy size is large enough. We therefore want 1418 // to align such objects passed to memory intrinsics. 1419 bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, 1420 unsigned &PrefAlign) const { 1421 if (!isa<MemIntrinsic>(CI)) 1422 return false; 1423 MinSize = 8; 1424 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1 1425 // cycle faster than 4-byte aligned LDM. 1426 PrefAlign = (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? 8 : 4); 1427 return true; 1428 } 1429 1430 // Create a fast isel object. 1431 FastISel * 1432 ARMTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo, 1433 const TargetLibraryInfo *libInfo) const { 1434 return ARM::createFastISel(funcInfo, libInfo); 1435 } 1436 1437 Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const { 1438 unsigned NumVals = N->getNumValues(); 1439 if (!NumVals) 1440 return Sched::RegPressure; 1441 1442 for (unsigned i = 0; i != NumVals; ++i) { 1443 EVT VT = N->getValueType(i); 1444 if (VT == MVT::Glue || VT == MVT::Other) 1445 continue; 1446 if (VT.isFloatingPoint() || VT.isVector()) 1447 return Sched::ILP; 1448 } 1449 1450 if (!N->isMachineOpcode()) 1451 return Sched::RegPressure; 1452 1453 // Load are scheduled for latency even if there instruction itinerary 1454 // is not available. 1455 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 1456 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); 1457 1458 if (MCID.getNumDefs() == 0) 1459 return Sched::RegPressure; 1460 if (!Itins->isEmpty() && 1461 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2) 1462 return Sched::ILP; 1463 1464 return Sched::RegPressure; 1465 } 1466 1467 //===----------------------------------------------------------------------===// 1468 // Lowering Code 1469 //===----------------------------------------------------------------------===// 1470 1471 static bool isSRL16(const SDValue &Op) { 1472 if (Op.getOpcode() != ISD::SRL) 1473 return false; 1474 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 1475 return Const->getZExtValue() == 16; 1476 return false; 1477 } 1478 1479 static bool isSRA16(const SDValue &Op) { 1480 if (Op.getOpcode() != ISD::SRA) 1481 return false; 1482 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 1483 return Const->getZExtValue() == 16; 1484 return false; 1485 } 1486 1487 static bool isSHL16(const SDValue &Op) { 1488 if (Op.getOpcode() != ISD::SHL) 1489 return false; 1490 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 1491 return Const->getZExtValue() == 16; 1492 return false; 1493 } 1494 1495 // Check for a signed 16-bit value. We special case SRA because it makes it 1496 // more simple when also looking for SRAs that aren't sign extending a 1497 // smaller value. Without the check, we'd need to take extra care with 1498 // checking order for some operations. 1499 static bool isS16(const SDValue &Op, SelectionDAG &DAG) { 1500 if (isSRA16(Op)) 1501 return isSHL16(Op.getOperand(0)); 1502 return DAG.ComputeNumSignBits(Op) == 17; 1503 } 1504 1505 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC 1506 static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) { 1507 switch (CC) { 1508 default: llvm_unreachable("Unknown condition code!"); 1509 case ISD::SETNE: return ARMCC::NE; 1510 case ISD::SETEQ: return ARMCC::EQ; 1511 case ISD::SETGT: return ARMCC::GT; 1512 case ISD::SETGE: return ARMCC::GE; 1513 case ISD::SETLT: return ARMCC::LT; 1514 case ISD::SETLE: return ARMCC::LE; 1515 case ISD::SETUGT: return ARMCC::HI; 1516 case ISD::SETUGE: return ARMCC::HS; 1517 case ISD::SETULT: return ARMCC::LO; 1518 case ISD::SETULE: return ARMCC::LS; 1519 } 1520 } 1521 1522 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC. 1523 static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, 1524 ARMCC::CondCodes &CondCode2, bool &InvalidOnQNaN) { 1525 CondCode2 = ARMCC::AL; 1526 InvalidOnQNaN = true; 1527 switch (CC) { 1528 default: llvm_unreachable("Unknown FP condition!"); 1529 case ISD::SETEQ: 1530 case ISD::SETOEQ: 1531 CondCode = ARMCC::EQ; 1532 InvalidOnQNaN = false; 1533 break; 1534 case ISD::SETGT: 1535 case ISD::SETOGT: CondCode = ARMCC::GT; break; 1536 case ISD::SETGE: 1537 case ISD::SETOGE: CondCode = ARMCC::GE; break; 1538 case ISD::SETOLT: CondCode = ARMCC::MI; break; 1539 case ISD::SETOLE: CondCode = ARMCC::LS; break; 1540 case ISD::SETONE: 1541 CondCode = ARMCC::MI; 1542 CondCode2 = ARMCC::GT; 1543 InvalidOnQNaN = false; 1544 break; 1545 case ISD::SETO: CondCode = ARMCC::VC; break; 1546 case ISD::SETUO: CondCode = ARMCC::VS; break; 1547 case ISD::SETUEQ: 1548 CondCode = ARMCC::EQ; 1549 CondCode2 = ARMCC::VS; 1550 InvalidOnQNaN = false; 1551 break; 1552 case ISD::SETUGT: CondCode = ARMCC::HI; break; 1553 case ISD::SETUGE: CondCode = ARMCC::PL; break; 1554 case ISD::SETLT: 1555 case ISD::SETULT: CondCode = ARMCC::LT; break; 1556 case ISD::SETLE: 1557 case ISD::SETULE: CondCode = ARMCC::LE; break; 1558 case ISD::SETNE: 1559 case ISD::SETUNE: 1560 CondCode = ARMCC::NE; 1561 InvalidOnQNaN = false; 1562 break; 1563 } 1564 } 1565 1566 //===----------------------------------------------------------------------===// 1567 // Calling Convention Implementation 1568 //===----------------------------------------------------------------------===// 1569 1570 #include "ARMGenCallingConv.inc" 1571 1572 /// getEffectiveCallingConv - Get the effective calling convention, taking into 1573 /// account presence of floating point hardware and calling convention 1574 /// limitations, such as support for variadic functions. 1575 CallingConv::ID 1576 ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC, 1577 bool isVarArg) const { 1578 switch (CC) { 1579 default: 1580 report_fatal_error("Unsupported calling convention"); 1581 case CallingConv::ARM_AAPCS: 1582 case CallingConv::ARM_APCS: 1583 case CallingConv::GHC: 1584 return CC; 1585 case CallingConv::PreserveMost: 1586 return CallingConv::PreserveMost; 1587 case CallingConv::ARM_AAPCS_VFP: 1588 case CallingConv::Swift: 1589 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP; 1590 case CallingConv::C: 1591 if (!Subtarget->isAAPCS_ABI()) 1592 return CallingConv::ARM_APCS; 1593 else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && 1594 getTargetMachine().Options.FloatABIType == FloatABI::Hard && 1595 !isVarArg) 1596 return CallingConv::ARM_AAPCS_VFP; 1597 else 1598 return CallingConv::ARM_AAPCS; 1599 case CallingConv::Fast: 1600 case CallingConv::CXX_FAST_TLS: 1601 if (!Subtarget->isAAPCS_ABI()) { 1602 if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg) 1603 return CallingConv::Fast; 1604 return CallingConv::ARM_APCS; 1605 } else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg) 1606 return CallingConv::ARM_AAPCS_VFP; 1607 else 1608 return CallingConv::ARM_AAPCS; 1609 } 1610 } 1611 1612 CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC, 1613 bool isVarArg) const { 1614 return CCAssignFnForNode(CC, false, isVarArg); 1615 } 1616 1617 CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC, 1618 bool isVarArg) const { 1619 return CCAssignFnForNode(CC, true, isVarArg); 1620 } 1621 1622 /// CCAssignFnForNode - Selects the correct CCAssignFn for the given 1623 /// CallingConvention. 1624 CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC, 1625 bool Return, 1626 bool isVarArg) const { 1627 switch (getEffectiveCallingConv(CC, isVarArg)) { 1628 default: 1629 report_fatal_error("Unsupported calling convention"); 1630 case CallingConv::ARM_APCS: 1631 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS); 1632 case CallingConv::ARM_AAPCS: 1633 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS); 1634 case CallingConv::ARM_AAPCS_VFP: 1635 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP); 1636 case CallingConv::Fast: 1637 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS); 1638 case CallingConv::GHC: 1639 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC); 1640 case CallingConv::PreserveMost: 1641 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS); 1642 } 1643 } 1644 1645 /// LowerCallResult - Lower the result values of a call into the 1646 /// appropriate copies out of appropriate physical registers. 1647 SDValue ARMTargetLowering::LowerCallResult( 1648 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 1649 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 1650 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn, 1651 SDValue ThisVal) const { 1652 // Assign locations to each value returned by this call. 1653 SmallVector<CCValAssign, 16> RVLocs; 1654 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 1655 *DAG.getContext()); 1656 CCInfo.AnalyzeCallResult(Ins, CCAssignFnForReturn(CallConv, isVarArg)); 1657 1658 // Copy all of the result registers out of their specified physreg. 1659 for (unsigned i = 0; i != RVLocs.size(); ++i) { 1660 CCValAssign VA = RVLocs[i]; 1661 1662 // Pass 'this' value directly from the argument to return value, to avoid 1663 // reg unit interference 1664 if (i == 0 && isThisReturn) { 1665 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 && 1666 "unexpected return calling convention register assignment"); 1667 InVals.push_back(ThisVal); 1668 continue; 1669 } 1670 1671 SDValue Val; 1672 if (VA.needsCustom()) { 1673 // Handle f64 or half of a v2f64. 1674 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 1675 InFlag); 1676 Chain = Lo.getValue(1); 1677 InFlag = Lo.getValue(2); 1678 VA = RVLocs[++i]; // skip ahead to next loc 1679 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 1680 InFlag); 1681 Chain = Hi.getValue(1); 1682 InFlag = Hi.getValue(2); 1683 if (!Subtarget->isLittle()) 1684 std::swap (Lo, Hi); 1685 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi); 1686 1687 if (VA.getLocVT() == MVT::v2f64) { 1688 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64); 1689 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val, 1690 DAG.getConstant(0, dl, MVT::i32)); 1691 1692 VA = RVLocs[++i]; // skip ahead to next loc 1693 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); 1694 Chain = Lo.getValue(1); 1695 InFlag = Lo.getValue(2); 1696 VA = RVLocs[++i]; // skip ahead to next loc 1697 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); 1698 Chain = Hi.getValue(1); 1699 InFlag = Hi.getValue(2); 1700 if (!Subtarget->isLittle()) 1701 std::swap (Lo, Hi); 1702 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi); 1703 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val, 1704 DAG.getConstant(1, dl, MVT::i32)); 1705 } 1706 } else { 1707 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(), 1708 InFlag); 1709 Chain = Val.getValue(1); 1710 InFlag = Val.getValue(2); 1711 } 1712 1713 switch (VA.getLocInfo()) { 1714 default: llvm_unreachable("Unknown loc info!"); 1715 case CCValAssign::Full: break; 1716 case CCValAssign::BCvt: 1717 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val); 1718 break; 1719 } 1720 1721 InVals.push_back(Val); 1722 } 1723 1724 return Chain; 1725 } 1726 1727 /// LowerMemOpCallTo - Store the argument to the stack. 1728 SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, 1729 SDValue Arg, const SDLoc &dl, 1730 SelectionDAG &DAG, 1731 const CCValAssign &VA, 1732 ISD::ArgFlagsTy Flags) const { 1733 unsigned LocMemOffset = VA.getLocMemOffset(); 1734 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 1735 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), 1736 StackPtr, PtrOff); 1737 return DAG.getStore( 1738 Chain, dl, Arg, PtrOff, 1739 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset)); 1740 } 1741 1742 void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, 1743 SDValue Chain, SDValue &Arg, 1744 RegsToPassVector &RegsToPass, 1745 CCValAssign &VA, CCValAssign &NextVA, 1746 SDValue &StackPtr, 1747 SmallVectorImpl<SDValue> &MemOpChains, 1748 ISD::ArgFlagsTy Flags) const { 1749 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl, 1750 DAG.getVTList(MVT::i32, MVT::i32), Arg); 1751 unsigned id = Subtarget->isLittle() ? 0 : 1; 1752 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id))); 1753 1754 if (NextVA.isRegLoc()) 1755 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id))); 1756 else { 1757 assert(NextVA.isMemLoc()); 1758 if (!StackPtr.getNode()) 1759 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, 1760 getPointerTy(DAG.getDataLayout())); 1761 1762 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id), 1763 dl, DAG, NextVA, 1764 Flags)); 1765 } 1766 } 1767 1768 /// LowerCall - Lowering a call into a callseq_start <- 1769 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter 1770 /// nodes. 1771 SDValue 1772 ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 1773 SmallVectorImpl<SDValue> &InVals) const { 1774 SelectionDAG &DAG = CLI.DAG; 1775 SDLoc &dl = CLI.DL; 1776 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 1777 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 1778 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 1779 SDValue Chain = CLI.Chain; 1780 SDValue Callee = CLI.Callee; 1781 bool &isTailCall = CLI.IsTailCall; 1782 CallingConv::ID CallConv = CLI.CallConv; 1783 bool doesNotRet = CLI.DoesNotReturn; 1784 bool isVarArg = CLI.IsVarArg; 1785 1786 MachineFunction &MF = DAG.getMachineFunction(); 1787 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet(); 1788 bool isThisReturn = false; 1789 bool isSibCall = false; 1790 auto Attr = MF.getFunction().getFnAttribute("disable-tail-calls"); 1791 1792 // Disable tail calls if they're not supported. 1793 if (!Subtarget->supportsTailCall() || Attr.getValueAsString() == "true") 1794 isTailCall = false; 1795 1796 if (isTailCall) { 1797 // Check if it's really possible to do a tail call. 1798 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, 1799 isVarArg, isStructRet, MF.getFunction().hasStructRetAttr(), 1800 Outs, OutVals, Ins, DAG); 1801 if (!isTailCall && CLI.CS && CLI.CS.isMustTailCall()) 1802 report_fatal_error("failed to perform tail call elimination on a call " 1803 "site marked musttail"); 1804 // We don't support GuaranteedTailCallOpt for ARM, only automatically 1805 // detected sibcalls. 1806 if (isTailCall) { 1807 ++NumTailCalls; 1808 isSibCall = true; 1809 } 1810 } 1811 1812 // Analyze operands of the call, assigning locations to each operand. 1813 SmallVector<CCValAssign, 16> ArgLocs; 1814 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 1815 *DAG.getContext()); 1816 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg)); 1817 1818 // Get a count of how many bytes are to be pushed on the stack. 1819 unsigned NumBytes = CCInfo.getNextStackOffset(); 1820 1821 // For tail calls, memory operands are available in our caller's stack. 1822 if (isSibCall) 1823 NumBytes = 0; 1824 1825 // Adjust the stack pointer for the new arguments... 1826 // These operations are automatically eliminated by the prolog/epilog pass 1827 if (!isSibCall) 1828 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 1829 1830 SDValue StackPtr = 1831 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout())); 1832 1833 RegsToPassVector RegsToPass; 1834 SmallVector<SDValue, 8> MemOpChains; 1835 1836 // Walk the register/memloc assignments, inserting copies/loads. In the case 1837 // of tail call optimization, arguments are handled later. 1838 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); 1839 i != e; 1840 ++i, ++realArgIdx) { 1841 CCValAssign &VA = ArgLocs[i]; 1842 SDValue Arg = OutVals[realArgIdx]; 1843 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; 1844 bool isByVal = Flags.isByVal(); 1845 1846 // Promote the value if needed. 1847 switch (VA.getLocInfo()) { 1848 default: llvm_unreachable("Unknown loc info!"); 1849 case CCValAssign::Full: break; 1850 case CCValAssign::SExt: 1851 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 1852 break; 1853 case CCValAssign::ZExt: 1854 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 1855 break; 1856 case CCValAssign::AExt: 1857 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 1858 break; 1859 case CCValAssign::BCvt: 1860 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); 1861 break; 1862 } 1863 1864 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces 1865 if (VA.needsCustom()) { 1866 if (VA.getLocVT() == MVT::v2f64) { 1867 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 1868 DAG.getConstant(0, dl, MVT::i32)); 1869 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 1870 DAG.getConstant(1, dl, MVT::i32)); 1871 1872 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, 1873 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags); 1874 1875 VA = ArgLocs[++i]; // skip ahead to next loc 1876 if (VA.isRegLoc()) { 1877 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, 1878 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags); 1879 } else { 1880 assert(VA.isMemLoc()); 1881 1882 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Op1, 1883 dl, DAG, VA, Flags)); 1884 } 1885 } else { 1886 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i], 1887 StackPtr, MemOpChains, Flags); 1888 } 1889 } else if (VA.isRegLoc()) { 1890 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() && 1891 Outs[0].VT == MVT::i32) { 1892 assert(VA.getLocVT() == MVT::i32 && 1893 "unexpected calling convention register assignment"); 1894 assert(!Ins.empty() && Ins[0].VT == MVT::i32 && 1895 "unexpected use of 'returned'"); 1896 isThisReturn = true; 1897 } 1898 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1899 } else if (isByVal) { 1900 assert(VA.isMemLoc()); 1901 unsigned offset = 0; 1902 1903 // True if this byval aggregate will be split between registers 1904 // and memory. 1905 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount(); 1906 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed(); 1907 1908 if (CurByValIdx < ByValArgsCount) { 1909 1910 unsigned RegBegin, RegEnd; 1911 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd); 1912 1913 EVT PtrVT = 1914 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 1915 unsigned int i, j; 1916 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) { 1917 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32); 1918 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 1919 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg, 1920 MachinePointerInfo(), 1921 DAG.InferPtrAlignment(AddArg)); 1922 MemOpChains.push_back(Load.getValue(1)); 1923 RegsToPass.push_back(std::make_pair(j, Load)); 1924 } 1925 1926 // If parameter size outsides register area, "offset" value 1927 // helps us to calculate stack slot for remained part properly. 1928 offset = RegEnd - RegBegin; 1929 1930 CCInfo.nextInRegsParam(); 1931 } 1932 1933 if (Flags.getByValSize() > 4*offset) { 1934 auto PtrVT = getPointerTy(DAG.getDataLayout()); 1935 unsigned LocMemOffset = VA.getLocMemOffset(); 1936 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 1937 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff); 1938 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl); 1939 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset); 1940 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl, 1941 MVT::i32); 1942 SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), dl, 1943 MVT::i32); 1944 1945 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 1946 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode}; 1947 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, 1948 Ops)); 1949 } 1950 } else if (!isSibCall) { 1951 assert(VA.isMemLoc()); 1952 1953 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, 1954 dl, DAG, VA, Flags)); 1955 } 1956 } 1957 1958 if (!MemOpChains.empty()) 1959 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 1960 1961 // Build a sequence of copy-to-reg nodes chained together with token chain 1962 // and flag operands which copy the outgoing args into the appropriate regs. 1963 SDValue InFlag; 1964 // Tail call byval lowering might overwrite argument registers so in case of 1965 // tail call optimization the copies to registers are lowered later. 1966 if (!isTailCall) 1967 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1968 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 1969 RegsToPass[i].second, InFlag); 1970 InFlag = Chain.getValue(1); 1971 } 1972 1973 // For tail calls lower the arguments to the 'real' stack slot. 1974 if (isTailCall) { 1975 // Force all the incoming stack arguments to be loaded from the stack 1976 // before any new outgoing arguments are stored to the stack, because the 1977 // outgoing stack slots may alias the incoming argument stack slots, and 1978 // the alias isn't otherwise explicit. This is slightly more conservative 1979 // than necessary, because it means that each store effectively depends 1980 // on every argument instead of just those arguments it would clobber. 1981 1982 // Do not flag preceding copytoreg stuff together with the following stuff. 1983 InFlag = SDValue(); 1984 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1985 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 1986 RegsToPass[i].second, InFlag); 1987 InFlag = Chain.getValue(1); 1988 } 1989 InFlag = SDValue(); 1990 } 1991 1992 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every 1993 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol 1994 // node so that legalize doesn't hack it. 1995 bool isDirect = false; 1996 1997 const TargetMachine &TM = getTargetMachine(); 1998 const Module *Mod = MF.getFunction().getParent(); 1999 const GlobalValue *GV = nullptr; 2000 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 2001 GV = G->getGlobal(); 2002 bool isStub = 2003 !TM.shouldAssumeDSOLocal(*Mod, GV) && Subtarget->isTargetMachO(); 2004 2005 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass()); 2006 bool isLocalARMFunc = false; 2007 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2008 auto PtrVt = getPointerTy(DAG.getDataLayout()); 2009 2010 if (Subtarget->genLongCalls()) { 2011 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) && 2012 "long-calls codegen is not position independent!"); 2013 // Handle a global address or an external symbol. If it's not one of 2014 // those, the target's already in a register, so we don't need to do 2015 // anything extra. 2016 if (isa<GlobalAddressSDNode>(Callee)) { 2017 // Create a constant pool entry for the callee address 2018 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2019 ARMConstantPoolValue *CPV = 2020 ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0); 2021 2022 // Get the address of the callee into a register 2023 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4); 2024 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 2025 Callee = DAG.getLoad( 2026 PtrVt, dl, DAG.getEntryNode(), CPAddr, 2027 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2028 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) { 2029 const char *Sym = S->getSymbol(); 2030 2031 // Create a constant pool entry for the callee address 2032 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2033 ARMConstantPoolValue *CPV = 2034 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym, 2035 ARMPCLabelIndex, 0); 2036 // Get the address of the callee into a register 2037 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4); 2038 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 2039 Callee = DAG.getLoad( 2040 PtrVt, dl, DAG.getEntryNode(), CPAddr, 2041 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2042 } 2043 } else if (isa<GlobalAddressSDNode>(Callee)) { 2044 // If we're optimizing for minimum size and the function is called three or 2045 // more times in this block, we can improve codesize by calling indirectly 2046 // as BLXr has a 16-bit encoding. 2047 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal(); 2048 auto *BB = CLI.CS.getParent(); 2049 bool PreferIndirect = 2050 Subtarget->isThumb() && MF.getFunction().optForMinSize() && 2051 count_if(GV->users(), [&BB](const User *U) { 2052 return isa<Instruction>(U) && cast<Instruction>(U)->getParent() == BB; 2053 }) > 2; 2054 2055 if (!PreferIndirect) { 2056 isDirect = true; 2057 bool isDef = GV->isStrongDefinitionForLinker(); 2058 2059 // ARM call to a local ARM function is predicable. 2060 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking); 2061 // tBX takes a register source operand. 2062 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { 2063 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?"); 2064 Callee = DAG.getNode( 2065 ARMISD::WrapperPIC, dl, PtrVt, 2066 DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY)); 2067 Callee = DAG.getLoad( 2068 PtrVt, dl, DAG.getEntryNode(), Callee, 2069 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 2070 /* Alignment = */ 0, MachineMemOperand::MODereferenceable | 2071 MachineMemOperand::MOInvariant); 2072 } else if (Subtarget->isTargetCOFF()) { 2073 assert(Subtarget->isTargetWindows() && 2074 "Windows is the only supported COFF target"); 2075 unsigned TargetFlags = GV->hasDLLImportStorageClass() 2076 ? ARMII::MO_DLLIMPORT 2077 : ARMII::MO_NO_FLAG; 2078 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, /*Offset=*/0, 2079 TargetFlags); 2080 if (GV->hasDLLImportStorageClass()) 2081 Callee = 2082 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), 2083 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee), 2084 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 2085 } else { 2086 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, 0); 2087 } 2088 } 2089 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 2090 isDirect = true; 2091 // tBX takes a register source operand. 2092 const char *Sym = S->getSymbol(); 2093 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { 2094 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2095 ARMConstantPoolValue *CPV = 2096 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym, 2097 ARMPCLabelIndex, 4); 2098 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4); 2099 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 2100 Callee = DAG.getLoad( 2101 PtrVt, dl, DAG.getEntryNode(), CPAddr, 2102 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2103 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 2104 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel); 2105 } else { 2106 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0); 2107 } 2108 } 2109 2110 // FIXME: handle tail calls differently. 2111 unsigned CallOpc; 2112 if (Subtarget->isThumb()) { 2113 if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps()) 2114 CallOpc = ARMISD::CALL_NOLINK; 2115 else 2116 CallOpc = ARMISD::CALL; 2117 } else { 2118 if (!isDirect && !Subtarget->hasV5TOps()) 2119 CallOpc = ARMISD::CALL_NOLINK; 2120 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() && 2121 // Emit regular call when code size is the priority 2122 !MF.getFunction().optForMinSize()) 2123 // "mov lr, pc; b _foo" to avoid confusing the RSP 2124 CallOpc = ARMISD::CALL_NOLINK; 2125 else 2126 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL; 2127 } 2128 2129 std::vector<SDValue> Ops; 2130 Ops.push_back(Chain); 2131 Ops.push_back(Callee); 2132 2133 // Add argument registers to the end of the list so that they are known live 2134 // into the call. 2135 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 2136 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 2137 RegsToPass[i].second.getValueType())); 2138 2139 // Add a register mask operand representing the call-preserved registers. 2140 if (!isTailCall) { 2141 const uint32_t *Mask; 2142 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo(); 2143 if (isThisReturn) { 2144 // For 'this' returns, use the R0-preserving mask if applicable 2145 Mask = ARI->getThisReturnPreservedMask(MF, CallConv); 2146 if (!Mask) { 2147 // Set isThisReturn to false if the calling convention is not one that 2148 // allows 'returned' to be modeled in this way, so LowerCallResult does 2149 // not try to pass 'this' straight through 2150 isThisReturn = false; 2151 Mask = ARI->getCallPreservedMask(MF, CallConv); 2152 } 2153 } else 2154 Mask = ARI->getCallPreservedMask(MF, CallConv); 2155 2156 assert(Mask && "Missing call preserved mask for calling convention"); 2157 Ops.push_back(DAG.getRegisterMask(Mask)); 2158 } 2159 2160 if (InFlag.getNode()) 2161 Ops.push_back(InFlag); 2162 2163 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2164 if (isTailCall) { 2165 MF.getFrameInfo().setHasTailCall(); 2166 return DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops); 2167 } 2168 2169 // Returns a chain and a flag for retval copy to use. 2170 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); 2171 InFlag = Chain.getValue(1); 2172 2173 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 2174 DAG.getIntPtrConstant(0, dl, true), InFlag, dl); 2175 if (!Ins.empty()) 2176 InFlag = Chain.getValue(1); 2177 2178 // Handle result values, copying them out of physregs into vregs that we 2179 // return. 2180 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG, 2181 InVals, isThisReturn, 2182 isThisReturn ? OutVals[0] : SDValue()); 2183 } 2184 2185 /// HandleByVal - Every parameter *after* a byval parameter is passed 2186 /// on the stack. Remember the next parameter register to allocate, 2187 /// and then confiscate the rest of the parameter registers to insure 2188 /// this. 2189 void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size, 2190 unsigned Align) const { 2191 // Byval (as with any stack) slots are always at least 4 byte aligned. 2192 Align = std::max(Align, 4U); 2193 2194 unsigned Reg = State->AllocateReg(GPRArgRegs); 2195 if (!Reg) 2196 return; 2197 2198 unsigned AlignInRegs = Align / 4; 2199 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs; 2200 for (unsigned i = 0; i < Waste; ++i) 2201 Reg = State->AllocateReg(GPRArgRegs); 2202 2203 if (!Reg) 2204 return; 2205 2206 unsigned Excess = 4 * (ARM::R4 - Reg); 2207 2208 // Special case when NSAA != SP and parameter size greater than size of 2209 // all remained GPR regs. In that case we can't split parameter, we must 2210 // send it to stack. We also must set NCRN to R4, so waste all 2211 // remained registers. 2212 const unsigned NSAAOffset = State->getNextStackOffset(); 2213 if (NSAAOffset != 0 && Size > Excess) { 2214 while (State->AllocateReg(GPRArgRegs)) 2215 ; 2216 return; 2217 } 2218 2219 // First register for byval parameter is the first register that wasn't 2220 // allocated before this method call, so it would be "reg". 2221 // If parameter is small enough to be saved in range [reg, r4), then 2222 // the end (first after last) register would be reg + param-size-in-regs, 2223 // else parameter would be splitted between registers and stack, 2224 // end register would be r4 in this case. 2225 unsigned ByValRegBegin = Reg; 2226 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4); 2227 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd); 2228 // Note, first register is allocated in the beginning of function already, 2229 // allocate remained amount of registers we need. 2230 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i) 2231 State->AllocateReg(GPRArgRegs); 2232 // A byval parameter that is split between registers and memory needs its 2233 // size truncated here. 2234 // In the case where the entire structure fits in registers, we set the 2235 // size in memory to zero. 2236 Size = std::max<int>(Size - Excess, 0); 2237 } 2238 2239 /// MatchingStackOffset - Return true if the given stack call argument is 2240 /// already available in the same position (relatively) of the caller's 2241 /// incoming argument stack. 2242 static 2243 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, 2244 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, 2245 const TargetInstrInfo *TII) { 2246 unsigned Bytes = Arg.getValueSizeInBits() / 8; 2247 int FI = std::numeric_limits<int>::max(); 2248 if (Arg.getOpcode() == ISD::CopyFromReg) { 2249 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); 2250 if (!TargetRegisterInfo::isVirtualRegister(VR)) 2251 return false; 2252 MachineInstr *Def = MRI->getVRegDef(VR); 2253 if (!Def) 2254 return false; 2255 if (!Flags.isByVal()) { 2256 if (!TII->isLoadFromStackSlot(*Def, FI)) 2257 return false; 2258 } else { 2259 return false; 2260 } 2261 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) { 2262 if (Flags.isByVal()) 2263 // ByVal argument is passed in as a pointer but it's now being 2264 // dereferenced. e.g. 2265 // define @foo(%struct.X* %A) { 2266 // tail call @bar(%struct.X* byval %A) 2267 // } 2268 return false; 2269 SDValue Ptr = Ld->getBasePtr(); 2270 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr); 2271 if (!FINode) 2272 return false; 2273 FI = FINode->getIndex(); 2274 } else 2275 return false; 2276 2277 assert(FI != std::numeric_limits<int>::max()); 2278 if (!MFI.isFixedObjectIndex(FI)) 2279 return false; 2280 return Offset == MFI.getObjectOffset(FI) && Bytes == MFI.getObjectSize(FI); 2281 } 2282 2283 /// IsEligibleForTailCallOptimization - Check whether the call is eligible 2284 /// for tail call optimization. Targets which want to do tail call 2285 /// optimization should implement this function. 2286 bool 2287 ARMTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 2288 CallingConv::ID CalleeCC, 2289 bool isVarArg, 2290 bool isCalleeStructRet, 2291 bool isCallerStructRet, 2292 const SmallVectorImpl<ISD::OutputArg> &Outs, 2293 const SmallVectorImpl<SDValue> &OutVals, 2294 const SmallVectorImpl<ISD::InputArg> &Ins, 2295 SelectionDAG& DAG) const { 2296 MachineFunction &MF = DAG.getMachineFunction(); 2297 const Function &CallerF = MF.getFunction(); 2298 CallingConv::ID CallerCC = CallerF.getCallingConv(); 2299 2300 assert(Subtarget->supportsTailCall()); 2301 2302 // Tail calls to function pointers cannot be optimized for Thumb1 if the args 2303 // to the call take up r0-r3. The reason is that there are no legal registers 2304 // left to hold the pointer to the function to be called. 2305 if (Subtarget->isThumb1Only() && Outs.size() >= 4 && 2306 !isa<GlobalAddressSDNode>(Callee.getNode())) 2307 return false; 2308 2309 // Look for obvious safe cases to perform tail call optimization that do not 2310 // require ABI changes. This is what gcc calls sibcall. 2311 2312 // Exception-handling functions need a special set of instructions to indicate 2313 // a return to the hardware. Tail-calling another function would probably 2314 // break this. 2315 if (CallerF.hasFnAttribute("interrupt")) 2316 return false; 2317 2318 // Also avoid sibcall optimization if either caller or callee uses struct 2319 // return semantics. 2320 if (isCalleeStructRet || isCallerStructRet) 2321 return false; 2322 2323 // Externally-defined functions with weak linkage should not be 2324 // tail-called on ARM when the OS does not support dynamic 2325 // pre-emption of symbols, as the AAELF spec requires normal calls 2326 // to undefined weak functions to be replaced with a NOP or jump to the 2327 // next instruction. The behaviour of branch instructions in this 2328 // situation (as used for tail calls) is implementation-defined, so we 2329 // cannot rely on the linker replacing the tail call with a return. 2330 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 2331 const GlobalValue *GV = G->getGlobal(); 2332 const Triple &TT = getTargetMachine().getTargetTriple(); 2333 if (GV->hasExternalWeakLinkage() && 2334 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO())) 2335 return false; 2336 } 2337 2338 // Check that the call results are passed in the same way. 2339 LLVMContext &C = *DAG.getContext(); 2340 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins, 2341 CCAssignFnForReturn(CalleeCC, isVarArg), 2342 CCAssignFnForReturn(CallerCC, isVarArg))) 2343 return false; 2344 // The callee has to preserve all registers the caller needs to preserve. 2345 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); 2346 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); 2347 if (CalleeCC != CallerCC) { 2348 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); 2349 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) 2350 return false; 2351 } 2352 2353 // If Caller's vararg or byval argument has been split between registers and 2354 // stack, do not perform tail call, since part of the argument is in caller's 2355 // local frame. 2356 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>(); 2357 if (AFI_Caller->getArgRegsSaveSize()) 2358 return false; 2359 2360 // If the callee takes no arguments then go on to check the results of the 2361 // call. 2362 if (!Outs.empty()) { 2363 // Check if stack adjustment is needed. For now, do not do this if any 2364 // argument is passed on the stack. 2365 SmallVector<CCValAssign, 16> ArgLocs; 2366 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C); 2367 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg)); 2368 if (CCInfo.getNextStackOffset()) { 2369 // Check if the arguments are already laid out in the right way as 2370 // the caller's fixed stack objects. 2371 MachineFrameInfo &MFI = MF.getFrameInfo(); 2372 const MachineRegisterInfo *MRI = &MF.getRegInfo(); 2373 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 2374 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); 2375 i != e; 2376 ++i, ++realArgIdx) { 2377 CCValAssign &VA = ArgLocs[i]; 2378 EVT RegVT = VA.getLocVT(); 2379 SDValue Arg = OutVals[realArgIdx]; 2380 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; 2381 if (VA.getLocInfo() == CCValAssign::Indirect) 2382 return false; 2383 if (VA.needsCustom()) { 2384 // f64 and vector types are split into multiple registers or 2385 // register/stack-slot combinations. The types will not match 2386 // the registers; give up on memory f64 refs until we figure 2387 // out what to do about this. 2388 if (!VA.isRegLoc()) 2389 return false; 2390 if (!ArgLocs[++i].isRegLoc()) 2391 return false; 2392 if (RegVT == MVT::v2f64) { 2393 if (!ArgLocs[++i].isRegLoc()) 2394 return false; 2395 if (!ArgLocs[++i].isRegLoc()) 2396 return false; 2397 } 2398 } else if (!VA.isRegLoc()) { 2399 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags, 2400 MFI, MRI, TII)) 2401 return false; 2402 } 2403 } 2404 } 2405 2406 const MachineRegisterInfo &MRI = MF.getRegInfo(); 2407 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) 2408 return false; 2409 } 2410 2411 return true; 2412 } 2413 2414 bool 2415 ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 2416 MachineFunction &MF, bool isVarArg, 2417 const SmallVectorImpl<ISD::OutputArg> &Outs, 2418 LLVMContext &Context) const { 2419 SmallVector<CCValAssign, 16> RVLocs; 2420 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 2421 return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg)); 2422 } 2423 2424 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps, 2425 const SDLoc &DL, SelectionDAG &DAG) { 2426 const MachineFunction &MF = DAG.getMachineFunction(); 2427 const Function &F = MF.getFunction(); 2428 2429 StringRef IntKind = F.getFnAttribute("interrupt").getValueAsString(); 2430 2431 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset 2432 // version of the "preferred return address". These offsets affect the return 2433 // instruction if this is a return from PL1 without hypervisor extensions. 2434 // IRQ/FIQ: +4 "subs pc, lr, #4" 2435 // SWI: 0 "subs pc, lr, #0" 2436 // ABORT: +4 "subs pc, lr, #4" 2437 // UNDEF: +4/+2 "subs pc, lr, #0" 2438 // UNDEF varies depending on where the exception came from ARM or Thumb 2439 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0. 2440 2441 int64_t LROffset; 2442 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" || 2443 IntKind == "ABORT") 2444 LROffset = 4; 2445 else if (IntKind == "SWI" || IntKind == "UNDEF") 2446 LROffset = 0; 2447 else 2448 report_fatal_error("Unsupported interrupt attribute. If present, value " 2449 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF"); 2450 2451 RetOps.insert(RetOps.begin() + 1, 2452 DAG.getConstant(LROffset, DL, MVT::i32, false)); 2453 2454 return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps); 2455 } 2456 2457 SDValue 2458 ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 2459 bool isVarArg, 2460 const SmallVectorImpl<ISD::OutputArg> &Outs, 2461 const SmallVectorImpl<SDValue> &OutVals, 2462 const SDLoc &dl, SelectionDAG &DAG) const { 2463 // CCValAssign - represent the assignment of the return value to a location. 2464 SmallVector<CCValAssign, 16> RVLocs; 2465 2466 // CCState - Info about the registers and stack slots. 2467 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 2468 *DAG.getContext()); 2469 2470 // Analyze outgoing return values. 2471 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg)); 2472 2473 SDValue Flag; 2474 SmallVector<SDValue, 4> RetOps; 2475 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 2476 bool isLittleEndian = Subtarget->isLittle(); 2477 2478 MachineFunction &MF = DAG.getMachineFunction(); 2479 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2480 AFI->setReturnRegsCount(RVLocs.size()); 2481 2482 // Copy the result values into the output registers. 2483 for (unsigned i = 0, realRVLocIdx = 0; 2484 i != RVLocs.size(); 2485 ++i, ++realRVLocIdx) { 2486 CCValAssign &VA = RVLocs[i]; 2487 assert(VA.isRegLoc() && "Can only return in registers!"); 2488 2489 SDValue Arg = OutVals[realRVLocIdx]; 2490 bool ReturnF16 = false; 2491 2492 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) { 2493 // Half-precision return values can be returned like this: 2494 // 2495 // t11 f16 = fadd ... 2496 // t12: i16 = bitcast t11 2497 // t13: i32 = zero_extend t12 2498 // t14: f32 = bitcast t13 <~~~~~~~ Arg 2499 // 2500 // to avoid code generation for bitcasts, we simply set Arg to the node 2501 // that produces the f16 value, t11 in this case. 2502 // 2503 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) { 2504 SDValue ZE = Arg.getOperand(0); 2505 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) { 2506 SDValue BC = ZE.getOperand(0); 2507 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) { 2508 Arg = BC.getOperand(0); 2509 ReturnF16 = true; 2510 } 2511 } 2512 } 2513 } 2514 2515 switch (VA.getLocInfo()) { 2516 default: llvm_unreachable("Unknown loc info!"); 2517 case CCValAssign::Full: break; 2518 case CCValAssign::BCvt: 2519 if (!ReturnF16) 2520 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); 2521 break; 2522 } 2523 2524 if (VA.needsCustom()) { 2525 if (VA.getLocVT() == MVT::v2f64) { 2526 // Extract the first half and return it in two registers. 2527 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 2528 DAG.getConstant(0, dl, MVT::i32)); 2529 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl, 2530 DAG.getVTList(MVT::i32, MVT::i32), Half); 2531 2532 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2533 HalfGPRs.getValue(isLittleEndian ? 0 : 1), 2534 Flag); 2535 Flag = Chain.getValue(1); 2536 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2537 VA = RVLocs[++i]; // skip ahead to next loc 2538 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2539 HalfGPRs.getValue(isLittleEndian ? 1 : 0), 2540 Flag); 2541 Flag = Chain.getValue(1); 2542 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2543 VA = RVLocs[++i]; // skip ahead to next loc 2544 2545 // Extract the 2nd half and fall through to handle it as an f64 value. 2546 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 2547 DAG.getConstant(1, dl, MVT::i32)); 2548 } 2549 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is 2550 // available. 2551 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl, 2552 DAG.getVTList(MVT::i32, MVT::i32), Arg); 2553 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2554 fmrrd.getValue(isLittleEndian ? 0 : 1), 2555 Flag); 2556 Flag = Chain.getValue(1); 2557 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2558 VA = RVLocs[++i]; // skip ahead to next loc 2559 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2560 fmrrd.getValue(isLittleEndian ? 1 : 0), 2561 Flag); 2562 } else 2563 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 2564 2565 // Guarantee that all emitted copies are 2566 // stuck together, avoiding something bad. 2567 Flag = Chain.getValue(1); 2568 RetOps.push_back(DAG.getRegister(VA.getLocReg(), 2569 ReturnF16 ? MVT::f16 : VA.getLocVT())); 2570 } 2571 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); 2572 const MCPhysReg *I = 2573 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 2574 if (I) { 2575 for (; *I; ++I) { 2576 if (ARM::GPRRegClass.contains(*I)) 2577 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); 2578 else if (ARM::DPRRegClass.contains(*I)) 2579 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); 2580 else 2581 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 2582 } 2583 } 2584 2585 // Update chain and glue. 2586 RetOps[0] = Chain; 2587 if (Flag.getNode()) 2588 RetOps.push_back(Flag); 2589 2590 // CPUs which aren't M-class use a special sequence to return from 2591 // exceptions (roughly, any instruction setting pc and cpsr simultaneously, 2592 // though we use "subs pc, lr, #N"). 2593 // 2594 // M-class CPUs actually use a normal return sequence with a special 2595 // (hardware-provided) value in LR, so the normal code path works. 2596 if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") && 2597 !Subtarget->isMClass()) { 2598 if (Subtarget->isThumb1Only()) 2599 report_fatal_error("interrupt attribute is not supported in Thumb1"); 2600 return LowerInterruptReturn(RetOps, dl, DAG); 2601 } 2602 2603 return DAG.getNode(ARMISD::RET_FLAG, dl, MVT::Other, RetOps); 2604 } 2605 2606 bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { 2607 if (N->getNumValues() != 1) 2608 return false; 2609 if (!N->hasNUsesOfValue(1, 0)) 2610 return false; 2611 2612 SDValue TCChain = Chain; 2613 SDNode *Copy = *N->use_begin(); 2614 if (Copy->getOpcode() == ISD::CopyToReg) { 2615 // If the copy has a glue operand, we conservatively assume it isn't safe to 2616 // perform a tail call. 2617 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) 2618 return false; 2619 TCChain = Copy->getOperand(0); 2620 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) { 2621 SDNode *VMov = Copy; 2622 // f64 returned in a pair of GPRs. 2623 SmallPtrSet<SDNode*, 2> Copies; 2624 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end(); 2625 UI != UE; ++UI) { 2626 if (UI->getOpcode() != ISD::CopyToReg) 2627 return false; 2628 Copies.insert(*UI); 2629 } 2630 if (Copies.size() > 2) 2631 return false; 2632 2633 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end(); 2634 UI != UE; ++UI) { 2635 SDValue UseChain = UI->getOperand(0); 2636 if (Copies.count(UseChain.getNode())) 2637 // Second CopyToReg 2638 Copy = *UI; 2639 else { 2640 // We are at the top of this chain. 2641 // If the copy has a glue operand, we conservatively assume it 2642 // isn't safe to perform a tail call. 2643 if (UI->getOperand(UI->getNumOperands()-1).getValueType() == MVT::Glue) 2644 return false; 2645 // First CopyToReg 2646 TCChain = UseChain; 2647 } 2648 } 2649 } else if (Copy->getOpcode() == ISD::BITCAST) { 2650 // f32 returned in a single GPR. 2651 if (!Copy->hasOneUse()) 2652 return false; 2653 Copy = *Copy->use_begin(); 2654 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0)) 2655 return false; 2656 // If the copy has a glue operand, we conservatively assume it isn't safe to 2657 // perform a tail call. 2658 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) 2659 return false; 2660 TCChain = Copy->getOperand(0); 2661 } else { 2662 return false; 2663 } 2664 2665 bool HasRet = false; 2666 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end(); 2667 UI != UE; ++UI) { 2668 if (UI->getOpcode() != ARMISD::RET_FLAG && 2669 UI->getOpcode() != ARMISD::INTRET_FLAG) 2670 return false; 2671 HasRet = true; 2672 } 2673 2674 if (!HasRet) 2675 return false; 2676 2677 Chain = TCChain; 2678 return true; 2679 } 2680 2681 bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 2682 if (!Subtarget->supportsTailCall()) 2683 return false; 2684 2685 auto Attr = 2686 CI->getParent()->getParent()->getFnAttribute("disable-tail-calls"); 2687 if (!CI->isTailCall() || Attr.getValueAsString() == "true") 2688 return false; 2689 2690 return true; 2691 } 2692 2693 // Trying to write a 64 bit value so need to split into two 32 bit values first, 2694 // and pass the lower and high parts through. 2695 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) { 2696 SDLoc DL(Op); 2697 SDValue WriteValue = Op->getOperand(2); 2698 2699 // This function is only supposed to be called for i64 type argument. 2700 assert(WriteValue.getValueType() == MVT::i64 2701 && "LowerWRITE_REGISTER called for non-i64 type argument."); 2702 2703 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue, 2704 DAG.getConstant(0, DL, MVT::i32)); 2705 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue, 2706 DAG.getConstant(1, DL, MVT::i32)); 2707 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi }; 2708 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops); 2709 } 2710 2711 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 2712 // their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is 2713 // one of the above mentioned nodes. It has to be wrapped because otherwise 2714 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 2715 // be used to form addressing mode. These wrapped nodes will be selected 2716 // into MOVi. 2717 SDValue ARMTargetLowering::LowerConstantPool(SDValue Op, 2718 SelectionDAG &DAG) const { 2719 EVT PtrVT = Op.getValueType(); 2720 // FIXME there is no actual debug info here 2721 SDLoc dl(Op); 2722 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 2723 SDValue Res; 2724 2725 // When generating execute-only code Constant Pools must be promoted to the 2726 // global data section. It's a bit ugly that we can't share them across basic 2727 // blocks, but this way we guarantee that execute-only behaves correct with 2728 // position-independent addressing modes. 2729 if (Subtarget->genExecuteOnly()) { 2730 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>(); 2731 auto T = const_cast<Type*>(CP->getType()); 2732 auto C = const_cast<Constant*>(CP->getConstVal()); 2733 auto M = const_cast<Module*>(DAG.getMachineFunction(). 2734 getFunction().getParent()); 2735 auto GV = new GlobalVariable( 2736 *M, T, /*isConst=*/true, GlobalVariable::InternalLinkage, C, 2737 Twine(DAG.getDataLayout().getPrivateGlobalPrefix()) + "CP" + 2738 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" + 2739 Twine(AFI->createPICLabelUId()) 2740 ); 2741 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV), 2742 dl, PtrVT); 2743 return LowerGlobalAddress(GA, DAG); 2744 } 2745 2746 if (CP->isMachineConstantPoolEntry()) 2747 Res = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 2748 CP->getAlignment()); 2749 else 2750 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 2751 CP->getAlignment()); 2752 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res); 2753 } 2754 2755 unsigned ARMTargetLowering::getJumpTableEncoding() const { 2756 return MachineJumpTableInfo::EK_Inline; 2757 } 2758 2759 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op, 2760 SelectionDAG &DAG) const { 2761 MachineFunction &MF = DAG.getMachineFunction(); 2762 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2763 unsigned ARMPCLabelIndex = 0; 2764 SDLoc DL(Op); 2765 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2766 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); 2767 SDValue CPAddr; 2768 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI(); 2769 if (!IsPositionIndependent) { 2770 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, 4); 2771 } else { 2772 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8; 2773 ARMPCLabelIndex = AFI->createPICLabelUId(); 2774 ARMConstantPoolValue *CPV = 2775 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex, 2776 ARMCP::CPBlockAddress, PCAdj); 2777 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2778 } 2779 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr); 2780 SDValue Result = DAG.getLoad( 2781 PtrVT, DL, DAG.getEntryNode(), CPAddr, 2782 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2783 if (!IsPositionIndependent) 2784 return Result; 2785 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32); 2786 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel); 2787 } 2788 2789 /// Convert a TLS address reference into the correct sequence of loads 2790 /// and calls to compute the variable's address for Darwin, and return an 2791 /// SDValue containing the final node. 2792 2793 /// Darwin only has one TLS scheme which must be capable of dealing with the 2794 /// fully general situation, in the worst case. This means: 2795 /// + "extern __thread" declaration. 2796 /// + Defined in a possibly unknown dynamic library. 2797 /// 2798 /// The general system is that each __thread variable has a [3 x i32] descriptor 2799 /// which contains information used by the runtime to calculate the address. The 2800 /// only part of this the compiler needs to know about is the first word, which 2801 /// contains a function pointer that must be called with the address of the 2802 /// entire descriptor in "r0". 2803 /// 2804 /// Since this descriptor may be in a different unit, in general access must 2805 /// proceed along the usual ARM rules. A common sequence to produce is: 2806 /// 2807 /// movw rT1, :lower16:_var$non_lazy_ptr 2808 /// movt rT1, :upper16:_var$non_lazy_ptr 2809 /// ldr r0, [rT1] 2810 /// ldr rT2, [r0] 2811 /// blx rT2 2812 /// [...address now in r0...] 2813 SDValue 2814 ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op, 2815 SelectionDAG &DAG) const { 2816 assert(Subtarget->isTargetDarwin() && 2817 "This function expects a Darwin target"); 2818 SDLoc DL(Op); 2819 2820 // First step is to get the address of the actua global symbol. This is where 2821 // the TLS descriptor lives. 2822 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG); 2823 2824 // The first entry in the descriptor is a function pointer that we must call 2825 // to obtain the address of the variable. 2826 SDValue Chain = DAG.getEntryNode(); 2827 SDValue FuncTLVGet = DAG.getLoad( 2828 MVT::i32, DL, Chain, DescAddr, 2829 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 2830 /* Alignment = */ 4, 2831 MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable | 2832 MachineMemOperand::MOInvariant); 2833 Chain = FuncTLVGet.getValue(1); 2834 2835 MachineFunction &F = DAG.getMachineFunction(); 2836 MachineFrameInfo &MFI = F.getFrameInfo(); 2837 MFI.setAdjustsStack(true); 2838 2839 // TLS calls preserve all registers except those that absolutely must be 2840 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be 2841 // silly). 2842 auto TRI = 2843 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo(); 2844 auto ARI = static_cast<const ARMRegisterInfo *>(TRI); 2845 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction()); 2846 2847 // Finally, we can make the call. This is just a degenerate version of a 2848 // normal AArch64 call node: r0 takes the address of the descriptor, and 2849 // returns the address of the variable in this thread. 2850 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue()); 2851 Chain = 2852 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue), 2853 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32), 2854 DAG.getRegisterMask(Mask), Chain.getValue(1)); 2855 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1)); 2856 } 2857 2858 SDValue 2859 ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op, 2860 SelectionDAG &DAG) const { 2861 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering"); 2862 2863 SDValue Chain = DAG.getEntryNode(); 2864 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2865 SDLoc DL(Op); 2866 2867 // Load the current TEB (thread environment block) 2868 SDValue Ops[] = {Chain, 2869 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32), 2870 DAG.getConstant(15, DL, MVT::i32), 2871 DAG.getConstant(0, DL, MVT::i32), 2872 DAG.getConstant(13, DL, MVT::i32), 2873 DAG.getConstant(0, DL, MVT::i32), 2874 DAG.getConstant(2, DL, MVT::i32)}; 2875 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, 2876 DAG.getVTList(MVT::i32, MVT::Other), Ops); 2877 2878 SDValue TEB = CurrentTEB.getValue(0); 2879 Chain = CurrentTEB.getValue(1); 2880 2881 // Load the ThreadLocalStoragePointer from the TEB 2882 // A pointer to the TLS array is located at offset 0x2c from the TEB. 2883 SDValue TLSArray = 2884 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL)); 2885 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo()); 2886 2887 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4 2888 // offset into the TLSArray. 2889 2890 // Load the TLS index from the C runtime 2891 SDValue TLSIndex = 2892 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG); 2893 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex); 2894 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo()); 2895 2896 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex, 2897 DAG.getConstant(2, DL, MVT::i32)); 2898 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain, 2899 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot), 2900 MachinePointerInfo()); 2901 2902 // Get the offset of the start of the .tls section (section base) 2903 const auto *GA = cast<GlobalAddressSDNode>(Op); 2904 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL); 2905 SDValue Offset = DAG.getLoad( 2906 PtrVT, DL, Chain, DAG.getNode(ARMISD::Wrapper, DL, MVT::i32, 2907 DAG.getTargetConstantPool(CPV, PtrVT, 4)), 2908 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2909 2910 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset); 2911 } 2912 2913 // Lower ISD::GlobalTLSAddress using the "general dynamic" model 2914 SDValue 2915 ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, 2916 SelectionDAG &DAG) const { 2917 SDLoc dl(GA); 2918 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2919 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; 2920 MachineFunction &MF = DAG.getMachineFunction(); 2921 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2922 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2923 ARMConstantPoolValue *CPV = 2924 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, 2925 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true); 2926 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2927 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument); 2928 Argument = DAG.getLoad( 2929 PtrVT, dl, DAG.getEntryNode(), Argument, 2930 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2931 SDValue Chain = Argument.getValue(1); 2932 2933 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 2934 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel); 2935 2936 // call __tls_get_addr. 2937 ArgListTy Args; 2938 ArgListEntry Entry; 2939 Entry.Node = Argument; 2940 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext()); 2941 Args.push_back(Entry); 2942 2943 // FIXME: is there useful debug info available here? 2944 TargetLowering::CallLoweringInfo CLI(DAG); 2945 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( 2946 CallingConv::C, Type::getInt32Ty(*DAG.getContext()), 2947 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args)); 2948 2949 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 2950 return CallResult.first; 2951 } 2952 2953 // Lower ISD::GlobalTLSAddress using the "initial exec" or 2954 // "local exec" model. 2955 SDValue 2956 ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA, 2957 SelectionDAG &DAG, 2958 TLSModel::Model model) const { 2959 const GlobalValue *GV = GA->getGlobal(); 2960 SDLoc dl(GA); 2961 SDValue Offset; 2962 SDValue Chain = DAG.getEntryNode(); 2963 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2964 // Get the Thread Pointer 2965 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT); 2966 2967 if (model == TLSModel::InitialExec) { 2968 MachineFunction &MF = DAG.getMachineFunction(); 2969 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2970 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2971 // Initial exec model. 2972 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; 2973 ARMConstantPoolValue *CPV = 2974 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, 2975 ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF, 2976 true); 2977 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2978 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset); 2979 Offset = DAG.getLoad( 2980 PtrVT, dl, Chain, Offset, 2981 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2982 Chain = Offset.getValue(1); 2983 2984 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 2985 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel); 2986 2987 Offset = DAG.getLoad( 2988 PtrVT, dl, Chain, Offset, 2989 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2990 } else { 2991 // local exec model 2992 assert(model == TLSModel::LocalExec); 2993 ARMConstantPoolValue *CPV = 2994 ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF); 2995 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2996 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset); 2997 Offset = DAG.getLoad( 2998 PtrVT, dl, Chain, Offset, 2999 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3000 } 3001 3002 // The address of the thread local variable is the add of the thread 3003 // pointer with the offset of the variable. 3004 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); 3005 } 3006 3007 SDValue 3008 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { 3009 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 3010 if (DAG.getTarget().useEmulatedTLS()) 3011 return LowerToTLSEmulatedModel(GA, DAG); 3012 3013 if (Subtarget->isTargetDarwin()) 3014 return LowerGlobalTLSAddressDarwin(Op, DAG); 3015 3016 if (Subtarget->isTargetWindows()) 3017 return LowerGlobalTLSAddressWindows(Op, DAG); 3018 3019 // TODO: implement the "local dynamic" model 3020 assert(Subtarget->isTargetELF() && "Only ELF implemented here"); 3021 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal()); 3022 3023 switch (model) { 3024 case TLSModel::GeneralDynamic: 3025 case TLSModel::LocalDynamic: 3026 return LowerToTLSGeneralDynamicModel(GA, DAG); 3027 case TLSModel::InitialExec: 3028 case TLSModel::LocalExec: 3029 return LowerToTLSExecModels(GA, DAG, model); 3030 } 3031 llvm_unreachable("bogus TLS model"); 3032 } 3033 3034 /// Return true if all users of V are within function F, looking through 3035 /// ConstantExprs. 3036 static bool allUsersAreInFunction(const Value *V, const Function *F) { 3037 SmallVector<const User*,4> Worklist; 3038 for (auto *U : V->users()) 3039 Worklist.push_back(U); 3040 while (!Worklist.empty()) { 3041 auto *U = Worklist.pop_back_val(); 3042 if (isa<ConstantExpr>(U)) { 3043 for (auto *UU : U->users()) 3044 Worklist.push_back(UU); 3045 continue; 3046 } 3047 3048 auto *I = dyn_cast<Instruction>(U); 3049 if (!I || I->getParent()->getParent() != F) 3050 return false; 3051 } 3052 return true; 3053 } 3054 3055 /// Return true if all users of V are within some (any) function, looking through 3056 /// ConstantExprs. In other words, are there any global constant users? 3057 static bool allUsersAreInFunctions(const Value *V) { 3058 SmallVector<const User*,4> Worklist; 3059 for (auto *U : V->users()) 3060 Worklist.push_back(U); 3061 while (!Worklist.empty()) { 3062 auto *U = Worklist.pop_back_val(); 3063 if (isa<ConstantExpr>(U)) { 3064 for (auto *UU : U->users()) 3065 Worklist.push_back(UU); 3066 continue; 3067 } 3068 3069 if (!isa<Instruction>(U)) 3070 return false; 3071 } 3072 return true; 3073 } 3074 3075 // Return true if T is an integer, float or an array/vector of either. 3076 static bool isSimpleType(Type *T) { 3077 if (T->isIntegerTy() || T->isFloatingPointTy()) 3078 return true; 3079 Type *SubT = nullptr; 3080 if (T->isArrayTy()) 3081 SubT = T->getArrayElementType(); 3082 else if (T->isVectorTy()) 3083 SubT = T->getVectorElementType(); 3084 else 3085 return false; 3086 return SubT->isIntegerTy() || SubT->isFloatingPointTy(); 3087 } 3088 3089 static SDValue promoteToConstantPool(const GlobalValue *GV, SelectionDAG &DAG, 3090 EVT PtrVT, const SDLoc &dl) { 3091 // If we're creating a pool entry for a constant global with unnamed address, 3092 // and the global is small enough, we can emit it inline into the constant pool 3093 // to save ourselves an indirection. 3094 // 3095 // This is a win if the constant is only used in one function (so it doesn't 3096 // need to be duplicated) or duplicating the constant wouldn't increase code 3097 // size (implying the constant is no larger than 4 bytes). 3098 const Function &F = DAG.getMachineFunction().getFunction(); 3099 3100 // We rely on this decision to inline being idemopotent and unrelated to the 3101 // use-site. We know that if we inline a variable at one use site, we'll 3102 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel 3103 // doesn't know about this optimization, so bail out if it's enabled else 3104 // we could decide to inline here (and thus never emit the GV) but require 3105 // the GV from fast-isel generated code. 3106 if (!EnableConstpoolPromotion || 3107 DAG.getMachineFunction().getTarget().Options.EnableFastISel) 3108 return SDValue(); 3109 3110 auto *GVar = dyn_cast<GlobalVariable>(GV); 3111 if (!GVar || !GVar->hasInitializer() || 3112 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() || 3113 !GVar->hasLocalLinkage()) 3114 return SDValue(); 3115 3116 // Ensure that we don't try and inline any type that contains pointers. If 3117 // we inline a value that contains relocations, we move the relocations from 3118 // .data to .text which is not ideal. 3119 auto *Init = GVar->getInitializer(); 3120 if (!isSimpleType(Init->getType())) 3121 return SDValue(); 3122 3123 // The constant islands pass can only really deal with alignment requests 3124 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote 3125 // any type wanting greater alignment requirements than 4 bytes. We also 3126 // can only promote constants that are multiples of 4 bytes in size or 3127 // are paddable to a multiple of 4. Currently we only try and pad constants 3128 // that are strings for simplicity. 3129 auto *CDAInit = dyn_cast<ConstantDataArray>(Init); 3130 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType()); 3131 unsigned Align = GVar->getAlignment(); 3132 unsigned RequiredPadding = 4 - (Size % 4); 3133 bool PaddingPossible = 3134 RequiredPadding == 4 || (CDAInit && CDAInit->isString()); 3135 if (!PaddingPossible || Align > 4 || Size > ConstpoolPromotionMaxSize || 3136 Size == 0) 3137 return SDValue(); 3138 3139 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding); 3140 MachineFunction &MF = DAG.getMachineFunction(); 3141 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3142 3143 // We can't bloat the constant pool too much, else the ConstantIslands pass 3144 // may fail to converge. If we haven't promoted this global yet (it may have 3145 // multiple uses), and promoting it would increase the constant pool size (Sz 3146 // > 4), ensure we have space to do so up to MaxTotal. 3147 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4) 3148 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >= 3149 ConstpoolPromotionMaxTotal) 3150 return SDValue(); 3151 3152 // This is only valid if all users are in a single function OR it has users 3153 // in multiple functions but it no larger than a pointer. We also check if 3154 // GVar has constant (non-ConstantExpr) users. If so, it essentially has its 3155 // address taken. 3156 if (!allUsersAreInFunction(GVar, &F) && 3157 !(Size <= 4 && allUsersAreInFunctions(GVar))) 3158 return SDValue(); 3159 3160 // We're going to inline this global. Pad it out if needed. 3161 if (RequiredPadding != 4) { 3162 StringRef S = CDAInit->getAsString(); 3163 3164 SmallVector<uint8_t,16> V(S.size()); 3165 std::copy(S.bytes_begin(), S.bytes_end(), V.begin()); 3166 while (RequiredPadding--) 3167 V.push_back(0); 3168 Init = ConstantDataArray::get(*DAG.getContext(), V); 3169 } 3170 3171 auto CPVal = ARMConstantPoolConstant::Create(GVar, Init); 3172 SDValue CPAddr = 3173 DAG.getTargetConstantPool(CPVal, PtrVT, /*Align=*/4); 3174 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) { 3175 AFI->markGlobalAsPromotedToConstantPool(GVar); 3176 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() + 3177 PaddedSize - 4); 3178 } 3179 ++NumConstpoolPromoted; 3180 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3181 } 3182 3183 bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const { 3184 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 3185 GV = GA->getBaseObject(); 3186 return (isa<GlobalVariable>(GV) && cast<GlobalVariable>(GV)->isConstant()) || 3187 isa<Function>(GV); 3188 } 3189 3190 SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op, 3191 SelectionDAG &DAG) const { 3192 switch (Subtarget->getTargetTriple().getObjectFormat()) { 3193 default: llvm_unreachable("unknown object format"); 3194 case Triple::COFF: 3195 return LowerGlobalAddressWindows(Op, DAG); 3196 case Triple::ELF: 3197 return LowerGlobalAddressELF(Op, DAG); 3198 case Triple::MachO: 3199 return LowerGlobalAddressDarwin(Op, DAG); 3200 } 3201 } 3202 3203 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op, 3204 SelectionDAG &DAG) const { 3205 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3206 SDLoc dl(Op); 3207 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3208 const TargetMachine &TM = getTargetMachine(); 3209 bool IsRO = isReadOnly(GV); 3210 3211 // promoteToConstantPool only if not generating XO text section 3212 if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV) && !Subtarget->genExecuteOnly()) 3213 if (SDValue V = promoteToConstantPool(GV, DAG, PtrVT, dl)) 3214 return V; 3215 3216 if (isPositionIndependent()) { 3217 bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV); 3218 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 3219 UseGOT_PREL ? ARMII::MO_GOT : 0); 3220 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G); 3221 if (UseGOT_PREL) 3222 Result = 3223 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result, 3224 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 3225 return Result; 3226 } else if (Subtarget->isROPI() && IsRO) { 3227 // PC-relative. 3228 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT); 3229 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G); 3230 return Result; 3231 } else if (Subtarget->isRWPI() && !IsRO) { 3232 // SB-relative. 3233 SDValue RelAddr; 3234 if (Subtarget->useMovt(DAG.getMachineFunction())) { 3235 ++NumMovwMovt; 3236 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL); 3237 RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G); 3238 } else { // use literal pool for address constant 3239 ARMConstantPoolValue *CPV = 3240 ARMConstantPoolConstant::Create(GV, ARMCP::SBREL); 3241 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4); 3242 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3243 RelAddr = DAG.getLoad( 3244 PtrVT, dl, DAG.getEntryNode(), CPAddr, 3245 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3246 } 3247 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT); 3248 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr); 3249 return Result; 3250 } 3251 3252 // If we have T2 ops, we can materialize the address directly via movt/movw 3253 // pair. This is always cheaper. 3254 if (Subtarget->useMovt(DAG.getMachineFunction())) { 3255 ++NumMovwMovt; 3256 // FIXME: Once remat is capable of dealing with instructions with register 3257 // operands, expand this into two nodes. 3258 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT, 3259 DAG.getTargetGlobalAddress(GV, dl, PtrVT)); 3260 } else { 3261 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4); 3262 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3263 return DAG.getLoad( 3264 PtrVT, dl, DAG.getEntryNode(), CPAddr, 3265 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3266 } 3267 } 3268 3269 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op, 3270 SelectionDAG &DAG) const { 3271 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && 3272 "ROPI/RWPI not currently supported for Darwin"); 3273 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3274 SDLoc dl(Op); 3275 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3276 3277 if (Subtarget->useMovt(DAG.getMachineFunction())) 3278 ++NumMovwMovt; 3279 3280 // FIXME: Once remat is capable of dealing with instructions with register 3281 // operands, expand this into multiple nodes 3282 unsigned Wrapper = 3283 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper; 3284 3285 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY); 3286 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G); 3287 3288 if (Subtarget->isGVIndirectSymbol(GV)) 3289 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result, 3290 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 3291 return Result; 3292 } 3293 3294 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op, 3295 SelectionDAG &DAG) const { 3296 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported"); 3297 assert(Subtarget->useMovt(DAG.getMachineFunction()) && 3298 "Windows on ARM expects to use movw/movt"); 3299 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && 3300 "ROPI/RWPI not currently supported for Windows"); 3301 3302 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3303 const ARMII::TOF TargetFlags = 3304 (GV->hasDLLImportStorageClass() ? ARMII::MO_DLLIMPORT : ARMII::MO_NO_FLAG); 3305 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3306 SDValue Result; 3307 SDLoc DL(Op); 3308 3309 ++NumMovwMovt; 3310 3311 // FIXME: Once remat is capable of dealing with instructions with register 3312 // operands, expand this into two nodes. 3313 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, 3314 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*Offset=*/0, 3315 TargetFlags)); 3316 if (GV->hasDLLImportStorageClass()) 3317 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 3318 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 3319 return Result; 3320 } 3321 3322 SDValue 3323 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const { 3324 SDLoc dl(Op); 3325 SDValue Val = DAG.getConstant(0, dl, MVT::i32); 3326 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl, 3327 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0), 3328 Op.getOperand(1), Val); 3329 } 3330 3331 SDValue 3332 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const { 3333 SDLoc dl(Op); 3334 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0), 3335 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32)); 3336 } 3337 3338 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, 3339 SelectionDAG &DAG) const { 3340 SDLoc dl(Op); 3341 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other, 3342 Op.getOperand(0)); 3343 } 3344 3345 SDValue 3346 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG, 3347 const ARMSubtarget *Subtarget) const { 3348 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 3349 SDLoc dl(Op); 3350 switch (IntNo) { 3351 default: return SDValue(); // Don't custom lower most intrinsics. 3352 case Intrinsic::thread_pointer: { 3353 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3354 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT); 3355 } 3356 case Intrinsic::eh_sjlj_lsda: { 3357 MachineFunction &MF = DAG.getMachineFunction(); 3358 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3359 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 3360 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3361 SDValue CPAddr; 3362 bool IsPositionIndependent = isPositionIndependent(); 3363 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0; 3364 ARMConstantPoolValue *CPV = 3365 ARMConstantPoolConstant::Create(&MF.getFunction(), ARMPCLabelIndex, 3366 ARMCP::CPLSDA, PCAdj); 3367 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4); 3368 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3369 SDValue Result = DAG.getLoad( 3370 PtrVT, dl, DAG.getEntryNode(), CPAddr, 3371 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3372 3373 if (IsPositionIndependent) { 3374 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 3375 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel); 3376 } 3377 return Result; 3378 } 3379 case Intrinsic::arm_neon_vabs: 3380 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(), 3381 Op.getOperand(1)); 3382 case Intrinsic::arm_neon_vmulls: 3383 case Intrinsic::arm_neon_vmullu: { 3384 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls) 3385 ? ARMISD::VMULLs : ARMISD::VMULLu; 3386 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3387 Op.getOperand(1), Op.getOperand(2)); 3388 } 3389 case Intrinsic::arm_neon_vminnm: 3390 case Intrinsic::arm_neon_vmaxnm: { 3391 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm) 3392 ? ISD::FMINNUM : ISD::FMAXNUM; 3393 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3394 Op.getOperand(1), Op.getOperand(2)); 3395 } 3396 case Intrinsic::arm_neon_vminu: 3397 case Intrinsic::arm_neon_vmaxu: { 3398 if (Op.getValueType().isFloatingPoint()) 3399 return SDValue(); 3400 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu) 3401 ? ISD::UMIN : ISD::UMAX; 3402 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3403 Op.getOperand(1), Op.getOperand(2)); 3404 } 3405 case Intrinsic::arm_neon_vmins: 3406 case Intrinsic::arm_neon_vmaxs: { 3407 // v{min,max}s is overloaded between signed integers and floats. 3408 if (!Op.getValueType().isFloatingPoint()) { 3409 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins) 3410 ? ISD::SMIN : ISD::SMAX; 3411 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3412 Op.getOperand(1), Op.getOperand(2)); 3413 } 3414 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins) 3415 ? ISD::FMINNAN : ISD::FMAXNAN; 3416 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3417 Op.getOperand(1), Op.getOperand(2)); 3418 } 3419 case Intrinsic::arm_neon_vtbl1: 3420 return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(), 3421 Op.getOperand(1), Op.getOperand(2)); 3422 case Intrinsic::arm_neon_vtbl2: 3423 return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(), 3424 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 3425 } 3426 } 3427 3428 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, 3429 const ARMSubtarget *Subtarget) { 3430 SDLoc dl(Op); 3431 ConstantSDNode *SSIDNode = cast<ConstantSDNode>(Op.getOperand(2)); 3432 auto SSID = static_cast<SyncScope::ID>(SSIDNode->getZExtValue()); 3433 if (SSID == SyncScope::SingleThread) 3434 return Op; 3435 3436 if (!Subtarget->hasDataBarrier()) { 3437 // Some ARMv6 cpus can support data barriers with an mcr instruction. 3438 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get 3439 // here. 3440 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() && 3441 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!"); 3442 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0), 3443 DAG.getConstant(0, dl, MVT::i32)); 3444 } 3445 3446 ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1)); 3447 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue()); 3448 ARM_MB::MemBOpt Domain = ARM_MB::ISH; 3449 if (Subtarget->isMClass()) { 3450 // Only a full system barrier exists in the M-class architectures. 3451 Domain = ARM_MB::SY; 3452 } else if (Subtarget->preferISHSTBarriers() && 3453 Ord == AtomicOrdering::Release) { 3454 // Swift happens to implement ISHST barriers in a way that's compatible with 3455 // Release semantics but weaker than ISH so we'd be fools not to use 3456 // it. Beware: other processors probably don't! 3457 Domain = ARM_MB::ISHST; 3458 } 3459 3460 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0), 3461 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32), 3462 DAG.getConstant(Domain, dl, MVT::i32)); 3463 } 3464 3465 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG, 3466 const ARMSubtarget *Subtarget) { 3467 // ARM pre v5TE and Thumb1 does not have preload instructions. 3468 if (!(Subtarget->isThumb2() || 3469 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps()))) 3470 // Just preserve the chain. 3471 return Op.getOperand(0); 3472 3473 SDLoc dl(Op); 3474 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1; 3475 if (!isRead && 3476 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension())) 3477 // ARMv7 with MP extension has PLDW. 3478 return Op.getOperand(0); 3479 3480 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 3481 if (Subtarget->isThumb()) { 3482 // Invert the bits. 3483 isRead = ~isRead & 1; 3484 isData = ~isData & 1; 3485 } 3486 3487 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0), 3488 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32), 3489 DAG.getConstant(isData, dl, MVT::i32)); 3490 } 3491 3492 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) { 3493 MachineFunction &MF = DAG.getMachineFunction(); 3494 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>(); 3495 3496 // vastart just stores the address of the VarArgsFrameIndex slot into the 3497 // memory location argument. 3498 SDLoc dl(Op); 3499 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 3500 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3501 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3502 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 3503 MachinePointerInfo(SV)); 3504 } 3505 3506 SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA, 3507 CCValAssign &NextVA, 3508 SDValue &Root, 3509 SelectionDAG &DAG, 3510 const SDLoc &dl) const { 3511 MachineFunction &MF = DAG.getMachineFunction(); 3512 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3513 3514 const TargetRegisterClass *RC; 3515 if (AFI->isThumb1OnlyFunction()) 3516 RC = &ARM::tGPRRegClass; 3517 else 3518 RC = &ARM::GPRRegClass; 3519 3520 // Transform the arguments stored in physical registers into virtual ones. 3521 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 3522 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32); 3523 3524 SDValue ArgValue2; 3525 if (NextVA.isMemLoc()) { 3526 MachineFrameInfo &MFI = MF.getFrameInfo(); 3527 int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true); 3528 3529 // Create load node to retrieve arguments from the stack. 3530 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 3531 ArgValue2 = DAG.getLoad( 3532 MVT::i32, dl, Root, FIN, 3533 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)); 3534 } else { 3535 Reg = MF.addLiveIn(NextVA.getLocReg(), RC); 3536 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32); 3537 } 3538 if (!Subtarget->isLittle()) 3539 std::swap (ArgValue, ArgValue2); 3540 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2); 3541 } 3542 3543 // The remaining GPRs hold either the beginning of variable-argument 3544 // data, or the beginning of an aggregate passed by value (usually 3545 // byval). Either way, we allocate stack slots adjacent to the data 3546 // provided by our caller, and store the unallocated registers there. 3547 // If this is a variadic function, the va_list pointer will begin with 3548 // these values; otherwise, this reassembles a (byval) structure that 3549 // was split between registers and memory. 3550 // Return: The frame index registers were stored into. 3551 int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG, 3552 const SDLoc &dl, SDValue &Chain, 3553 const Value *OrigArg, 3554 unsigned InRegsParamRecordIdx, 3555 int ArgOffset, unsigned ArgSize) const { 3556 // Currently, two use-cases possible: 3557 // Case #1. Non-var-args function, and we meet first byval parameter. 3558 // Setup first unallocated register as first byval register; 3559 // eat all remained registers 3560 // (these two actions are performed by HandleByVal method). 3561 // Then, here, we initialize stack frame with 3562 // "store-reg" instructions. 3563 // Case #2. Var-args function, that doesn't contain byval parameters. 3564 // The same: eat all remained unallocated registers, 3565 // initialize stack frame. 3566 3567 MachineFunction &MF = DAG.getMachineFunction(); 3568 MachineFrameInfo &MFI = MF.getFrameInfo(); 3569 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3570 unsigned RBegin, REnd; 3571 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) { 3572 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd); 3573 } else { 3574 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs); 3575 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx]; 3576 REnd = ARM::R4; 3577 } 3578 3579 if (REnd != RBegin) 3580 ArgOffset = -4 * (ARM::R4 - RBegin); 3581 3582 auto PtrVT = getPointerTy(DAG.getDataLayout()); 3583 int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false); 3584 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT); 3585 3586 SmallVector<SDValue, 4> MemOps; 3587 const TargetRegisterClass *RC = 3588 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass; 3589 3590 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) { 3591 unsigned VReg = MF.addLiveIn(Reg, RC); 3592 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); 3593 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3594 MachinePointerInfo(OrigArg, 4 * i)); 3595 MemOps.push_back(Store); 3596 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT)); 3597 } 3598 3599 if (!MemOps.empty()) 3600 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3601 return FrameIndex; 3602 } 3603 3604 // Setup stack frame, the va_list pointer will start from. 3605 void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG, 3606 const SDLoc &dl, SDValue &Chain, 3607 unsigned ArgOffset, 3608 unsigned TotalArgRegsSaveSize, 3609 bool ForceMutable) const { 3610 MachineFunction &MF = DAG.getMachineFunction(); 3611 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3612 3613 // Try to store any remaining integer argument regs 3614 // to their spots on the stack so that they may be loaded by dereferencing 3615 // the result of va_next. 3616 // If there is no regs to be stored, just point address after last 3617 // argument passed via stack. 3618 int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, nullptr, 3619 CCInfo.getInRegsParamsCount(), 3620 CCInfo.getNextStackOffset(), 4); 3621 AFI->setVarArgsFrameIndex(FrameIndex); 3622 } 3623 3624 SDValue ARMTargetLowering::LowerFormalArguments( 3625 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3626 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3627 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3628 MachineFunction &MF = DAG.getMachineFunction(); 3629 MachineFrameInfo &MFI = MF.getFrameInfo(); 3630 3631 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3632 3633 // Assign locations to all of the incoming arguments. 3634 SmallVector<CCValAssign, 16> ArgLocs; 3635 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 3636 *DAG.getContext()); 3637 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg)); 3638 3639 SmallVector<SDValue, 16> ArgValues; 3640 SDValue ArgValue; 3641 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin(); 3642 unsigned CurArgIdx = 0; 3643 3644 // Initially ArgRegsSaveSize is zero. 3645 // Then we increase this value each time we meet byval parameter. 3646 // We also increase this value in case of varargs function. 3647 AFI->setArgRegsSaveSize(0); 3648 3649 // Calculate the amount of stack space that we need to allocate to store 3650 // byval and variadic arguments that are passed in registers. 3651 // We need to know this before we allocate the first byval or variadic 3652 // argument, as they will be allocated a stack slot below the CFA (Canonical 3653 // Frame Address, the stack pointer at entry to the function). 3654 unsigned ArgRegBegin = ARM::R4; 3655 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3656 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount()) 3657 break; 3658 3659 CCValAssign &VA = ArgLocs[i]; 3660 unsigned Index = VA.getValNo(); 3661 ISD::ArgFlagsTy Flags = Ins[Index].Flags; 3662 if (!Flags.isByVal()) 3663 continue; 3664 3665 assert(VA.isMemLoc() && "unexpected byval pointer in reg"); 3666 unsigned RBegin, REnd; 3667 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd); 3668 ArgRegBegin = std::min(ArgRegBegin, RBegin); 3669 3670 CCInfo.nextInRegsParam(); 3671 } 3672 CCInfo.rewindByValRegsInfo(); 3673 3674 int lastInsIndex = -1; 3675 if (isVarArg && MFI.hasVAStart()) { 3676 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs); 3677 if (RegIdx != array_lengthof(GPRArgRegs)) 3678 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]); 3679 } 3680 3681 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin); 3682 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize); 3683 auto PtrVT = getPointerTy(DAG.getDataLayout()); 3684 3685 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3686 CCValAssign &VA = ArgLocs[i]; 3687 if (Ins[VA.getValNo()].isOrigArg()) { 3688 std::advance(CurOrigArg, 3689 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx); 3690 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex(); 3691 } 3692 // Arguments stored in registers. 3693 if (VA.isRegLoc()) { 3694 EVT RegVT = VA.getLocVT(); 3695 3696 if (VA.needsCustom()) { 3697 // f64 and vector types are split up into multiple registers or 3698 // combinations of registers and stack slots. 3699 if (VA.getLocVT() == MVT::v2f64) { 3700 SDValue ArgValue1 = GetF64FormalArgument(VA, ArgLocs[++i], 3701 Chain, DAG, dl); 3702 VA = ArgLocs[++i]; // skip ahead to next loc 3703 SDValue ArgValue2; 3704 if (VA.isMemLoc()) { 3705 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true); 3706 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3707 ArgValue2 = DAG.getLoad(MVT::f64, dl, Chain, FIN, 3708 MachinePointerInfo::getFixedStack( 3709 DAG.getMachineFunction(), FI)); 3710 } else { 3711 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], 3712 Chain, DAG, dl); 3713 } 3714 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64); 3715 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, 3716 ArgValue, ArgValue1, 3717 DAG.getIntPtrConstant(0, dl)); 3718 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, 3719 ArgValue, ArgValue2, 3720 DAG.getIntPtrConstant(1, dl)); 3721 } else 3722 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl); 3723 } else { 3724 const TargetRegisterClass *RC; 3725 3726 3727 if (RegVT == MVT::f16) 3728 RC = &ARM::HPRRegClass; 3729 else if (RegVT == MVT::f32) 3730 RC = &ARM::SPRRegClass; 3731 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16) 3732 RC = &ARM::DPRRegClass; 3733 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16) 3734 RC = &ARM::QPRRegClass; 3735 else if (RegVT == MVT::i32) 3736 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass 3737 : &ARM::GPRRegClass; 3738 else 3739 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering"); 3740 3741 // Transform the arguments in physical registers into virtual ones. 3742 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 3743 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); 3744 } 3745 3746 // If this is an 8 or 16-bit value, it is really passed promoted 3747 // to 32 bits. Insert an assert[sz]ext to capture this, then 3748 // truncate to the right size. 3749 switch (VA.getLocInfo()) { 3750 default: llvm_unreachable("Unknown loc info!"); 3751 case CCValAssign::Full: break; 3752 case CCValAssign::BCvt: 3753 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); 3754 break; 3755 case CCValAssign::SExt: 3756 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, 3757 DAG.getValueType(VA.getValVT())); 3758 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 3759 break; 3760 case CCValAssign::ZExt: 3761 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, 3762 DAG.getValueType(VA.getValVT())); 3763 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 3764 break; 3765 } 3766 3767 InVals.push_back(ArgValue); 3768 } else { // VA.isRegLoc() 3769 // sanity check 3770 assert(VA.isMemLoc()); 3771 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered"); 3772 3773 int index = VA.getValNo(); 3774 3775 // Some Ins[] entries become multiple ArgLoc[] entries. 3776 // Process them only once. 3777 if (index != lastInsIndex) 3778 { 3779 ISD::ArgFlagsTy Flags = Ins[index].Flags; 3780 // FIXME: For now, all byval parameter objects are marked mutable. 3781 // This can be changed with more analysis. 3782 // In case of tail call optimization mark all arguments mutable. 3783 // Since they could be overwritten by lowering of arguments in case of 3784 // a tail call. 3785 if (Flags.isByVal()) { 3786 assert(Ins[index].isOrigArg() && 3787 "Byval arguments cannot be implicit"); 3788 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed(); 3789 3790 int FrameIndex = StoreByValRegs( 3791 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex, 3792 VA.getLocMemOffset(), Flags.getByValSize()); 3793 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT)); 3794 CCInfo.nextInRegsParam(); 3795 } else { 3796 unsigned FIOffset = VA.getLocMemOffset(); 3797 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8, 3798 FIOffset, true); 3799 3800 // Create load nodes to retrieve arguments from the stack. 3801 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3802 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN, 3803 MachinePointerInfo::getFixedStack( 3804 DAG.getMachineFunction(), FI))); 3805 } 3806 lastInsIndex = index; 3807 } 3808 } 3809 } 3810 3811 // varargs 3812 if (isVarArg && MFI.hasVAStart()) 3813 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, 3814 CCInfo.getNextStackOffset(), 3815 TotalArgRegsSaveSize); 3816 3817 AFI->setArgumentStackSize(CCInfo.getNextStackOffset()); 3818 3819 return Chain; 3820 } 3821 3822 /// isFloatingPointZero - Return true if this is +0.0. 3823 static bool isFloatingPointZero(SDValue Op) { 3824 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 3825 return CFP->getValueAPF().isPosZero(); 3826 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 3827 // Maybe this has already been legalized into the constant pool? 3828 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) { 3829 SDValue WrapperOp = Op.getOperand(1).getOperand(0); 3830 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp)) 3831 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 3832 return CFP->getValueAPF().isPosZero(); 3833 } 3834 } else if (Op->getOpcode() == ISD::BITCAST && 3835 Op->getValueType(0) == MVT::f64) { 3836 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64) 3837 // created by LowerConstantFP(). 3838 SDValue BitcastOp = Op->getOperand(0); 3839 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM && 3840 isNullConstant(BitcastOp->getOperand(0))) 3841 return true; 3842 } 3843 return false; 3844 } 3845 3846 /// Returns appropriate ARM CMP (cmp) and corresponding condition code for 3847 /// the given operands. 3848 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, 3849 SDValue &ARMcc, SelectionDAG &DAG, 3850 const SDLoc &dl) const { 3851 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) { 3852 unsigned C = RHSC->getZExtValue(); 3853 if (!isLegalICmpImmediate((int32_t)C)) { 3854 // Constant does not fit, try adjusting it by one. 3855 switch (CC) { 3856 default: break; 3857 case ISD::SETLT: 3858 case ISD::SETGE: 3859 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) { 3860 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; 3861 RHS = DAG.getConstant(C - 1, dl, MVT::i32); 3862 } 3863 break; 3864 case ISD::SETULT: 3865 case ISD::SETUGE: 3866 if (C != 0 && isLegalICmpImmediate(C-1)) { 3867 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; 3868 RHS = DAG.getConstant(C - 1, dl, MVT::i32); 3869 } 3870 break; 3871 case ISD::SETLE: 3872 case ISD::SETGT: 3873 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) { 3874 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; 3875 RHS = DAG.getConstant(C + 1, dl, MVT::i32); 3876 } 3877 break; 3878 case ISD::SETULE: 3879 case ISD::SETUGT: 3880 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) { 3881 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; 3882 RHS = DAG.getConstant(C + 1, dl, MVT::i32); 3883 } 3884 break; 3885 } 3886 } 3887 } else if ((ARM_AM::getShiftOpcForNode(LHS.getOpcode()) != ARM_AM::no_shift) && 3888 (ARM_AM::getShiftOpcForNode(RHS.getOpcode()) == ARM_AM::no_shift)) { 3889 // In ARM and Thumb-2, the compare instructions can shift their second 3890 // operand. 3891 CC = ISD::getSetCCSwappedOperands(CC); 3892 std::swap(LHS, RHS); 3893 } 3894 3895 ARMCC::CondCodes CondCode = IntCCToARMCC(CC); 3896 ARMISD::NodeType CompareType; 3897 switch (CondCode) { 3898 default: 3899 CompareType = ARMISD::CMP; 3900 break; 3901 case ARMCC::EQ: 3902 case ARMCC::NE: 3903 // Uses only Z Flag 3904 CompareType = ARMISD::CMPZ; 3905 break; 3906 } 3907 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 3908 return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS); 3909 } 3910 3911 /// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands. 3912 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS, 3913 SelectionDAG &DAG, const SDLoc &dl, 3914 bool InvalidOnQNaN) const { 3915 assert(!Subtarget->isFPOnlySP() || RHS.getValueType() != MVT::f64); 3916 SDValue Cmp; 3917 SDValue C = DAG.getConstant(InvalidOnQNaN, dl, MVT::i32); 3918 if (!isFloatingPointZero(RHS)) 3919 Cmp = DAG.getNode(ARMISD::CMPFP, dl, MVT::Glue, LHS, RHS, C); 3920 else 3921 Cmp = DAG.getNode(ARMISD::CMPFPw0, dl, MVT::Glue, LHS, C); 3922 return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Cmp); 3923 } 3924 3925 /// duplicateCmp - Glue values can have only one use, so this function 3926 /// duplicates a comparison node. 3927 SDValue 3928 ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const { 3929 unsigned Opc = Cmp.getOpcode(); 3930 SDLoc DL(Cmp); 3931 if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ) 3932 return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1)); 3933 3934 assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation"); 3935 Cmp = Cmp.getOperand(0); 3936 Opc = Cmp.getOpcode(); 3937 if (Opc == ARMISD::CMPFP) 3938 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0), 3939 Cmp.getOperand(1), Cmp.getOperand(2)); 3940 else { 3941 assert(Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT"); 3942 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0), 3943 Cmp.getOperand(1)); 3944 } 3945 return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Cmp); 3946 } 3947 3948 // This function returns three things: the arithmetic computation itself 3949 // (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The 3950 // comparison and the condition code define the case in which the arithmetic 3951 // computation *does not* overflow. 3952 std::pair<SDValue, SDValue> 3953 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG, 3954 SDValue &ARMcc) const { 3955 assert(Op.getValueType() == MVT::i32 && "Unsupported value type"); 3956 3957 SDValue Value, OverflowCmp; 3958 SDValue LHS = Op.getOperand(0); 3959 SDValue RHS = Op.getOperand(1); 3960 SDLoc dl(Op); 3961 3962 // FIXME: We are currently always generating CMPs because we don't support 3963 // generating CMN through the backend. This is not as good as the natural 3964 // CMP case because it causes a register dependency and cannot be folded 3965 // later. 3966 3967 switch (Op.getOpcode()) { 3968 default: 3969 llvm_unreachable("Unknown overflow instruction!"); 3970 case ISD::SADDO: 3971 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32); 3972 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS); 3973 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS); 3974 break; 3975 case ISD::UADDO: 3976 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32); 3977 // We use ADDC here to correspond to its use in LowerUnsignedALUO. 3978 // We do not use it in the USUBO case as Value may not be used. 3979 Value = DAG.getNode(ARMISD::ADDC, dl, 3980 DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS) 3981 .getValue(0); 3982 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS); 3983 break; 3984 case ISD::SSUBO: 3985 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32); 3986 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS); 3987 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS); 3988 break; 3989 case ISD::USUBO: 3990 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32); 3991 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS); 3992 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS); 3993 break; 3994 case ISD::UMULO: 3995 // We generate a UMUL_LOHI and then check if the high word is 0. 3996 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32); 3997 Value = DAG.getNode(ISD::UMUL_LOHI, dl, 3998 DAG.getVTList(Op.getValueType(), Op.getValueType()), 3999 LHS, RHS); 4000 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1), 4001 DAG.getConstant(0, dl, MVT::i32)); 4002 Value = Value.getValue(0); // We only want the low 32 bits for the result. 4003 break; 4004 case ISD::SMULO: 4005 // We generate a SMUL_LOHI and then check if all the bits of the high word 4006 // are the same as the sign bit of the low word. 4007 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32); 4008 Value = DAG.getNode(ISD::SMUL_LOHI, dl, 4009 DAG.getVTList(Op.getValueType(), Op.getValueType()), 4010 LHS, RHS); 4011 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1), 4012 DAG.getNode(ISD::SRA, dl, Op.getValueType(), 4013 Value.getValue(0), 4014 DAG.getConstant(31, dl, MVT::i32))); 4015 Value = Value.getValue(0); // We only want the low 32 bits for the result. 4016 break; 4017 } // switch (...) 4018 4019 return std::make_pair(Value, OverflowCmp); 4020 } 4021 4022 SDValue 4023 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const { 4024 // Let legalize expand this if it isn't a legal type yet. 4025 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType())) 4026 return SDValue(); 4027 4028 SDValue Value, OverflowCmp; 4029 SDValue ARMcc; 4030 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc); 4031 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4032 SDLoc dl(Op); 4033 // We use 0 and 1 as false and true values. 4034 SDValue TVal = DAG.getConstant(1, dl, MVT::i32); 4035 SDValue FVal = DAG.getConstant(0, dl, MVT::i32); 4036 EVT VT = Op.getValueType(); 4037 4038 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal, 4039 ARMcc, CCR, OverflowCmp); 4040 4041 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); 4042 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); 4043 } 4044 4045 static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, 4046 SelectionDAG &DAG) { 4047 SDLoc DL(BoolCarry); 4048 EVT CarryVT = BoolCarry.getValueType(); 4049 4050 // This converts the boolean value carry into the carry flag by doing 4051 // ARMISD::SUBC Carry, 1 4052 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL, 4053 DAG.getVTList(CarryVT, MVT::i32), 4054 BoolCarry, DAG.getConstant(1, DL, CarryVT)); 4055 return Carry.getValue(1); 4056 } 4057 4058 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, 4059 SelectionDAG &DAG) { 4060 SDLoc DL(Flags); 4061 4062 // Now convert the carry flag into a boolean carry. We do this 4063 // using ARMISD:ADDE 0, 0, Carry 4064 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32), 4065 DAG.getConstant(0, DL, MVT::i32), 4066 DAG.getConstant(0, DL, MVT::i32), Flags); 4067 } 4068 4069 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op, 4070 SelectionDAG &DAG) const { 4071 // Let legalize expand this if it isn't a legal type yet. 4072 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType())) 4073 return SDValue(); 4074 4075 SDValue LHS = Op.getOperand(0); 4076 SDValue RHS = Op.getOperand(1); 4077 SDLoc dl(Op); 4078 4079 EVT VT = Op.getValueType(); 4080 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 4081 SDValue Value; 4082 SDValue Overflow; 4083 switch (Op.getOpcode()) { 4084 default: 4085 llvm_unreachable("Unknown overflow instruction!"); 4086 case ISD::UADDO: 4087 Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS); 4088 // Convert the carry flag into a boolean value. 4089 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); 4090 break; 4091 case ISD::USUBO: { 4092 Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS); 4093 // Convert the carry flag into a boolean value. 4094 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); 4095 // ARMISD::SUBC returns 0 when we have to borrow, so make it an overflow 4096 // value. So compute 1 - C. 4097 Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32, 4098 DAG.getConstant(1, dl, MVT::i32), Overflow); 4099 break; 4100 } 4101 } 4102 4103 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); 4104 } 4105 4106 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { 4107 SDValue Cond = Op.getOperand(0); 4108 SDValue SelectTrue = Op.getOperand(1); 4109 SDValue SelectFalse = Op.getOperand(2); 4110 SDLoc dl(Op); 4111 unsigned Opc = Cond.getOpcode(); 4112 4113 if (Cond.getResNo() == 1 && 4114 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 4115 Opc == ISD::USUBO)) { 4116 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) 4117 return SDValue(); 4118 4119 SDValue Value, OverflowCmp; 4120 SDValue ARMcc; 4121 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc); 4122 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4123 EVT VT = Op.getValueType(); 4124 4125 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR, 4126 OverflowCmp, DAG); 4127 } 4128 4129 // Convert: 4130 // 4131 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond) 4132 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond) 4133 // 4134 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) { 4135 const ConstantSDNode *CMOVTrue = 4136 dyn_cast<ConstantSDNode>(Cond.getOperand(0)); 4137 const ConstantSDNode *CMOVFalse = 4138 dyn_cast<ConstantSDNode>(Cond.getOperand(1)); 4139 4140 if (CMOVTrue && CMOVFalse) { 4141 unsigned CMOVTrueVal = CMOVTrue->getZExtValue(); 4142 unsigned CMOVFalseVal = CMOVFalse->getZExtValue(); 4143 4144 SDValue True; 4145 SDValue False; 4146 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) { 4147 True = SelectTrue; 4148 False = SelectFalse; 4149 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) { 4150 True = SelectFalse; 4151 False = SelectTrue; 4152 } 4153 4154 if (True.getNode() && False.getNode()) { 4155 EVT VT = Op.getValueType(); 4156 SDValue ARMcc = Cond.getOperand(2); 4157 SDValue CCR = Cond.getOperand(3); 4158 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG); 4159 assert(True.getValueType() == VT); 4160 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG); 4161 } 4162 } 4163 } 4164 4165 // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the 4166 // undefined bits before doing a full-word comparison with zero. 4167 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond, 4168 DAG.getConstant(1, dl, Cond.getValueType())); 4169 4170 return DAG.getSelectCC(dl, Cond, 4171 DAG.getConstant(0, dl, Cond.getValueType()), 4172 SelectTrue, SelectFalse, ISD::SETNE); 4173 } 4174 4175 static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, 4176 bool &swpCmpOps, bool &swpVselOps) { 4177 // Start by selecting the GE condition code for opcodes that return true for 4178 // 'equality' 4179 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE || 4180 CC == ISD::SETULE) 4181 CondCode = ARMCC::GE; 4182 4183 // and GT for opcodes that return false for 'equality'. 4184 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT || 4185 CC == ISD::SETULT) 4186 CondCode = ARMCC::GT; 4187 4188 // Since we are constrained to GE/GT, if the opcode contains 'less', we need 4189 // to swap the compare operands. 4190 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT || 4191 CC == ISD::SETULT) 4192 swpCmpOps = true; 4193 4194 // Both GT and GE are ordered comparisons, and return false for 'unordered'. 4195 // If we have an unordered opcode, we need to swap the operands to the VSEL 4196 // instruction (effectively negating the condition). 4197 // 4198 // This also has the effect of swapping which one of 'less' or 'greater' 4199 // returns true, so we also swap the compare operands. It also switches 4200 // whether we return true for 'equality', so we compensate by picking the 4201 // opposite condition code to our original choice. 4202 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE || 4203 CC == ISD::SETUGT) { 4204 swpCmpOps = !swpCmpOps; 4205 swpVselOps = !swpVselOps; 4206 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT; 4207 } 4208 4209 // 'ordered' is 'anything but unordered', so use the VS condition code and 4210 // swap the VSEL operands. 4211 if (CC == ISD::SETO) { 4212 CondCode = ARMCC::VS; 4213 swpVselOps = true; 4214 } 4215 4216 // 'unordered or not equal' is 'anything but equal', so use the EQ condition 4217 // code and swap the VSEL operands. 4218 if (CC == ISD::SETUNE) { 4219 CondCode = ARMCC::EQ; 4220 swpVselOps = true; 4221 } 4222 } 4223 4224 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, 4225 SDValue TrueVal, SDValue ARMcc, SDValue CCR, 4226 SDValue Cmp, SelectionDAG &DAG) const { 4227 if (Subtarget->isFPOnlySP() && VT == MVT::f64) { 4228 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl, 4229 DAG.getVTList(MVT::i32, MVT::i32), FalseVal); 4230 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl, 4231 DAG.getVTList(MVT::i32, MVT::i32), TrueVal); 4232 4233 SDValue TrueLow = TrueVal.getValue(0); 4234 SDValue TrueHigh = TrueVal.getValue(1); 4235 SDValue FalseLow = FalseVal.getValue(0); 4236 SDValue FalseHigh = FalseVal.getValue(1); 4237 4238 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow, 4239 ARMcc, CCR, Cmp); 4240 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh, 4241 ARMcc, CCR, duplicateCmp(Cmp, DAG)); 4242 4243 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High); 4244 } else { 4245 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR, 4246 Cmp); 4247 } 4248 } 4249 4250 static bool isGTorGE(ISD::CondCode CC) { 4251 return CC == ISD::SETGT || CC == ISD::SETGE; 4252 } 4253 4254 static bool isLTorLE(ISD::CondCode CC) { 4255 return CC == ISD::SETLT || CC == ISD::SETLE; 4256 } 4257 4258 // See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating. 4259 // All of these conditions (and their <= and >= counterparts) will do: 4260 // x < k ? k : x 4261 // x > k ? x : k 4262 // k < x ? x : k 4263 // k > x ? k : x 4264 static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, 4265 const SDValue TrueVal, const SDValue FalseVal, 4266 const ISD::CondCode CC, const SDValue K) { 4267 return (isGTorGE(CC) && 4268 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) || 4269 (isLTorLE(CC) && 4270 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))); 4271 } 4272 4273 // Similar to isLowerSaturate(), but checks for upper-saturating conditions. 4274 static bool isUpperSaturate(const SDValue LHS, const SDValue RHS, 4275 const SDValue TrueVal, const SDValue FalseVal, 4276 const ISD::CondCode CC, const SDValue K) { 4277 return (isGTorGE(CC) && 4278 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))) || 4279 (isLTorLE(CC) && 4280 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))); 4281 } 4282 4283 // Check if two chained conditionals could be converted into SSAT or USAT. 4284 // 4285 // SSAT can replace a set of two conditional selectors that bound a number to an 4286 // interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples: 4287 // 4288 // x < -k ? -k : (x > k ? k : x) 4289 // x < -k ? -k : (x < k ? x : k) 4290 // x > -k ? (x > k ? k : x) : -k 4291 // x < k ? (x < -k ? -k : x) : k 4292 // etc. 4293 // 4294 // USAT works similarily to SSAT but bounds on the interval [0, k] where k + 1 is 4295 // a power of 2. 4296 // 4297 // It returns true if the conversion can be done, false otherwise. 4298 // Additionally, the variable is returned in parameter V, the constant in K and 4299 // usat is set to true if the conditional represents an unsigned saturation 4300 static bool isSaturatingConditional(const SDValue &Op, SDValue &V, 4301 uint64_t &K, bool &usat) { 4302 SDValue LHS1 = Op.getOperand(0); 4303 SDValue RHS1 = Op.getOperand(1); 4304 SDValue TrueVal1 = Op.getOperand(2); 4305 SDValue FalseVal1 = Op.getOperand(3); 4306 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4307 4308 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1; 4309 if (Op2.getOpcode() != ISD::SELECT_CC) 4310 return false; 4311 4312 SDValue LHS2 = Op2.getOperand(0); 4313 SDValue RHS2 = Op2.getOperand(1); 4314 SDValue TrueVal2 = Op2.getOperand(2); 4315 SDValue FalseVal2 = Op2.getOperand(3); 4316 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get(); 4317 4318 // Find out which are the constants and which are the variables 4319 // in each conditional 4320 SDValue *K1 = isa<ConstantSDNode>(LHS1) ? &LHS1 : isa<ConstantSDNode>(RHS1) 4321 ? &RHS1 4322 : nullptr; 4323 SDValue *K2 = isa<ConstantSDNode>(LHS2) ? &LHS2 : isa<ConstantSDNode>(RHS2) 4324 ? &RHS2 4325 : nullptr; 4326 SDValue K2Tmp = isa<ConstantSDNode>(TrueVal2) ? TrueVal2 : FalseVal2; 4327 SDValue V1Tmp = (K1 && *K1 == LHS1) ? RHS1 : LHS1; 4328 SDValue V2Tmp = (K2 && *K2 == LHS2) ? RHS2 : LHS2; 4329 SDValue V2 = (K2Tmp == TrueVal2) ? FalseVal2 : TrueVal2; 4330 4331 // We must detect cases where the original operations worked with 16- or 4332 // 8-bit values. In such case, V2Tmp != V2 because the comparison operations 4333 // must work with sign-extended values but the select operations return 4334 // the original non-extended value. 4335 SDValue V2TmpReg = V2Tmp; 4336 if (V2Tmp->getOpcode() == ISD::SIGN_EXTEND_INREG) 4337 V2TmpReg = V2Tmp->getOperand(0); 4338 4339 // Check that the registers and the constants have the correct values 4340 // in both conditionals 4341 if (!K1 || !K2 || *K1 == Op2 || *K2 != K2Tmp || V1Tmp != V2Tmp || 4342 V2TmpReg != V2) 4343 return false; 4344 4345 // Figure out which conditional is saturating the lower/upper bound. 4346 const SDValue *LowerCheckOp = 4347 isLowerSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1) 4348 ? &Op 4349 : isLowerSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2) 4350 ? &Op2 4351 : nullptr; 4352 const SDValue *UpperCheckOp = 4353 isUpperSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1) 4354 ? &Op 4355 : isUpperSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2) 4356 ? &Op2 4357 : nullptr; 4358 4359 if (!UpperCheckOp || !LowerCheckOp || LowerCheckOp == UpperCheckOp) 4360 return false; 4361 4362 // Check that the constant in the lower-bound check is 4363 // the opposite of the constant in the upper-bound check 4364 // in 1's complement. 4365 int64_t Val1 = cast<ConstantSDNode>(*K1)->getSExtValue(); 4366 int64_t Val2 = cast<ConstantSDNode>(*K2)->getSExtValue(); 4367 int64_t PosVal = std::max(Val1, Val2); 4368 int64_t NegVal = std::min(Val1, Val2); 4369 4370 if (((Val1 > Val2 && UpperCheckOp == &Op) || 4371 (Val1 < Val2 && UpperCheckOp == &Op2)) && 4372 isPowerOf2_64(PosVal + 1)) { 4373 4374 // Handle the difference between USAT (unsigned) and SSAT (signed) saturation 4375 if (Val1 == ~Val2) 4376 usat = false; 4377 else if (NegVal == 0) 4378 usat = true; 4379 else 4380 return false; 4381 4382 V = V2; 4383 K = (uint64_t)PosVal; // At this point, PosVal is guaranteed to be positive 4384 4385 return true; 4386 } 4387 4388 return false; 4389 } 4390 4391 // Check if a condition of the type x < k ? k : x can be converted into a 4392 // bit operation instead of conditional moves. 4393 // Currently this is allowed given: 4394 // - The conditions and values match up 4395 // - k is 0 or -1 (all ones) 4396 // This function will not check the last condition, thats up to the caller 4397 // It returns true if the transformation can be made, and in such case 4398 // returns x in V, and k in SatK. 4399 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, 4400 SDValue &SatK) 4401 { 4402 SDValue LHS = Op.getOperand(0); 4403 SDValue RHS = Op.getOperand(1); 4404 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4405 SDValue TrueVal = Op.getOperand(2); 4406 SDValue FalseVal = Op.getOperand(3); 4407 4408 SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS) 4409 ? &RHS 4410 : nullptr; 4411 4412 // No constant operation in comparison, early out 4413 if (!K) 4414 return false; 4415 4416 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal; 4417 V = (KTmp == TrueVal) ? FalseVal : TrueVal; 4418 SDValue VTmp = (K && *K == LHS) ? RHS : LHS; 4419 4420 // If the constant on left and right side, or variable on left and right, 4421 // does not match, early out 4422 if (*K != KTmp || V != VTmp) 4423 return false; 4424 4425 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, *K)) { 4426 SatK = *K; 4427 return true; 4428 } 4429 4430 return false; 4431 } 4432 4433 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 4434 EVT VT = Op.getValueType(); 4435 SDLoc dl(Op); 4436 4437 // Try to convert two saturating conditional selects into a single SSAT 4438 SDValue SatValue; 4439 uint64_t SatConstant; 4440 bool SatUSat; 4441 if (((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2()) && 4442 isSaturatingConditional(Op, SatValue, SatConstant, SatUSat)) { 4443 if (SatUSat) 4444 return DAG.getNode(ARMISD::USAT, dl, VT, SatValue, 4445 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT)); 4446 else 4447 return DAG.getNode(ARMISD::SSAT, dl, VT, SatValue, 4448 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT)); 4449 } 4450 4451 // Try to convert expressions of the form x < k ? k : x (and similar forms) 4452 // into more efficient bit operations, which is possible when k is 0 or -1 4453 // On ARM and Thumb-2 which have flexible operand 2 this will result in 4454 // single instructions. On Thumb the shift and the bit operation will be two 4455 // instructions. 4456 // Only allow this transformation on full-width (32-bit) operations 4457 SDValue LowerSatConstant; 4458 if (VT == MVT::i32 && 4459 isLowerSaturatingConditional(Op, SatValue, LowerSatConstant)) { 4460 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue, 4461 DAG.getConstant(31, dl, VT)); 4462 if (isNullConstant(LowerSatConstant)) { 4463 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV, 4464 DAG.getAllOnesConstant(dl, VT)); 4465 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV); 4466 } else if (isAllOnesConstant(LowerSatConstant)) 4467 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV); 4468 } 4469 4470 SDValue LHS = Op.getOperand(0); 4471 SDValue RHS = Op.getOperand(1); 4472 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4473 SDValue TrueVal = Op.getOperand(2); 4474 SDValue FalseVal = Op.getOperand(3); 4475 4476 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) { 4477 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC, 4478 dl); 4479 4480 // If softenSetCCOperands only returned one value, we should compare it to 4481 // zero. 4482 if (!RHS.getNode()) { 4483 RHS = DAG.getConstant(0, dl, LHS.getValueType()); 4484 CC = ISD::SETNE; 4485 } 4486 } 4487 4488 if (LHS.getValueType() == MVT::i32) { 4489 // Try to generate VSEL on ARMv8. 4490 // The VSEL instruction can't use all the usual ARM condition 4491 // codes: it only has two bits to select the condition code, so it's 4492 // constrained to use only GE, GT, VS and EQ. 4493 // 4494 // To implement all the various ISD::SETXXX opcodes, we sometimes need to 4495 // swap the operands of the previous compare instruction (effectively 4496 // inverting the compare condition, swapping 'less' and 'greater') and 4497 // sometimes need to swap the operands to the VSEL (which inverts the 4498 // condition in the sense of firing whenever the previous condition didn't) 4499 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 || 4500 TrueVal.getValueType() == MVT::f64)) { 4501 ARMCC::CondCodes CondCode = IntCCToARMCC(CC); 4502 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE || 4503 CondCode == ARMCC::VC || CondCode == ARMCC::NE) { 4504 CC = ISD::getSetCCInverse(CC, true); 4505 std::swap(TrueVal, FalseVal); 4506 } 4507 } 4508 4509 SDValue ARMcc; 4510 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4511 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl); 4512 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); 4513 } 4514 4515 ARMCC::CondCodes CondCode, CondCode2; 4516 bool InvalidOnQNaN; 4517 FPCCToARMCC(CC, CondCode, CondCode2, InvalidOnQNaN); 4518 4519 // Normalize the fp compare. If RHS is zero we keep it there so we match 4520 // CMPFPw0 instead of CMPFP. 4521 if (Subtarget->hasFPARMv8() && !isFloatingPointZero(RHS) && 4522 (TrueVal.getValueType() == MVT::f16 || 4523 TrueVal.getValueType() == MVT::f32 || 4524 TrueVal.getValueType() == MVT::f64)) { 4525 bool swpCmpOps = false; 4526 bool swpVselOps = false; 4527 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps); 4528 4529 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE || 4530 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) { 4531 if (swpCmpOps) 4532 std::swap(LHS, RHS); 4533 if (swpVselOps) 4534 std::swap(TrueVal, FalseVal); 4535 } 4536 } 4537 4538 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 4539 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, InvalidOnQNaN); 4540 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4541 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); 4542 if (CondCode2 != ARMCC::AL) { 4543 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32); 4544 // FIXME: Needs another CMP because flag can have but one use. 4545 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl, InvalidOnQNaN); 4546 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG); 4547 } 4548 return Result; 4549 } 4550 4551 /// canChangeToInt - Given the fp compare operand, return true if it is suitable 4552 /// to morph to an integer compare sequence. 4553 static bool canChangeToInt(SDValue Op, bool &SeenZero, 4554 const ARMSubtarget *Subtarget) { 4555 SDNode *N = Op.getNode(); 4556 if (!N->hasOneUse()) 4557 // Otherwise it requires moving the value from fp to integer registers. 4558 return false; 4559 if (!N->getNumValues()) 4560 return false; 4561 EVT VT = Op.getValueType(); 4562 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow()) 4563 // f32 case is generally profitable. f64 case only makes sense when vcmpe + 4564 // vmrs are very slow, e.g. cortex-a8. 4565 return false; 4566 4567 if (isFloatingPointZero(Op)) { 4568 SeenZero = true; 4569 return true; 4570 } 4571 return ISD::isNormalLoad(N); 4572 } 4573 4574 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) { 4575 if (isFloatingPointZero(Op)) 4576 return DAG.getConstant(0, SDLoc(Op), MVT::i32); 4577 4578 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) 4579 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(), 4580 Ld->getPointerInfo(), Ld->getAlignment(), 4581 Ld->getMemOperand()->getFlags()); 4582 4583 llvm_unreachable("Unknown VFP cmp argument!"); 4584 } 4585 4586 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, 4587 SDValue &RetVal1, SDValue &RetVal2) { 4588 SDLoc dl(Op); 4589 4590 if (isFloatingPointZero(Op)) { 4591 RetVal1 = DAG.getConstant(0, dl, MVT::i32); 4592 RetVal2 = DAG.getConstant(0, dl, MVT::i32); 4593 return; 4594 } 4595 4596 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) { 4597 SDValue Ptr = Ld->getBasePtr(); 4598 RetVal1 = 4599 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(), 4600 Ld->getAlignment(), Ld->getMemOperand()->getFlags()); 4601 4602 EVT PtrType = Ptr.getValueType(); 4603 unsigned NewAlign = MinAlign(Ld->getAlignment(), 4); 4604 SDValue NewPtr = DAG.getNode(ISD::ADD, dl, 4605 PtrType, Ptr, DAG.getConstant(4, dl, PtrType)); 4606 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr, 4607 Ld->getPointerInfo().getWithOffset(4), NewAlign, 4608 Ld->getMemOperand()->getFlags()); 4609 return; 4610 } 4611 4612 llvm_unreachable("Unknown VFP cmp argument!"); 4613 } 4614 4615 /// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some 4616 /// f32 and even f64 comparisons to integer ones. 4617 SDValue 4618 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const { 4619 SDValue Chain = Op.getOperand(0); 4620 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 4621 SDValue LHS = Op.getOperand(2); 4622 SDValue RHS = Op.getOperand(3); 4623 SDValue Dest = Op.getOperand(4); 4624 SDLoc dl(Op); 4625 4626 bool LHSSeenZero = false; 4627 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget); 4628 bool RHSSeenZero = false; 4629 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget); 4630 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) { 4631 // If unsafe fp math optimization is enabled and there are no other uses of 4632 // the CMP operands, and the condition code is EQ or NE, we can optimize it 4633 // to an integer comparison. 4634 if (CC == ISD::SETOEQ) 4635 CC = ISD::SETEQ; 4636 else if (CC == ISD::SETUNE) 4637 CC = ISD::SETNE; 4638 4639 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32); 4640 SDValue ARMcc; 4641 if (LHS.getValueType() == MVT::f32) { 4642 LHS = DAG.getNode(ISD::AND, dl, MVT::i32, 4643 bitcastf32Toi32(LHS, DAG), Mask); 4644 RHS = DAG.getNode(ISD::AND, dl, MVT::i32, 4645 bitcastf32Toi32(RHS, DAG), Mask); 4646 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl); 4647 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4648 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, 4649 Chain, Dest, ARMcc, CCR, Cmp); 4650 } 4651 4652 SDValue LHS1, LHS2; 4653 SDValue RHS1, RHS2; 4654 expandf64Toi32(LHS, DAG, LHS1, LHS2); 4655 expandf64Toi32(RHS, DAG, RHS1, RHS2); 4656 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask); 4657 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask); 4658 ARMCC::CondCodes CondCode = IntCCToARMCC(CC); 4659 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 4660 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue); 4661 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest }; 4662 return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops); 4663 } 4664 4665 return SDValue(); 4666 } 4667 4668 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 4669 SDValue Chain = Op.getOperand(0); 4670 SDValue Cond = Op.getOperand(1); 4671 SDValue Dest = Op.getOperand(2); 4672 SDLoc dl(Op); 4673 4674 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch 4675 // instruction. 4676 unsigned Opc = Cond.getOpcode(); 4677 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) && 4678 !Subtarget->isThumb1Only(); 4679 if (Cond.getResNo() == 1 && 4680 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 4681 Opc == ISD::USUBO || OptimizeMul)) { 4682 // Only lower legal XALUO ops. 4683 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) 4684 return SDValue(); 4685 4686 // The actual operation with overflow check. 4687 SDValue Value, OverflowCmp; 4688 SDValue ARMcc; 4689 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc); 4690 4691 // Reverse the condition code. 4692 ARMCC::CondCodes CondCode = 4693 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue(); 4694 CondCode = ARMCC::getOppositeCondition(CondCode); 4695 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32); 4696 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4697 4698 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR, 4699 OverflowCmp); 4700 } 4701 4702 return SDValue(); 4703 } 4704 4705 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 4706 SDValue Chain = Op.getOperand(0); 4707 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 4708 SDValue LHS = Op.getOperand(2); 4709 SDValue RHS = Op.getOperand(3); 4710 SDValue Dest = Op.getOperand(4); 4711 SDLoc dl(Op); 4712 4713 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) { 4714 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC, 4715 dl); 4716 4717 // If softenSetCCOperands only returned one value, we should compare it to 4718 // zero. 4719 if (!RHS.getNode()) { 4720 RHS = DAG.getConstant(0, dl, LHS.getValueType()); 4721 CC = ISD::SETNE; 4722 } 4723 } 4724 4725 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch 4726 // instruction. 4727 unsigned Opc = LHS.getOpcode(); 4728 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) && 4729 !Subtarget->isThumb1Only(); 4730 if (LHS.getResNo() == 1 && (isOneConstant(RHS) || isNullConstant(RHS)) && 4731 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 4732 Opc == ISD::USUBO || OptimizeMul) && 4733 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 4734 // Only lower legal XALUO ops. 4735 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0))) 4736 return SDValue(); 4737 4738 // The actual operation with overflow check. 4739 SDValue Value, OverflowCmp; 4740 SDValue ARMcc; 4741 std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc); 4742 4743 if ((CC == ISD::SETNE) != isOneConstant(RHS)) { 4744 // Reverse the condition code. 4745 ARMCC::CondCodes CondCode = 4746 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue(); 4747 CondCode = ARMCC::getOppositeCondition(CondCode); 4748 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32); 4749 } 4750 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4751 4752 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR, 4753 OverflowCmp); 4754 } 4755 4756 if (LHS.getValueType() == MVT::i32) { 4757 SDValue ARMcc; 4758 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl); 4759 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4760 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, 4761 Chain, Dest, ARMcc, CCR, Cmp); 4762 } 4763 4764 if (getTargetMachine().Options.UnsafeFPMath && 4765 (CC == ISD::SETEQ || CC == ISD::SETOEQ || 4766 CC == ISD::SETNE || CC == ISD::SETUNE)) { 4767 if (SDValue Result = OptimizeVFPBrcond(Op, DAG)) 4768 return Result; 4769 } 4770 4771 ARMCC::CondCodes CondCode, CondCode2; 4772 bool InvalidOnQNaN; 4773 FPCCToARMCC(CC, CondCode, CondCode2, InvalidOnQNaN); 4774 4775 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 4776 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, InvalidOnQNaN); 4777 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4778 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue); 4779 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp }; 4780 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops); 4781 if (CondCode2 != ARMCC::AL) { 4782 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32); 4783 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) }; 4784 Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops); 4785 } 4786 return Res; 4787 } 4788 4789 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const { 4790 SDValue Chain = Op.getOperand(0); 4791 SDValue Table = Op.getOperand(1); 4792 SDValue Index = Op.getOperand(2); 4793 SDLoc dl(Op); 4794 4795 EVT PTy = getPointerTy(DAG.getDataLayout()); 4796 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table); 4797 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy); 4798 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI); 4799 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy)); 4800 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index); 4801 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) { 4802 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table 4803 // which does another jump to the destination. This also makes it easier 4804 // to translate it to TBB / TBH later (Thumb2 only). 4805 // FIXME: This might not work if the function is extremely large. 4806 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain, 4807 Addr, Op.getOperand(2), JTI); 4808 } 4809 if (isPositionIndependent() || Subtarget->isROPI()) { 4810 Addr = 4811 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr, 4812 MachinePointerInfo::getJumpTable(DAG.getMachineFunction())); 4813 Chain = Addr.getValue(1); 4814 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr); 4815 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI); 4816 } else { 4817 Addr = 4818 DAG.getLoad(PTy, dl, Chain, Addr, 4819 MachinePointerInfo::getJumpTable(DAG.getMachineFunction())); 4820 Chain = Addr.getValue(1); 4821 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI); 4822 } 4823 } 4824 4825 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) { 4826 EVT VT = Op.getValueType(); 4827 SDLoc dl(Op); 4828 4829 if (Op.getValueType().getVectorElementType() == MVT::i32) { 4830 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32) 4831 return Op; 4832 return DAG.UnrollVectorOp(Op.getNode()); 4833 } 4834 4835 assert(Op.getOperand(0).getValueType() == MVT::v4f32 && 4836 "Invalid type for custom lowering!"); 4837 if (VT != MVT::v4i16) 4838 return DAG.UnrollVectorOp(Op.getNode()); 4839 4840 Op = DAG.getNode(Op.getOpcode(), dl, MVT::v4i32, Op.getOperand(0)); 4841 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op); 4842 } 4843 4844 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const { 4845 EVT VT = Op.getValueType(); 4846 if (VT.isVector()) 4847 return LowerVectorFP_TO_INT(Op, DAG); 4848 if (Subtarget->isFPOnlySP() && Op.getOperand(0).getValueType() == MVT::f64) { 4849 RTLIB::Libcall LC; 4850 if (Op.getOpcode() == ISD::FP_TO_SINT) 4851 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), 4852 Op.getValueType()); 4853 else 4854 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), 4855 Op.getValueType()); 4856 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0), 4857 /*isSigned*/ false, SDLoc(Op)).first; 4858 } 4859 4860 return Op; 4861 } 4862 4863 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 4864 EVT VT = Op.getValueType(); 4865 SDLoc dl(Op); 4866 4867 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) { 4868 if (VT.getVectorElementType() == MVT::f32) 4869 return Op; 4870 return DAG.UnrollVectorOp(Op.getNode()); 4871 } 4872 4873 assert(Op.getOperand(0).getValueType() == MVT::v4i16 && 4874 "Invalid type for custom lowering!"); 4875 if (VT != MVT::v4f32) 4876 return DAG.UnrollVectorOp(Op.getNode()); 4877 4878 unsigned CastOpc; 4879 unsigned Opc; 4880 switch (Op.getOpcode()) { 4881 default: llvm_unreachable("Invalid opcode!"); 4882 case ISD::SINT_TO_FP: 4883 CastOpc = ISD::SIGN_EXTEND; 4884 Opc = ISD::SINT_TO_FP; 4885 break; 4886 case ISD::UINT_TO_FP: 4887 CastOpc = ISD::ZERO_EXTEND; 4888 Opc = ISD::UINT_TO_FP; 4889 break; 4890 } 4891 4892 Op = DAG.getNode(CastOpc, dl, MVT::v4i32, Op.getOperand(0)); 4893 return DAG.getNode(Opc, dl, VT, Op); 4894 } 4895 4896 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { 4897 EVT VT = Op.getValueType(); 4898 if (VT.isVector()) 4899 return LowerVectorINT_TO_FP(Op, DAG); 4900 if (Subtarget->isFPOnlySP() && Op.getValueType() == MVT::f64) { 4901 RTLIB::Libcall LC; 4902 if (Op.getOpcode() == ISD::SINT_TO_FP) 4903 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), 4904 Op.getValueType()); 4905 else 4906 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), 4907 Op.getValueType()); 4908 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0), 4909 /*isSigned*/ false, SDLoc(Op)).first; 4910 } 4911 4912 return Op; 4913 } 4914 4915 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { 4916 // Implement fcopysign with a fabs and a conditional fneg. 4917 SDValue Tmp0 = Op.getOperand(0); 4918 SDValue Tmp1 = Op.getOperand(1); 4919 SDLoc dl(Op); 4920 EVT VT = Op.getValueType(); 4921 EVT SrcVT = Tmp1.getValueType(); 4922 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST || 4923 Tmp0.getOpcode() == ARMISD::VMOVDRR; 4924 bool UseNEON = !InGPR && Subtarget->hasNEON(); 4925 4926 if (UseNEON) { 4927 // Use VBSL to copy the sign bit. 4928 unsigned EncodedVal = ARM_AM::createNEONModImm(0x6, 0x80); 4929 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32, 4930 DAG.getTargetConstant(EncodedVal, dl, MVT::i32)); 4931 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64; 4932 if (VT == MVT::f64) 4933 Mask = DAG.getNode(ARMISD::VSHL, dl, OpVT, 4934 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask), 4935 DAG.getConstant(32, dl, MVT::i32)); 4936 else /*if (VT == MVT::f32)*/ 4937 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0); 4938 if (SrcVT == MVT::f32) { 4939 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1); 4940 if (VT == MVT::f64) 4941 Tmp1 = DAG.getNode(ARMISD::VSHL, dl, OpVT, 4942 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1), 4943 DAG.getConstant(32, dl, MVT::i32)); 4944 } else if (VT == MVT::f32) 4945 Tmp1 = DAG.getNode(ARMISD::VSHRu, dl, MVT::v1i64, 4946 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1), 4947 DAG.getConstant(32, dl, MVT::i32)); 4948 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0); 4949 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1); 4950 4951 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createNEONModImm(0xe, 0xff), 4952 dl, MVT::i32); 4953 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes); 4954 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask, 4955 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes)); 4956 4957 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT, 4958 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask), 4959 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot)); 4960 if (VT == MVT::f32) { 4961 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res); 4962 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, 4963 DAG.getConstant(0, dl, MVT::i32)); 4964 } else { 4965 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res); 4966 } 4967 4968 return Res; 4969 } 4970 4971 // Bitcast operand 1 to i32. 4972 if (SrcVT == MVT::f64) 4973 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32), 4974 Tmp1).getValue(1); 4975 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1); 4976 4977 // Or in the signbit with integer operations. 4978 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32); 4979 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32); 4980 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1); 4981 if (VT == MVT::f32) { 4982 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32, 4983 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2); 4984 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4985 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1)); 4986 } 4987 4988 // f64: Or the high part with signbit and then combine two parts. 4989 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32), 4990 Tmp0); 4991 SDValue Lo = Tmp0.getValue(0); 4992 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2); 4993 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1); 4994 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi); 4995 } 4996 4997 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{ 4998 MachineFunction &MF = DAG.getMachineFunction(); 4999 MachineFrameInfo &MFI = MF.getFrameInfo(); 5000 MFI.setReturnAddressIsTaken(true); 5001 5002 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 5003 return SDValue(); 5004 5005 EVT VT = Op.getValueType(); 5006 SDLoc dl(Op); 5007 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 5008 if (Depth) { 5009 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 5010 SDValue Offset = DAG.getConstant(4, dl, MVT::i32); 5011 return DAG.getLoad(VT, dl, DAG.getEntryNode(), 5012 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), 5013 MachinePointerInfo()); 5014 } 5015 5016 // Return LR, which contains the return address. Mark it an implicit live-in. 5017 unsigned Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32)); 5018 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); 5019 } 5020 5021 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { 5022 const ARMBaseRegisterInfo &ARI = 5023 *static_cast<const ARMBaseRegisterInfo*>(RegInfo); 5024 MachineFunction &MF = DAG.getMachineFunction(); 5025 MachineFrameInfo &MFI = MF.getFrameInfo(); 5026 MFI.setFrameAddressIsTaken(true); 5027 5028 EVT VT = Op.getValueType(); 5029 SDLoc dl(Op); // FIXME probably not meaningful 5030 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 5031 unsigned FrameReg = ARI.getFrameRegister(MF); 5032 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); 5033 while (Depth--) 5034 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, 5035 MachinePointerInfo()); 5036 return FrameAddr; 5037 } 5038 5039 // FIXME? Maybe this could be a TableGen attribute on some registers and 5040 // this table could be generated automatically from RegInfo. 5041 unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT, 5042 SelectionDAG &DAG) const { 5043 unsigned Reg = StringSwitch<unsigned>(RegName) 5044 .Case("sp", ARM::SP) 5045 .Default(0); 5046 if (Reg) 5047 return Reg; 5048 report_fatal_error(Twine("Invalid register name \"" 5049 + StringRef(RegName) + "\".")); 5050 } 5051 5052 // Result is 64 bit value so split into two 32 bit values and return as a 5053 // pair of values. 5054 static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results, 5055 SelectionDAG &DAG) { 5056 SDLoc DL(N); 5057 5058 // This function is only supposed to be called for i64 type destination. 5059 assert(N->getValueType(0) == MVT::i64 5060 && "ExpandREAD_REGISTER called for non-i64 type result."); 5061 5062 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL, 5063 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other), 5064 N->getOperand(0), 5065 N->getOperand(1)); 5066 5067 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0), 5068 Read.getValue(1))); 5069 Results.push_back(Read.getOperand(0)); 5070 } 5071 5072 /// \p BC is a bitcast that is about to be turned into a VMOVDRR. 5073 /// When \p DstVT, the destination type of \p BC, is on the vector 5074 /// register bank and the source of bitcast, \p Op, operates on the same bank, 5075 /// it might be possible to combine them, such that everything stays on the 5076 /// vector register bank. 5077 /// \p return The node that would replace \p BT, if the combine 5078 /// is possible. 5079 static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, 5080 SelectionDAG &DAG) { 5081 SDValue Op = BC->getOperand(0); 5082 EVT DstVT = BC->getValueType(0); 5083 5084 // The only vector instruction that can produce a scalar (remember, 5085 // since the bitcast was about to be turned into VMOVDRR, the source 5086 // type is i64) from a vector is EXTRACT_VECTOR_ELT. 5087 // Moreover, we can do this combine only if there is one use. 5088 // Finally, if the destination type is not a vector, there is not 5089 // much point on forcing everything on the vector bank. 5090 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 5091 !Op.hasOneUse()) 5092 return SDValue(); 5093 5094 // If the index is not constant, we will introduce an additional 5095 // multiply that will stick. 5096 // Give up in that case. 5097 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 5098 if (!Index) 5099 return SDValue(); 5100 unsigned DstNumElt = DstVT.getVectorNumElements(); 5101 5102 // Compute the new index. 5103 const APInt &APIntIndex = Index->getAPIntValue(); 5104 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt); 5105 NewIndex *= APIntIndex; 5106 // Check if the new constant index fits into i32. 5107 if (NewIndex.getBitWidth() > 32) 5108 return SDValue(); 5109 5110 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) -> 5111 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M) 5112 SDLoc dl(Op); 5113 SDValue ExtractSrc = Op.getOperand(0); 5114 EVT VecVT = EVT::getVectorVT( 5115 *DAG.getContext(), DstVT.getScalarType(), 5116 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt); 5117 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc); 5118 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast, 5119 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32)); 5120 } 5121 5122 /// ExpandBITCAST - If the target supports VFP, this function is called to 5123 /// expand a bit convert where either the source or destination type is i64 to 5124 /// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64 5125 /// operand type is illegal (e.g., v2f32 for a target that doesn't support 5126 /// vectors), since the legalizer won't know what to do with that. 5127 static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG, 5128 const ARMSubtarget *Subtarget) { 5129 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5130 SDLoc dl(N); 5131 SDValue Op = N->getOperand(0); 5132 5133 // This function is only supposed to be called for i64 types, either as the 5134 // source or destination of the bit convert. 5135 EVT SrcVT = Op.getValueType(); 5136 EVT DstVT = N->getValueType(0); 5137 const bool HasFullFP16 = Subtarget->hasFullFP16(); 5138 5139 if (SrcVT == MVT::f32 && DstVT == MVT::i32) { 5140 // FullFP16: half values are passed in S-registers, and we don't 5141 // need any of the bitcast and moves: 5142 // 5143 // t2: f32,ch = CopyFromReg t0, Register:f32 %0 5144 // t5: i32 = bitcast t2 5145 // t18: f16 = ARMISD::VMOVhr t5 5146 if (Op.getOpcode() != ISD::CopyFromReg || 5147 Op.getValueType() != MVT::f32) 5148 return SDValue(); 5149 5150 auto Move = N->use_begin(); 5151 if (Move->getOpcode() != ARMISD::VMOVhr) 5152 return SDValue(); 5153 5154 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 5155 SDValue Copy = DAG.getNode(ISD::CopyFromReg, SDLoc(Op), MVT::f16, Ops); 5156 DAG.ReplaceAllUsesWith(*Move, &Copy); 5157 return Copy; 5158 } 5159 5160 if (SrcVT == MVT::i16 && DstVT == MVT::f16) { 5161 if (!HasFullFP16) 5162 return SDValue(); 5163 // SoftFP: read half-precision arguments: 5164 // 5165 // t2: i32,ch = ... 5166 // t7: i16 = truncate t2 <~~~~ Op 5167 // t8: f16 = bitcast t7 <~~~~ N 5168 // 5169 if (Op.getOperand(0).getValueType() == MVT::i32) 5170 return DAG.getNode(ARMISD::VMOVhr, SDLoc(Op), 5171 MVT::f16, Op.getOperand(0)); 5172 5173 return SDValue(); 5174 } 5175 5176 // Half-precision return values 5177 if (SrcVT == MVT::f16 && DstVT == MVT::i16) { 5178 if (!HasFullFP16) 5179 return SDValue(); 5180 // 5181 // t11: f16 = fadd t8, t10 5182 // t12: i16 = bitcast t11 <~~~ SDNode N 5183 // t13: i32 = zero_extend t12 5184 // t16: ch,glue = CopyToReg t0, Register:i32 %r0, t13 5185 // t17: ch = ARMISD::RET_FLAG t16, Register:i32 %r0, t16:1 5186 // 5187 // transform this into: 5188 // 5189 // t20: i32 = ARMISD::VMOVrh t11 5190 // t16: ch,glue = CopyToReg t0, Register:i32 %r0, t20 5191 // 5192 auto ZeroExtend = N->use_begin(); 5193 if (N->use_size() != 1 || ZeroExtend->getOpcode() != ISD::ZERO_EXTEND || 5194 ZeroExtend->getValueType(0) != MVT::i32) 5195 return SDValue(); 5196 5197 auto Copy = ZeroExtend->use_begin(); 5198 if (Copy->getOpcode() == ISD::CopyToReg && 5199 Copy->use_begin()->getOpcode() == ARMISD::RET_FLAG) { 5200 SDValue Cvt = DAG.getNode(ARMISD::VMOVrh, SDLoc(Op), MVT::i32, Op); 5201 DAG.ReplaceAllUsesWith(*ZeroExtend, &Cvt); 5202 return Cvt; 5203 } 5204 return SDValue(); 5205 } 5206 5207 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64)) 5208 return SDValue(); 5209 5210 // Turn i64->f64 into VMOVDRR. 5211 if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) { 5212 // Do not force values to GPRs (this is what VMOVDRR does for the inputs) 5213 // if we can combine the bitcast with its source. 5214 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG)) 5215 return Val; 5216 5217 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op, 5218 DAG.getConstant(0, dl, MVT::i32)); 5219 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op, 5220 DAG.getConstant(1, dl, MVT::i32)); 5221 return DAG.getNode(ISD::BITCAST, dl, DstVT, 5222 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi)); 5223 } 5224 5225 // Turn f64->i64 into VMOVRRD. 5226 if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) { 5227 SDValue Cvt; 5228 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() && 5229 SrcVT.getVectorNumElements() > 1) 5230 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl, 5231 DAG.getVTList(MVT::i32, MVT::i32), 5232 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op)); 5233 else 5234 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl, 5235 DAG.getVTList(MVT::i32, MVT::i32), Op); 5236 // Merge the pieces into a single i64 value. 5237 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1)); 5238 } 5239 5240 return SDValue(); 5241 } 5242 5243 /// getZeroVector - Returns a vector of specified type with all zero elements. 5244 /// Zero vectors are used to represent vector negation and in those cases 5245 /// will be implemented with the NEON VNEG instruction. However, VNEG does 5246 /// not support i64 elements, so sometimes the zero vectors will need to be 5247 /// explicitly constructed. Regardless, use a canonical VMOV to create the 5248 /// zero vector. 5249 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { 5250 assert(VT.isVector() && "Expected a vector type"); 5251 // The canonical modified immediate encoding of a zero vector is....0! 5252 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32); 5253 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; 5254 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal); 5255 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); 5256 } 5257 5258 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two 5259 /// i32 values and take a 2 x i32 value to shift plus a shift amount. 5260 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op, 5261 SelectionDAG &DAG) const { 5262 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 5263 EVT VT = Op.getValueType(); 5264 unsigned VTBits = VT.getSizeInBits(); 5265 SDLoc dl(Op); 5266 SDValue ShOpLo = Op.getOperand(0); 5267 SDValue ShOpHi = Op.getOperand(1); 5268 SDValue ShAmt = Op.getOperand(2); 5269 SDValue ARMcc; 5270 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 5271 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; 5272 5273 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); 5274 5275 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, 5276 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt); 5277 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); 5278 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, 5279 DAG.getConstant(VTBits, dl, MVT::i32)); 5280 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); 5281 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 5282 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); 5283 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5284 ISD::SETGE, ARMcc, DAG, dl); 5285 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, 5286 ARMcc, CCR, CmpLo); 5287 5288 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); 5289 SDValue HiBigShift = Opc == ISD::SRA 5290 ? DAG.getNode(Opc, dl, VT, ShOpHi, 5291 DAG.getConstant(VTBits - 1, dl, VT)) 5292 : DAG.getConstant(0, dl, VT); 5293 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5294 ISD::SETGE, ARMcc, DAG, dl); 5295 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, 5296 ARMcc, CCR, CmpHi); 5297 5298 SDValue Ops[2] = { Lo, Hi }; 5299 return DAG.getMergeValues(Ops, dl); 5300 } 5301 5302 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two 5303 /// i32 values and take a 2 x i32 value to shift plus a shift amount. 5304 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op, 5305 SelectionDAG &DAG) const { 5306 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 5307 EVT VT = Op.getValueType(); 5308 unsigned VTBits = VT.getSizeInBits(); 5309 SDLoc dl(Op); 5310 SDValue ShOpLo = Op.getOperand(0); 5311 SDValue ShOpHi = Op.getOperand(1); 5312 SDValue ShAmt = Op.getOperand(2); 5313 SDValue ARMcc; 5314 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 5315 5316 assert(Op.getOpcode() == ISD::SHL_PARTS); 5317 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, 5318 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt); 5319 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); 5320 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); 5321 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 5322 5323 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, 5324 DAG.getConstant(VTBits, dl, MVT::i32)); 5325 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); 5326 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5327 ISD::SETGE, ARMcc, DAG, dl); 5328 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, 5329 ARMcc, CCR, CmpHi); 5330 5331 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5332 ISD::SETGE, ARMcc, DAG, dl); 5333 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 5334 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, 5335 DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo); 5336 5337 SDValue Ops[2] = { Lo, Hi }; 5338 return DAG.getMergeValues(Ops, dl); 5339 } 5340 5341 SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 5342 SelectionDAG &DAG) const { 5343 // The rounding mode is in bits 23:22 of the FPSCR. 5344 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0 5345 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3) 5346 // so that the shift + and get folded into a bitfield extract. 5347 SDLoc dl(Op); 5348 SDValue Ops[] = { DAG.getEntryNode(), 5349 DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32) }; 5350 5351 SDValue FPSCR = DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, MVT::i32, Ops); 5352 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR, 5353 DAG.getConstant(1U << 22, dl, MVT::i32)); 5354 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds, 5355 DAG.getConstant(22, dl, MVT::i32)); 5356 return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE, 5357 DAG.getConstant(3, dl, MVT::i32)); 5358 } 5359 5360 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, 5361 const ARMSubtarget *ST) { 5362 SDLoc dl(N); 5363 EVT VT = N->getValueType(0); 5364 if (VT.isVector()) { 5365 assert(ST->hasNEON()); 5366 5367 // Compute the least significant set bit: LSB = X & -X 5368 SDValue X = N->getOperand(0); 5369 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X); 5370 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX); 5371 5372 EVT ElemTy = VT.getVectorElementType(); 5373 5374 if (ElemTy == MVT::i8) { 5375 // Compute with: cttz(x) = ctpop(lsb - 1) 5376 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5377 DAG.getTargetConstant(1, dl, ElemTy)); 5378 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); 5379 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); 5380 } 5381 5382 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) && 5383 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) { 5384 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0 5385 unsigned NumBits = ElemTy.getSizeInBits(); 5386 SDValue WidthMinus1 = 5387 DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5388 DAG.getTargetConstant(NumBits - 1, dl, ElemTy)); 5389 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB); 5390 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ); 5391 } 5392 5393 // Compute with: cttz(x) = ctpop(lsb - 1) 5394 5395 // Since we can only compute the number of bits in a byte with vcnt.8, we 5396 // have to gather the result with pairwise addition (vpaddl) for i16, i32, 5397 // and i64. 5398 5399 // Compute LSB - 1. 5400 SDValue Bits; 5401 if (ElemTy == MVT::i64) { 5402 // Load constant 0xffff'ffff'ffff'ffff to register. 5403 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5404 DAG.getTargetConstant(0x1eff, dl, MVT::i32)); 5405 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF); 5406 } else { 5407 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5408 DAG.getTargetConstant(1, dl, ElemTy)); 5409 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); 5410 } 5411 5412 // Count #bits with vcnt.8. 5413 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; 5414 SDValue BitsVT8 = DAG.getNode(ISD::BITCAST, dl, VT8Bit, Bits); 5415 SDValue Cnt8 = DAG.getNode(ISD::CTPOP, dl, VT8Bit, BitsVT8); 5416 5417 // Gather the #bits with vpaddl (pairwise add.) 5418 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16; 5419 SDValue Cnt16 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT16Bit, 5420 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32), 5421 Cnt8); 5422 if (ElemTy == MVT::i16) 5423 return Cnt16; 5424 5425 EVT VT32Bit = VT.is64BitVector() ? MVT::v2i32 : MVT::v4i32; 5426 SDValue Cnt32 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT32Bit, 5427 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32), 5428 Cnt16); 5429 if (ElemTy == MVT::i32) 5430 return Cnt32; 5431 5432 assert(ElemTy == MVT::i64); 5433 SDValue Cnt64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 5434 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32), 5435 Cnt32); 5436 return Cnt64; 5437 } 5438 5439 if (!ST->hasV6T2Ops()) 5440 return SDValue(); 5441 5442 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); 5443 return DAG.getNode(ISD::CTLZ, dl, VT, rbit); 5444 } 5445 5446 /// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count 5447 /// for each 16-bit element from operand, repeated. The basic idea is to 5448 /// leverage vcnt to get the 8-bit counts, gather and add the results. 5449 /// 5450 /// Trace for v4i16: 5451 /// input = [v0 v1 v2 v3 ] (vi 16-bit element) 5452 /// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element) 5453 /// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi) 5454 /// vrev: N2 = [b1 b0 b3 b2 b5 b4 b7 b6] 5455 /// [b0 b1 b2 b3 b4 b5 b6 b7] 5456 /// +[b1 b0 b3 b2 b5 b4 b7 b6] 5457 /// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0, 5458 /// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits) 5459 static SDValue getCTPOP16BitCounts(SDNode *N, SelectionDAG &DAG) { 5460 EVT VT = N->getValueType(0); 5461 SDLoc DL(N); 5462 5463 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; 5464 SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0)); 5465 SDValue N1 = DAG.getNode(ISD::CTPOP, DL, VT8Bit, N0); 5466 SDValue N2 = DAG.getNode(ARMISD::VREV16, DL, VT8Bit, N1); 5467 SDValue N3 = DAG.getNode(ISD::ADD, DL, VT8Bit, N1, N2); 5468 return DAG.getNode(ARMISD::VUZP, DL, VT8Bit, N3, N3); 5469 } 5470 5471 /// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the 5472 /// bit-count for each 16-bit element from the operand. We need slightly 5473 /// different sequencing for v4i16 and v8i16 to stay within NEON's available 5474 /// 64/128-bit registers. 5475 /// 5476 /// Trace for v4i16: 5477 /// input = [v0 v1 v2 v3 ] (vi 16-bit element) 5478 /// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi) 5479 /// v8i16:Extended = [k0 k1 k2 k3 k0 k1 k2 k3 ] 5480 /// v4i16:Extracted = [k0 k1 k2 k3 ] 5481 static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) { 5482 EVT VT = N->getValueType(0); 5483 SDLoc DL(N); 5484 5485 SDValue BitCounts = getCTPOP16BitCounts(N, DAG); 5486 if (VT.is64BitVector()) { 5487 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts); 5488 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended, 5489 DAG.getIntPtrConstant(0, DL)); 5490 } else { 5491 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, 5492 BitCounts, DAG.getIntPtrConstant(0, DL)); 5493 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted); 5494 } 5495 } 5496 5497 /// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the 5498 /// bit-count for each 32-bit element from the operand. The idea here is 5499 /// to split the vector into 16-bit elements, leverage the 16-bit count 5500 /// routine, and then combine the results. 5501 /// 5502 /// Trace for v2i32 (v4i32 similar with Extracted/Extended exchanged): 5503 /// input = [v0 v1 ] (vi: 32-bit elements) 5504 /// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1]) 5505 /// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi) 5506 /// vrev: N0 = [k1 k0 k3 k2 ] 5507 /// [k0 k1 k2 k3 ] 5508 /// N1 =+[k1 k0 k3 k2 ] 5509 /// [k0 k2 k1 k3 ] 5510 /// N2 =+[k1 k3 k0 k2 ] 5511 /// [k0 k2 k1 k3 ] 5512 /// Extended =+[k1 k3 k0 k2 ] 5513 /// [k0 k2 ] 5514 /// Extracted=+[k1 k3 ] 5515 /// 5516 static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) { 5517 EVT VT = N->getValueType(0); 5518 SDLoc DL(N); 5519 5520 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16; 5521 5522 SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0)); 5523 SDValue Counts16 = lowerCTPOP16BitElements(Bitcast.getNode(), DAG); 5524 SDValue N0 = DAG.getNode(ARMISD::VREV32, DL, VT16Bit, Counts16); 5525 SDValue N1 = DAG.getNode(ISD::ADD, DL, VT16Bit, Counts16, N0); 5526 SDValue N2 = DAG.getNode(ARMISD::VUZP, DL, VT16Bit, N1, N1); 5527 5528 if (VT.is64BitVector()) { 5529 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2); 5530 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended, 5531 DAG.getIntPtrConstant(0, DL)); 5532 } else { 5533 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2, 5534 DAG.getIntPtrConstant(0, DL)); 5535 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted); 5536 } 5537 } 5538 5539 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG, 5540 const ARMSubtarget *ST) { 5541 EVT VT = N->getValueType(0); 5542 5543 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON."); 5544 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || 5545 VT == MVT::v4i16 || VT == MVT::v8i16) && 5546 "Unexpected type for custom ctpop lowering"); 5547 5548 if (VT.getVectorElementType() == MVT::i32) 5549 return lowerCTPOP32BitElements(N, DAG); 5550 else 5551 return lowerCTPOP16BitElements(N, DAG); 5552 } 5553 5554 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, 5555 const ARMSubtarget *ST) { 5556 EVT VT = N->getValueType(0); 5557 SDLoc dl(N); 5558 5559 if (!VT.isVector()) 5560 return SDValue(); 5561 5562 // Lower vector shifts on NEON to use VSHL. 5563 assert(ST->hasNEON() && "unexpected vector shift"); 5564 5565 // Left shifts translate directly to the vshiftu intrinsic. 5566 if (N->getOpcode() == ISD::SHL) 5567 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 5568 DAG.getConstant(Intrinsic::arm_neon_vshiftu, dl, 5569 MVT::i32), 5570 N->getOperand(0), N->getOperand(1)); 5571 5572 assert((N->getOpcode() == ISD::SRA || 5573 N->getOpcode() == ISD::SRL) && "unexpected vector shift opcode"); 5574 5575 // NEON uses the same intrinsics for both left and right shifts. For 5576 // right shifts, the shift amounts are negative, so negate the vector of 5577 // shift amounts. 5578 EVT ShiftVT = N->getOperand(1).getValueType(); 5579 SDValue NegatedCount = DAG.getNode(ISD::SUB, dl, ShiftVT, 5580 getZeroVector(ShiftVT, DAG, dl), 5581 N->getOperand(1)); 5582 Intrinsic::ID vshiftInt = (N->getOpcode() == ISD::SRA ? 5583 Intrinsic::arm_neon_vshifts : 5584 Intrinsic::arm_neon_vshiftu); 5585 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 5586 DAG.getConstant(vshiftInt, dl, MVT::i32), 5587 N->getOperand(0), NegatedCount); 5588 } 5589 5590 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, 5591 const ARMSubtarget *ST) { 5592 EVT VT = N->getValueType(0); 5593 SDLoc dl(N); 5594 5595 // We can get here for a node like i32 = ISD::SHL i32, i64 5596 if (VT != MVT::i64) 5597 return SDValue(); 5598 5599 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && 5600 "Unknown shift to lower!"); 5601 5602 // We only lower SRA, SRL of 1 here, all others use generic lowering. 5603 if (!isOneConstant(N->getOperand(1))) 5604 return SDValue(); 5605 5606 // If we are in thumb mode, we don't have RRX. 5607 if (ST->isThumb1Only()) return SDValue(); 5608 5609 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr. 5610 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0), 5611 DAG.getConstant(0, dl, MVT::i32)); 5612 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0), 5613 DAG.getConstant(1, dl, MVT::i32)); 5614 5615 // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and 5616 // captures the result into a carry flag. 5617 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG; 5618 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Glue), Hi); 5619 5620 // The low part is an ARMISD::RRX operand, which shifts the carry in. 5621 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1)); 5622 5623 // Merge the pieces into a single i64 value. 5624 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 5625 } 5626 5627 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) { 5628 SDValue TmpOp0, TmpOp1; 5629 bool Invert = false; 5630 bool Swap = false; 5631 unsigned Opc = 0; 5632 5633 SDValue Op0 = Op.getOperand(0); 5634 SDValue Op1 = Op.getOperand(1); 5635 SDValue CC = Op.getOperand(2); 5636 EVT CmpVT = Op0.getValueType().changeVectorElementTypeToInteger(); 5637 EVT VT = Op.getValueType(); 5638 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 5639 SDLoc dl(Op); 5640 5641 if (Op0.getValueType().getVectorElementType() == MVT::i64 && 5642 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) { 5643 // Special-case integer 64-bit equality comparisons. They aren't legal, 5644 // but they can be lowered with a few vector instructions. 5645 unsigned CmpElements = CmpVT.getVectorNumElements() * 2; 5646 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements); 5647 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0); 5648 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1); 5649 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1, 5650 DAG.getCondCode(ISD::SETEQ)); 5651 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp); 5652 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed); 5653 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged); 5654 if (SetCCOpcode == ISD::SETNE) 5655 Merged = DAG.getNOT(dl, Merged, CmpVT); 5656 Merged = DAG.getSExtOrTrunc(Merged, dl, VT); 5657 return Merged; 5658 } 5659 5660 if (CmpVT.getVectorElementType() == MVT::i64) 5661 // 64-bit comparisons are not legal in general. 5662 return SDValue(); 5663 5664 if (Op1.getValueType().isFloatingPoint()) { 5665 switch (SetCCOpcode) { 5666 default: llvm_unreachable("Illegal FP comparison"); 5667 case ISD::SETUNE: 5668 case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH; 5669 case ISD::SETOEQ: 5670 case ISD::SETEQ: Opc = ARMISD::VCEQ; break; 5671 case ISD::SETOLT: 5672 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH; 5673 case ISD::SETOGT: 5674 case ISD::SETGT: Opc = ARMISD::VCGT; break; 5675 case ISD::SETOLE: 5676 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH; 5677 case ISD::SETOGE: 5678 case ISD::SETGE: Opc = ARMISD::VCGE; break; 5679 case ISD::SETUGE: Swap = true; LLVM_FALLTHROUGH; 5680 case ISD::SETULE: Invert = true; Opc = ARMISD::VCGT; break; 5681 case ISD::SETUGT: Swap = true; LLVM_FALLTHROUGH; 5682 case ISD::SETULT: Invert = true; Opc = ARMISD::VCGE; break; 5683 case ISD::SETUEQ: Invert = true; LLVM_FALLTHROUGH; 5684 case ISD::SETONE: 5685 // Expand this to (OLT | OGT). 5686 TmpOp0 = Op0; 5687 TmpOp1 = Op1; 5688 Opc = ISD::OR; 5689 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0); 5690 Op1 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp0, TmpOp1); 5691 break; 5692 case ISD::SETUO: 5693 Invert = true; 5694 LLVM_FALLTHROUGH; 5695 case ISD::SETO: 5696 // Expand this to (OLT | OGE). 5697 TmpOp0 = Op0; 5698 TmpOp1 = Op1; 5699 Opc = ISD::OR; 5700 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0); 5701 Op1 = DAG.getNode(ARMISD::VCGE, dl, CmpVT, TmpOp0, TmpOp1); 5702 break; 5703 } 5704 } else { 5705 // Integer comparisons. 5706 switch (SetCCOpcode) { 5707 default: llvm_unreachable("Illegal integer comparison"); 5708 case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH; 5709 case ISD::SETEQ: Opc = ARMISD::VCEQ; break; 5710 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH; 5711 case ISD::SETGT: Opc = ARMISD::VCGT; break; 5712 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH; 5713 case ISD::SETGE: Opc = ARMISD::VCGE; break; 5714 case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH; 5715 case ISD::SETUGT: Opc = ARMISD::VCGTU; break; 5716 case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH; 5717 case ISD::SETUGE: Opc = ARMISD::VCGEU; break; 5718 } 5719 5720 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero). 5721 if (Opc == ARMISD::VCEQ) { 5722 SDValue AndOp; 5723 if (ISD::isBuildVectorAllZeros(Op1.getNode())) 5724 AndOp = Op0; 5725 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) 5726 AndOp = Op1; 5727 5728 // Ignore bitconvert. 5729 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST) 5730 AndOp = AndOp.getOperand(0); 5731 5732 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) { 5733 Opc = ARMISD::VTST; 5734 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0)); 5735 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1)); 5736 Invert = !Invert; 5737 } 5738 } 5739 } 5740 5741 if (Swap) 5742 std::swap(Op0, Op1); 5743 5744 // If one of the operands is a constant vector zero, attempt to fold the 5745 // comparison to a specialized compare-against-zero form. 5746 SDValue SingleOp; 5747 if (ISD::isBuildVectorAllZeros(Op1.getNode())) 5748 SingleOp = Op0; 5749 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) { 5750 if (Opc == ARMISD::VCGE) 5751 Opc = ARMISD::VCLEZ; 5752 else if (Opc == ARMISD::VCGT) 5753 Opc = ARMISD::VCLTZ; 5754 SingleOp = Op1; 5755 } 5756 5757 SDValue Result; 5758 if (SingleOp.getNode()) { 5759 switch (Opc) { 5760 case ARMISD::VCEQ: 5761 Result = DAG.getNode(ARMISD::VCEQZ, dl, CmpVT, SingleOp); break; 5762 case ARMISD::VCGE: 5763 Result = DAG.getNode(ARMISD::VCGEZ, dl, CmpVT, SingleOp); break; 5764 case ARMISD::VCLEZ: 5765 Result = DAG.getNode(ARMISD::VCLEZ, dl, CmpVT, SingleOp); break; 5766 case ARMISD::VCGT: 5767 Result = DAG.getNode(ARMISD::VCGTZ, dl, CmpVT, SingleOp); break; 5768 case ARMISD::VCLTZ: 5769 Result = DAG.getNode(ARMISD::VCLTZ, dl, CmpVT, SingleOp); break; 5770 default: 5771 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1); 5772 } 5773 } else { 5774 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1); 5775 } 5776 5777 Result = DAG.getSExtOrTrunc(Result, dl, VT); 5778 5779 if (Invert) 5780 Result = DAG.getNOT(dl, Result, VT); 5781 5782 return Result; 5783 } 5784 5785 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) { 5786 SDValue LHS = Op.getOperand(0); 5787 SDValue RHS = Op.getOperand(1); 5788 SDValue Carry = Op.getOperand(2); 5789 SDValue Cond = Op.getOperand(3); 5790 SDLoc DL(Op); 5791 5792 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only."); 5793 5794 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we 5795 // have to invert the carry first. 5796 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, 5797 DAG.getConstant(1, DL, MVT::i32), Carry); 5798 // This converts the boolean value carry into the carry flag. 5799 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG); 5800 5801 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); 5802 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry); 5803 5804 SDValue FVal = DAG.getConstant(0, DL, MVT::i32); 5805 SDValue TVal = DAG.getConstant(1, DL, MVT::i32); 5806 SDValue ARMcc = DAG.getConstant( 5807 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32); 5808 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 5809 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR, 5810 Cmp.getValue(1), SDValue()); 5811 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc, 5812 CCR, Chain.getValue(1)); 5813 } 5814 5815 /// isNEONModifiedImm - Check if the specified splat value corresponds to a 5816 /// valid vector constant for a NEON instruction with a "modified immediate" 5817 /// operand (e.g., VMOV). If so, return the encoded value. 5818 static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, 5819 unsigned SplatBitSize, SelectionDAG &DAG, 5820 const SDLoc &dl, EVT &VT, bool is128Bits, 5821 NEONModImmType type) { 5822 unsigned OpCmode, Imm; 5823 5824 // SplatBitSize is set to the smallest size that splats the vector, so a 5825 // zero vector will always have SplatBitSize == 8. However, NEON modified 5826 // immediate instructions others than VMOV do not support the 8-bit encoding 5827 // of a zero vector, and the default encoding of zero is supposed to be the 5828 // 32-bit version. 5829 if (SplatBits == 0) 5830 SplatBitSize = 32; 5831 5832 switch (SplatBitSize) { 5833 case 8: 5834 if (type != VMOVModImm) 5835 return SDValue(); 5836 // Any 1-byte value is OK. Op=0, Cmode=1110. 5837 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big"); 5838 OpCmode = 0xe; 5839 Imm = SplatBits; 5840 VT = is128Bits ? MVT::v16i8 : MVT::v8i8; 5841 break; 5842 5843 case 16: 5844 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero. 5845 VT = is128Bits ? MVT::v8i16 : MVT::v4i16; 5846 if ((SplatBits & ~0xff) == 0) { 5847 // Value = 0x00nn: Op=x, Cmode=100x. 5848 OpCmode = 0x8; 5849 Imm = SplatBits; 5850 break; 5851 } 5852 if ((SplatBits & ~0xff00) == 0) { 5853 // Value = 0xnn00: Op=x, Cmode=101x. 5854 OpCmode = 0xa; 5855 Imm = SplatBits >> 8; 5856 break; 5857 } 5858 return SDValue(); 5859 5860 case 32: 5861 // NEON's 32-bit VMOV supports splat values where: 5862 // * only one byte is nonzero, or 5863 // * the least significant byte is 0xff and the second byte is nonzero, or 5864 // * the least significant 2 bytes are 0xff and the third is nonzero. 5865 VT = is128Bits ? MVT::v4i32 : MVT::v2i32; 5866 if ((SplatBits & ~0xff) == 0) { 5867 // Value = 0x000000nn: Op=x, Cmode=000x. 5868 OpCmode = 0; 5869 Imm = SplatBits; 5870 break; 5871 } 5872 if ((SplatBits & ~0xff00) == 0) { 5873 // Value = 0x0000nn00: Op=x, Cmode=001x. 5874 OpCmode = 0x2; 5875 Imm = SplatBits >> 8; 5876 break; 5877 } 5878 if ((SplatBits & ~0xff0000) == 0) { 5879 // Value = 0x00nn0000: Op=x, Cmode=010x. 5880 OpCmode = 0x4; 5881 Imm = SplatBits >> 16; 5882 break; 5883 } 5884 if ((SplatBits & ~0xff000000) == 0) { 5885 // Value = 0xnn000000: Op=x, Cmode=011x. 5886 OpCmode = 0x6; 5887 Imm = SplatBits >> 24; 5888 break; 5889 } 5890 5891 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC 5892 if (type == OtherModImm) return SDValue(); 5893 5894 if ((SplatBits & ~0xffff) == 0 && 5895 ((SplatBits | SplatUndef) & 0xff) == 0xff) { 5896 // Value = 0x0000nnff: Op=x, Cmode=1100. 5897 OpCmode = 0xc; 5898 Imm = SplatBits >> 8; 5899 break; 5900 } 5901 5902 if ((SplatBits & ~0xffffff) == 0 && 5903 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) { 5904 // Value = 0x00nnffff: Op=x, Cmode=1101. 5905 OpCmode = 0xd; 5906 Imm = SplatBits >> 16; 5907 break; 5908 } 5909 5910 // Note: there are a few 32-bit splat values (specifically: 00ffff00, 5911 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not 5912 // VMOV.I32. A (very) minor optimization would be to replicate the value 5913 // and fall through here to test for a valid 64-bit splat. But, then the 5914 // caller would also need to check and handle the change in size. 5915 return SDValue(); 5916 5917 case 64: { 5918 if (type != VMOVModImm) 5919 return SDValue(); 5920 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff. 5921 uint64_t BitMask = 0xff; 5922 uint64_t Val = 0; 5923 unsigned ImmMask = 1; 5924 Imm = 0; 5925 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) { 5926 if (((SplatBits | SplatUndef) & BitMask) == BitMask) { 5927 Val |= BitMask; 5928 Imm |= ImmMask; 5929 } else if ((SplatBits & BitMask) != 0) { 5930 return SDValue(); 5931 } 5932 BitMask <<= 8; 5933 ImmMask <<= 1; 5934 } 5935 5936 if (DAG.getDataLayout().isBigEndian()) 5937 // swap higher and lower 32 bit word 5938 Imm = ((Imm & 0xf) << 4) | ((Imm & 0xf0) >> 4); 5939 5940 // Op=1, Cmode=1110. 5941 OpCmode = 0x1e; 5942 VT = is128Bits ? MVT::v2i64 : MVT::v1i64; 5943 break; 5944 } 5945 5946 default: 5947 llvm_unreachable("unexpected size for isNEONModifiedImm"); 5948 } 5949 5950 unsigned EncodedVal = ARM_AM::createNEONModImm(OpCmode, Imm); 5951 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32); 5952 } 5953 5954 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG, 5955 const ARMSubtarget *ST) const { 5956 EVT VT = Op.getValueType(); 5957 bool IsDouble = (VT == MVT::f64); 5958 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op); 5959 const APFloat &FPVal = CFP->getValueAPF(); 5960 5961 // Prevent floating-point constants from using literal loads 5962 // when execute-only is enabled. 5963 if (ST->genExecuteOnly()) { 5964 // If we can represent the constant as an immediate, don't lower it 5965 if (isFPImmLegal(FPVal, VT)) 5966 return Op; 5967 // Otherwise, construct as integer, and move to float register 5968 APInt INTVal = FPVal.bitcastToAPInt(); 5969 SDLoc DL(CFP); 5970 switch (VT.getSimpleVT().SimpleTy) { 5971 default: 5972 llvm_unreachable("Unknown floating point type!"); 5973 break; 5974 case MVT::f64: { 5975 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32); 5976 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32); 5977 if (!ST->isLittle()) 5978 std::swap(Lo, Hi); 5979 return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi); 5980 } 5981 case MVT::f32: 5982 return DAG.getNode(ARMISD::VMOVSR, DL, VT, 5983 DAG.getConstant(INTVal, DL, MVT::i32)); 5984 } 5985 } 5986 5987 if (!ST->hasVFP3()) 5988 return SDValue(); 5989 5990 // Use the default (constant pool) lowering for double constants when we have 5991 // an SP-only FPU 5992 if (IsDouble && Subtarget->isFPOnlySP()) 5993 return SDValue(); 5994 5995 // Try splatting with a VMOV.f32... 5996 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal); 5997 5998 if (ImmVal != -1) { 5999 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) { 6000 // We have code in place to select a valid ConstantFP already, no need to 6001 // do any mangling. 6002 return Op; 6003 } 6004 6005 // It's a float and we are trying to use NEON operations where 6006 // possible. Lower it to a splat followed by an extract. 6007 SDLoc DL(Op); 6008 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32); 6009 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32, 6010 NewVal); 6011 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant, 6012 DAG.getConstant(0, DL, MVT::i32)); 6013 } 6014 6015 // The rest of our options are NEON only, make sure that's allowed before 6016 // proceeding.. 6017 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP())) 6018 return SDValue(); 6019 6020 EVT VMovVT; 6021 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue(); 6022 6023 // It wouldn't really be worth bothering for doubles except for one very 6024 // important value, which does happen to match: 0.0. So make sure we don't do 6025 // anything stupid. 6026 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32)) 6027 return SDValue(); 6028 6029 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too). 6030 SDValue NewVal = isNEONModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), 6031 VMovVT, false, VMOVModImm); 6032 if (NewVal != SDValue()) { 6033 SDLoc DL(Op); 6034 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT, 6035 NewVal); 6036 if (IsDouble) 6037 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant); 6038 6039 // It's a float: cast and extract a vector element. 6040 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, 6041 VecConstant); 6042 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant, 6043 DAG.getConstant(0, DL, MVT::i32)); 6044 } 6045 6046 // Finally, try a VMVN.i32 6047 NewVal = isNEONModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT, 6048 false, VMVNModImm); 6049 if (NewVal != SDValue()) { 6050 SDLoc DL(Op); 6051 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal); 6052 6053 if (IsDouble) 6054 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant); 6055 6056 // It's a float: cast and extract a vector element. 6057 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, 6058 VecConstant); 6059 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant, 6060 DAG.getConstant(0, DL, MVT::i32)); 6061 } 6062 6063 return SDValue(); 6064 } 6065 6066 // check if an VEXT instruction can handle the shuffle mask when the 6067 // vector sources of the shuffle are the same. 6068 static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { 6069 unsigned NumElts = VT.getVectorNumElements(); 6070 6071 // Assume that the first shuffle index is not UNDEF. Fail if it is. 6072 if (M[0] < 0) 6073 return false; 6074 6075 Imm = M[0]; 6076 6077 // If this is a VEXT shuffle, the immediate value is the index of the first 6078 // element. The other shuffle indices must be the successive elements after 6079 // the first one. 6080 unsigned ExpectedElt = Imm; 6081 for (unsigned i = 1; i < NumElts; ++i) { 6082 // Increment the expected index. If it wraps around, just follow it 6083 // back to index zero and keep going. 6084 ++ExpectedElt; 6085 if (ExpectedElt == NumElts) 6086 ExpectedElt = 0; 6087 6088 if (M[i] < 0) continue; // ignore UNDEF indices 6089 if (ExpectedElt != static_cast<unsigned>(M[i])) 6090 return false; 6091 } 6092 6093 return true; 6094 } 6095 6096 static bool isVEXTMask(ArrayRef<int> M, EVT VT, 6097 bool &ReverseVEXT, unsigned &Imm) { 6098 unsigned NumElts = VT.getVectorNumElements(); 6099 ReverseVEXT = false; 6100 6101 // Assume that the first shuffle index is not UNDEF. Fail if it is. 6102 if (M[0] < 0) 6103 return false; 6104 6105 Imm = M[0]; 6106 6107 // If this is a VEXT shuffle, the immediate value is the index of the first 6108 // element. The other shuffle indices must be the successive elements after 6109 // the first one. 6110 unsigned ExpectedElt = Imm; 6111 for (unsigned i = 1; i < NumElts; ++i) { 6112 // Increment the expected index. If it wraps around, it may still be 6113 // a VEXT but the source vectors must be swapped. 6114 ExpectedElt += 1; 6115 if (ExpectedElt == NumElts * 2) { 6116 ExpectedElt = 0; 6117 ReverseVEXT = true; 6118 } 6119 6120 if (M[i] < 0) continue; // ignore UNDEF indices 6121 if (ExpectedElt != static_cast<unsigned>(M[i])) 6122 return false; 6123 } 6124 6125 // Adjust the index value if the source operands will be swapped. 6126 if (ReverseVEXT) 6127 Imm -= NumElts; 6128 6129 return true; 6130 } 6131 6132 /// isVREVMask - Check if a vector shuffle corresponds to a VREV 6133 /// instruction with the specified blocksize. (The order of the elements 6134 /// within each block of the vector is reversed.) 6135 static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { 6136 assert((BlockSize==16 || BlockSize==32 || BlockSize==64) && 6137 "Only possible block sizes for VREV are: 16, 32, 64"); 6138 6139 unsigned EltSz = VT.getScalarSizeInBits(); 6140 if (EltSz == 64) 6141 return false; 6142 6143 unsigned NumElts = VT.getVectorNumElements(); 6144 unsigned BlockElts = M[0] + 1; 6145 // If the first shuffle index is UNDEF, be optimistic. 6146 if (M[0] < 0) 6147 BlockElts = BlockSize / EltSz; 6148 6149 if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz) 6150 return false; 6151 6152 for (unsigned i = 0; i < NumElts; ++i) { 6153 if (M[i] < 0) continue; // ignore UNDEF indices 6154 if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts)) 6155 return false; 6156 } 6157 6158 return true; 6159 } 6160 6161 static bool isVTBLMask(ArrayRef<int> M, EVT VT) { 6162 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of 6163 // range, then 0 is placed into the resulting vector. So pretty much any mask 6164 // of 8 elements can work here. 6165 return VT == MVT::v8i8 && M.size() == 8; 6166 } 6167 6168 static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask, 6169 unsigned Index) { 6170 if (Mask.size() == Elements * 2) 6171 return Index / Elements; 6172 return Mask[Index] == 0 ? 0 : 1; 6173 } 6174 6175 // Checks whether the shuffle mask represents a vector transpose (VTRN) by 6176 // checking that pairs of elements in the shuffle mask represent the same index 6177 // in each vector, incrementing the expected index by 2 at each step. 6178 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6] 6179 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g} 6180 // v2={e,f,g,h} 6181 // WhichResult gives the offset for each element in the mask based on which 6182 // of the two results it belongs to. 6183 // 6184 // The transpose can be represented either as: 6185 // result1 = shufflevector v1, v2, result1_shuffle_mask 6186 // result2 = shufflevector v1, v2, result2_shuffle_mask 6187 // where v1/v2 and the shuffle masks have the same number of elements 6188 // (here WhichResult (see below) indicates which result is being checked) 6189 // 6190 // or as: 6191 // results = shufflevector v1, v2, shuffle_mask 6192 // where both results are returned in one vector and the shuffle mask has twice 6193 // as many elements as v1/v2 (here WhichResult will always be 0 if true) here we 6194 // want to check the low half and high half of the shuffle mask as if it were 6195 // the other case 6196 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 6197 unsigned EltSz = VT.getScalarSizeInBits(); 6198 if (EltSz == 64) 6199 return false; 6200 6201 unsigned NumElts = VT.getVectorNumElements(); 6202 if (M.size() != NumElts && M.size() != NumElts*2) 6203 return false; 6204 6205 // If the mask is twice as long as the input vector then we need to check the 6206 // upper and lower parts of the mask with a matching value for WhichResult 6207 // FIXME: A mask with only even values will be rejected in case the first 6208 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only 6209 // M[0] is used to determine WhichResult 6210 for (unsigned i = 0; i < M.size(); i += NumElts) { 6211 WhichResult = SelectPairHalf(NumElts, M, i); 6212 for (unsigned j = 0; j < NumElts; j += 2) { 6213 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) || 6214 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult)) 6215 return false; 6216 } 6217 } 6218 6219 if (M.size() == NumElts*2) 6220 WhichResult = 0; 6221 6222 return true; 6223 } 6224 6225 /// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of 6226 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 6227 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>. 6228 static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ 6229 unsigned EltSz = VT.getScalarSizeInBits(); 6230 if (EltSz == 64) 6231 return false; 6232 6233 unsigned NumElts = VT.getVectorNumElements(); 6234 if (M.size() != NumElts && M.size() != NumElts*2) 6235 return false; 6236 6237 for (unsigned i = 0; i < M.size(); i += NumElts) { 6238 WhichResult = SelectPairHalf(NumElts, M, i); 6239 for (unsigned j = 0; j < NumElts; j += 2) { 6240 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) || 6241 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult)) 6242 return false; 6243 } 6244 } 6245 6246 if (M.size() == NumElts*2) 6247 WhichResult = 0; 6248 6249 return true; 6250 } 6251 6252 // Checks whether the shuffle mask represents a vector unzip (VUZP) by checking 6253 // that the mask elements are either all even and in steps of size 2 or all odd 6254 // and in steps of size 2. 6255 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6] 6256 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g} 6257 // v2={e,f,g,h} 6258 // Requires similar checks to that of isVTRNMask with 6259 // respect the how results are returned. 6260 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 6261 unsigned EltSz = VT.getScalarSizeInBits(); 6262 if (EltSz == 64) 6263 return false; 6264 6265 unsigned NumElts = VT.getVectorNumElements(); 6266 if (M.size() != NumElts && M.size() != NumElts*2) 6267 return false; 6268 6269 for (unsigned i = 0; i < M.size(); i += NumElts) { 6270 WhichResult = SelectPairHalf(NumElts, M, i); 6271 for (unsigned j = 0; j < NumElts; ++j) { 6272 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult) 6273 return false; 6274 } 6275 } 6276 6277 if (M.size() == NumElts*2) 6278 WhichResult = 0; 6279 6280 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6281 if (VT.is64BitVector() && EltSz == 32) 6282 return false; 6283 6284 return true; 6285 } 6286 6287 /// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of 6288 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 6289 /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>, 6290 static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ 6291 unsigned EltSz = VT.getScalarSizeInBits(); 6292 if (EltSz == 64) 6293 return false; 6294 6295 unsigned NumElts = VT.getVectorNumElements(); 6296 if (M.size() != NumElts && M.size() != NumElts*2) 6297 return false; 6298 6299 unsigned Half = NumElts / 2; 6300 for (unsigned i = 0; i < M.size(); i += NumElts) { 6301 WhichResult = SelectPairHalf(NumElts, M, i); 6302 for (unsigned j = 0; j < NumElts; j += Half) { 6303 unsigned Idx = WhichResult; 6304 for (unsigned k = 0; k < Half; ++k) { 6305 int MIdx = M[i + j + k]; 6306 if (MIdx >= 0 && (unsigned) MIdx != Idx) 6307 return false; 6308 Idx += 2; 6309 } 6310 } 6311 } 6312 6313 if (M.size() == NumElts*2) 6314 WhichResult = 0; 6315 6316 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6317 if (VT.is64BitVector() && EltSz == 32) 6318 return false; 6319 6320 return true; 6321 } 6322 6323 // Checks whether the shuffle mask represents a vector zip (VZIP) by checking 6324 // that pairs of elements of the shufflemask represent the same index in each 6325 // vector incrementing sequentially through the vectors. 6326 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5] 6327 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f} 6328 // v2={e,f,g,h} 6329 // Requires similar checks to that of isVTRNMask with respect the how results 6330 // are returned. 6331 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 6332 unsigned EltSz = VT.getScalarSizeInBits(); 6333 if (EltSz == 64) 6334 return false; 6335 6336 unsigned NumElts = VT.getVectorNumElements(); 6337 if (M.size() != NumElts && M.size() != NumElts*2) 6338 return false; 6339 6340 for (unsigned i = 0; i < M.size(); i += NumElts) { 6341 WhichResult = SelectPairHalf(NumElts, M, i); 6342 unsigned Idx = WhichResult * NumElts / 2; 6343 for (unsigned j = 0; j < NumElts; j += 2) { 6344 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) || 6345 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts)) 6346 return false; 6347 Idx += 1; 6348 } 6349 } 6350 6351 if (M.size() == NumElts*2) 6352 WhichResult = 0; 6353 6354 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6355 if (VT.is64BitVector() && EltSz == 32) 6356 return false; 6357 6358 return true; 6359 } 6360 6361 /// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of 6362 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 6363 /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>. 6364 static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ 6365 unsigned EltSz = VT.getScalarSizeInBits(); 6366 if (EltSz == 64) 6367 return false; 6368 6369 unsigned NumElts = VT.getVectorNumElements(); 6370 if (M.size() != NumElts && M.size() != NumElts*2) 6371 return false; 6372 6373 for (unsigned i = 0; i < M.size(); i += NumElts) { 6374 WhichResult = SelectPairHalf(NumElts, M, i); 6375 unsigned Idx = WhichResult * NumElts / 2; 6376 for (unsigned j = 0; j < NumElts; j += 2) { 6377 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) || 6378 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx)) 6379 return false; 6380 Idx += 1; 6381 } 6382 } 6383 6384 if (M.size() == NumElts*2) 6385 WhichResult = 0; 6386 6387 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6388 if (VT.is64BitVector() && EltSz == 32) 6389 return false; 6390 6391 return true; 6392 } 6393 6394 /// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), 6395 /// and return the corresponding ARMISD opcode if it is, or 0 if it isn't. 6396 static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT, 6397 unsigned &WhichResult, 6398 bool &isV_UNDEF) { 6399 isV_UNDEF = false; 6400 if (isVTRNMask(ShuffleMask, VT, WhichResult)) 6401 return ARMISD::VTRN; 6402 if (isVUZPMask(ShuffleMask, VT, WhichResult)) 6403 return ARMISD::VUZP; 6404 if (isVZIPMask(ShuffleMask, VT, WhichResult)) 6405 return ARMISD::VZIP; 6406 6407 isV_UNDEF = true; 6408 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) 6409 return ARMISD::VTRN; 6410 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) 6411 return ARMISD::VUZP; 6412 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) 6413 return ARMISD::VZIP; 6414 6415 return 0; 6416 } 6417 6418 /// \return true if this is a reverse operation on an vector. 6419 static bool isReverseMask(ArrayRef<int> M, EVT VT) { 6420 unsigned NumElts = VT.getVectorNumElements(); 6421 // Make sure the mask has the right size. 6422 if (NumElts != M.size()) 6423 return false; 6424 6425 // Look for <15, ..., 3, -1, 1, 0>. 6426 for (unsigned i = 0; i != NumElts; ++i) 6427 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i)) 6428 return false; 6429 6430 return true; 6431 } 6432 6433 // If N is an integer constant that can be moved into a register in one 6434 // instruction, return an SDValue of such a constant (will become a MOV 6435 // instruction). Otherwise return null. 6436 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, 6437 const ARMSubtarget *ST, const SDLoc &dl) { 6438 uint64_t Val; 6439 if (!isa<ConstantSDNode>(N)) 6440 return SDValue(); 6441 Val = cast<ConstantSDNode>(N)->getZExtValue(); 6442 6443 if (ST->isThumb1Only()) { 6444 if (Val <= 255 || ~Val <= 255) 6445 return DAG.getConstant(Val, dl, MVT::i32); 6446 } else { 6447 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1) 6448 return DAG.getConstant(Val, dl, MVT::i32); 6449 } 6450 return SDValue(); 6451 } 6452 6453 // If this is a case we can't handle, return null and let the default 6454 // expansion code take care of it. 6455 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, 6456 const ARMSubtarget *ST) const { 6457 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); 6458 SDLoc dl(Op); 6459 EVT VT = Op.getValueType(); 6460 6461 APInt SplatBits, SplatUndef; 6462 unsigned SplatBitSize; 6463 bool HasAnyUndefs; 6464 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { 6465 if (SplatUndef.isAllOnesValue()) 6466 return DAG.getUNDEF(VT); 6467 6468 if (SplatBitSize <= 64) { 6469 // Check if an immediate VMOV works. 6470 EVT VmovVT; 6471 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(), 6472 SplatUndef.getZExtValue(), SplatBitSize, 6473 DAG, dl, VmovVT, VT.is128BitVector(), 6474 VMOVModImm); 6475 if (Val.getNode()) { 6476 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val); 6477 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); 6478 } 6479 6480 // Try an immediate VMVN. 6481 uint64_t NegatedImm = (~SplatBits).getZExtValue(); 6482 Val = isNEONModifiedImm(NegatedImm, 6483 SplatUndef.getZExtValue(), SplatBitSize, 6484 DAG, dl, VmovVT, VT.is128BitVector(), 6485 VMVNModImm); 6486 if (Val.getNode()) { 6487 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val); 6488 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); 6489 } 6490 6491 // Use vmov.f32 to materialize other v2f32 and v4f32 splats. 6492 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) { 6493 int ImmVal = ARM_AM::getFP32Imm(SplatBits); 6494 if (ImmVal != -1) { 6495 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32); 6496 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val); 6497 } 6498 } 6499 } 6500 } 6501 6502 // Scan through the operands to see if only one value is used. 6503 // 6504 // As an optimisation, even if more than one value is used it may be more 6505 // profitable to splat with one value then change some lanes. 6506 // 6507 // Heuristically we decide to do this if the vector has a "dominant" value, 6508 // defined as splatted to more than half of the lanes. 6509 unsigned NumElts = VT.getVectorNumElements(); 6510 bool isOnlyLowElement = true; 6511 bool usesOnlyOneValue = true; 6512 bool hasDominantValue = false; 6513 bool isConstant = true; 6514 6515 // Map of the number of times a particular SDValue appears in the 6516 // element list. 6517 DenseMap<SDValue, unsigned> ValueCounts; 6518 SDValue Value; 6519 for (unsigned i = 0; i < NumElts; ++i) { 6520 SDValue V = Op.getOperand(i); 6521 if (V.isUndef()) 6522 continue; 6523 if (i > 0) 6524 isOnlyLowElement = false; 6525 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 6526 isConstant = false; 6527 6528 ValueCounts.insert(std::make_pair(V, 0)); 6529 unsigned &Count = ValueCounts[V]; 6530 6531 // Is this value dominant? (takes up more than half of the lanes) 6532 if (++Count > (NumElts / 2)) { 6533 hasDominantValue = true; 6534 Value = V; 6535 } 6536 } 6537 if (ValueCounts.size() != 1) 6538 usesOnlyOneValue = false; 6539 if (!Value.getNode() && !ValueCounts.empty()) 6540 Value = ValueCounts.begin()->first; 6541 6542 if (ValueCounts.empty()) 6543 return DAG.getUNDEF(VT); 6544 6545 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR. 6546 // Keep going if we are hitting this case. 6547 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode())) 6548 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); 6549 6550 unsigned EltSize = VT.getScalarSizeInBits(); 6551 6552 // Use VDUP for non-constant splats. For f32 constant splats, reduce to 6553 // i32 and try again. 6554 if (hasDominantValue && EltSize <= 32) { 6555 if (!isConstant) { 6556 SDValue N; 6557 6558 // If we are VDUPing a value that comes directly from a vector, that will 6559 // cause an unnecessary move to and from a GPR, where instead we could 6560 // just use VDUPLANE. We can only do this if the lane being extracted 6561 // is at a constant index, as the VDUP from lane instructions only have 6562 // constant-index forms. 6563 ConstantSDNode *constIndex; 6564 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT && 6565 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) { 6566 // We need to create a new undef vector to use for the VDUPLANE if the 6567 // size of the vector from which we get the value is different than the 6568 // size of the vector that we need to create. We will insert the element 6569 // such that the register coalescer will remove unnecessary copies. 6570 if (VT != Value->getOperand(0).getValueType()) { 6571 unsigned index = constIndex->getAPIntValue().getLimitedValue() % 6572 VT.getVectorNumElements(); 6573 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, 6574 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT), 6575 Value, DAG.getConstant(index, dl, MVT::i32)), 6576 DAG.getConstant(index, dl, MVT::i32)); 6577 } else 6578 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, 6579 Value->getOperand(0), Value->getOperand(1)); 6580 } else 6581 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value); 6582 6583 if (!usesOnlyOneValue) { 6584 // The dominant value was splatted as 'N', but we now have to insert 6585 // all differing elements. 6586 for (unsigned I = 0; I < NumElts; ++I) { 6587 if (Op.getOperand(I) == Value) 6588 continue; 6589 SmallVector<SDValue, 3> Ops; 6590 Ops.push_back(N); 6591 Ops.push_back(Op.getOperand(I)); 6592 Ops.push_back(DAG.getConstant(I, dl, MVT::i32)); 6593 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops); 6594 } 6595 } 6596 return N; 6597 } 6598 if (VT.getVectorElementType().isFloatingPoint()) { 6599 SmallVector<SDValue, 8> Ops; 6600 for (unsigned i = 0; i < NumElts; ++i) 6601 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, MVT::i32, 6602 Op.getOperand(i))); 6603 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts); 6604 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops); 6605 Val = LowerBUILD_VECTOR(Val, DAG, ST); 6606 if (Val.getNode()) 6607 return DAG.getNode(ISD::BITCAST, dl, VT, Val); 6608 } 6609 if (usesOnlyOneValue) { 6610 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl); 6611 if (isConstant && Val.getNode()) 6612 return DAG.getNode(ARMISD::VDUP, dl, VT, Val); 6613 } 6614 } 6615 6616 // If all elements are constants and the case above didn't get hit, fall back 6617 // to the default expansion, which will generate a load from the constant 6618 // pool. 6619 if (isConstant) 6620 return SDValue(); 6621 6622 // Empirical tests suggest this is rarely worth it for vectors of length <= 2. 6623 if (NumElts >= 4) { 6624 SDValue shuffle = ReconstructShuffle(Op, DAG); 6625 if (shuffle != SDValue()) 6626 return shuffle; 6627 } 6628 6629 if (VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) { 6630 // If we haven't found an efficient lowering, try splitting a 128-bit vector 6631 // into two 64-bit vectors; we might discover a better way to lower it. 6632 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts); 6633 EVT ExtVT = VT.getVectorElementType(); 6634 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2); 6635 SDValue Lower = 6636 DAG.getBuildVector(HVT, dl, makeArrayRef(&Ops[0], NumElts / 2)); 6637 if (Lower.getOpcode() == ISD::BUILD_VECTOR) 6638 Lower = LowerBUILD_VECTOR(Lower, DAG, ST); 6639 SDValue Upper = DAG.getBuildVector( 6640 HVT, dl, makeArrayRef(&Ops[NumElts / 2], NumElts / 2)); 6641 if (Upper.getOpcode() == ISD::BUILD_VECTOR) 6642 Upper = LowerBUILD_VECTOR(Upper, DAG, ST); 6643 if (Lower && Upper) 6644 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper); 6645 } 6646 6647 // Vectors with 32- or 64-bit elements can be built by directly assigning 6648 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands 6649 // will be legalized. 6650 if (EltSize >= 32) { 6651 // Do the expansion with floating-point types, since that is what the VFP 6652 // registers are defined to use, and since i64 is not legal. 6653 EVT EltVT = EVT::getFloatingPointVT(EltSize); 6654 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts); 6655 SmallVector<SDValue, 8> Ops; 6656 for (unsigned i = 0; i < NumElts; ++i) 6657 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i))); 6658 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops); 6659 return DAG.getNode(ISD::BITCAST, dl, VT, Val); 6660 } 6661 6662 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we 6663 // know the default expansion would otherwise fall back on something even 6664 // worse. For a vector with one or two non-undef values, that's 6665 // scalar_to_vector for the elements followed by a shuffle (provided the 6666 // shuffle is valid for the target) and materialization element by element 6667 // on the stack followed by a load for everything else. 6668 if (!isConstant && !usesOnlyOneValue) { 6669 SDValue Vec = DAG.getUNDEF(VT); 6670 for (unsigned i = 0 ; i < NumElts; ++i) { 6671 SDValue V = Op.getOperand(i); 6672 if (V.isUndef()) 6673 continue; 6674 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32); 6675 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); 6676 } 6677 return Vec; 6678 } 6679 6680 return SDValue(); 6681 } 6682 6683 // Gather data to see if the operation can be modelled as a 6684 // shuffle in combination with VEXTs. 6685 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op, 6686 SelectionDAG &DAG) const { 6687 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); 6688 SDLoc dl(Op); 6689 EVT VT = Op.getValueType(); 6690 unsigned NumElts = VT.getVectorNumElements(); 6691 6692 struct ShuffleSourceInfo { 6693 SDValue Vec; 6694 unsigned MinElt = std::numeric_limits<unsigned>::max(); 6695 unsigned MaxElt = 0; 6696 6697 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to 6698 // be compatible with the shuffle we intend to construct. As a result 6699 // ShuffleVec will be some sliding window into the original Vec. 6700 SDValue ShuffleVec; 6701 6702 // Code should guarantee that element i in Vec starts at element "WindowBase 6703 // + i * WindowScale in ShuffleVec". 6704 int WindowBase = 0; 6705 int WindowScale = 1; 6706 6707 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {} 6708 6709 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; } 6710 }; 6711 6712 // First gather all vectors used as an immediate source for this BUILD_VECTOR 6713 // node. 6714 SmallVector<ShuffleSourceInfo, 2> Sources; 6715 for (unsigned i = 0; i < NumElts; ++i) { 6716 SDValue V = Op.getOperand(i); 6717 if (V.isUndef()) 6718 continue; 6719 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) { 6720 // A shuffle can only come from building a vector from various 6721 // elements of other vectors. 6722 return SDValue(); 6723 } else if (!isa<ConstantSDNode>(V.getOperand(1))) { 6724 // Furthermore, shuffles require a constant mask, whereas extractelts 6725 // accept variable indices. 6726 return SDValue(); 6727 } 6728 6729 // Add this element source to the list if it's not already there. 6730 SDValue SourceVec = V.getOperand(0); 6731 auto Source = llvm::find(Sources, SourceVec); 6732 if (Source == Sources.end()) 6733 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec)); 6734 6735 // Update the minimum and maximum lane number seen. 6736 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue(); 6737 Source->MinElt = std::min(Source->MinElt, EltNo); 6738 Source->MaxElt = std::max(Source->MaxElt, EltNo); 6739 } 6740 6741 // Currently only do something sane when at most two source vectors 6742 // are involved. 6743 if (Sources.size() > 2) 6744 return SDValue(); 6745 6746 // Find out the smallest element size among result and two sources, and use 6747 // it as element size to build the shuffle_vector. 6748 EVT SmallestEltTy = VT.getVectorElementType(); 6749 for (auto &Source : Sources) { 6750 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType(); 6751 if (SrcEltTy.bitsLT(SmallestEltTy)) 6752 SmallestEltTy = SrcEltTy; 6753 } 6754 unsigned ResMultiplier = 6755 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits(); 6756 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); 6757 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts); 6758 6759 // If the source vector is too wide or too narrow, we may nevertheless be able 6760 // to construct a compatible shuffle either by concatenating it with UNDEF or 6761 // extracting a suitable range of elements. 6762 for (auto &Src : Sources) { 6763 EVT SrcVT = Src.ShuffleVec.getValueType(); 6764 6765 if (SrcVT.getSizeInBits() == VT.getSizeInBits()) 6766 continue; 6767 6768 // This stage of the search produces a source with the same element type as 6769 // the original, but with a total width matching the BUILD_VECTOR output. 6770 EVT EltVT = SrcVT.getVectorElementType(); 6771 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits(); 6772 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts); 6773 6774 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) { 6775 if (2 * SrcVT.getSizeInBits() != VT.getSizeInBits()) 6776 return SDValue(); 6777 // We can pad out the smaller vector for free, so if it's part of a 6778 // shuffle... 6779 Src.ShuffleVec = 6780 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, 6781 DAG.getUNDEF(Src.ShuffleVec.getValueType())); 6782 continue; 6783 } 6784 6785 if (SrcVT.getSizeInBits() != 2 * VT.getSizeInBits()) 6786 return SDValue(); 6787 6788 if (Src.MaxElt - Src.MinElt >= NumSrcElts) { 6789 // Span too large for a VEXT to cope 6790 return SDValue(); 6791 } 6792 6793 if (Src.MinElt >= NumSrcElts) { 6794 // The extraction can just take the second half 6795 Src.ShuffleVec = 6796 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6797 DAG.getConstant(NumSrcElts, dl, MVT::i32)); 6798 Src.WindowBase = -NumSrcElts; 6799 } else if (Src.MaxElt < NumSrcElts) { 6800 // The extraction can just take the first half 6801 Src.ShuffleVec = 6802 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6803 DAG.getConstant(0, dl, MVT::i32)); 6804 } else { 6805 // An actual VEXT is needed 6806 SDValue VEXTSrc1 = 6807 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6808 DAG.getConstant(0, dl, MVT::i32)); 6809 SDValue VEXTSrc2 = 6810 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6811 DAG.getConstant(NumSrcElts, dl, MVT::i32)); 6812 6813 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1, 6814 VEXTSrc2, 6815 DAG.getConstant(Src.MinElt, dl, MVT::i32)); 6816 Src.WindowBase = -Src.MinElt; 6817 } 6818 } 6819 6820 // Another possible incompatibility occurs from the vector element types. We 6821 // can fix this by bitcasting the source vectors to the same type we intend 6822 // for the shuffle. 6823 for (auto &Src : Sources) { 6824 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType(); 6825 if (SrcEltTy == SmallestEltTy) 6826 continue; 6827 assert(ShuffleVT.getVectorElementType() == SmallestEltTy); 6828 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); 6829 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits(); 6830 Src.WindowBase *= Src.WindowScale; 6831 } 6832 6833 // Final sanity check before we try to actually produce a shuffle. 6834 LLVM_DEBUG(for (auto Src 6835 : Sources) 6836 assert(Src.ShuffleVec.getValueType() == ShuffleVT);); 6837 6838 // The stars all align, our next step is to produce the mask for the shuffle. 6839 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1); 6840 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits(); 6841 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 6842 SDValue Entry = Op.getOperand(i); 6843 if (Entry.isUndef()) 6844 continue; 6845 6846 auto Src = llvm::find(Sources, Entry.getOperand(0)); 6847 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue(); 6848 6849 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit 6850 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this 6851 // segment. 6852 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType(); 6853 int BitsDefined = std::min(OrigEltTy.getSizeInBits(), 6854 VT.getScalarSizeInBits()); 6855 int LanesDefined = BitsDefined / BitsPerShuffleLane; 6856 6857 // This source is expected to fill ResMultiplier lanes of the final shuffle, 6858 // starting at the appropriate offset. 6859 int *LaneMask = &Mask[i * ResMultiplier]; 6860 6861 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase; 6862 ExtractBase += NumElts * (Src - Sources.begin()); 6863 for (int j = 0; j < LanesDefined; ++j) 6864 LaneMask[j] = ExtractBase + j; 6865 } 6866 6867 // Final check before we try to produce nonsense... 6868 if (!isShuffleMaskLegal(Mask, ShuffleVT)) 6869 return SDValue(); 6870 6871 // We can't handle more than two sources. This should have already 6872 // been checked before this point. 6873 assert(Sources.size() <= 2 && "Too many sources!"); 6874 6875 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) }; 6876 for (unsigned i = 0; i < Sources.size(); ++i) 6877 ShuffleOps[i] = Sources[i].ShuffleVec; 6878 6879 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0], 6880 ShuffleOps[1], Mask); 6881 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); 6882 } 6883 6884 /// isShuffleMaskLegal - Targets can use this to indicate that they only 6885 /// support *some* VECTOR_SHUFFLE operations, those with specific masks. 6886 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 6887 /// are assumed to be legal. 6888 bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { 6889 if (VT.getVectorNumElements() == 4 && 6890 (VT.is128BitVector() || VT.is64BitVector())) { 6891 unsigned PFIndexes[4]; 6892 for (unsigned i = 0; i != 4; ++i) { 6893 if (M[i] < 0) 6894 PFIndexes[i] = 8; 6895 else 6896 PFIndexes[i] = M[i]; 6897 } 6898 6899 // Compute the index in the perfect shuffle table. 6900 unsigned PFTableIndex = 6901 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 6902 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 6903 unsigned Cost = (PFEntry >> 30); 6904 6905 if (Cost <= 4) 6906 return true; 6907 } 6908 6909 bool ReverseVEXT, isV_UNDEF; 6910 unsigned Imm, WhichResult; 6911 6912 unsigned EltSize = VT.getScalarSizeInBits(); 6913 return (EltSize >= 32 || 6914 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || 6915 isVREVMask(M, VT, 64) || 6916 isVREVMask(M, VT, 32) || 6917 isVREVMask(M, VT, 16) || 6918 isVEXTMask(M, VT, ReverseVEXT, Imm) || 6919 isVTBLMask(M, VT) || 6920 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF) || 6921 ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(M, VT))); 6922 } 6923 6924 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 6925 /// the specified operations to build the shuffle. 6926 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 6927 SDValue RHS, SelectionDAG &DAG, 6928 const SDLoc &dl) { 6929 unsigned OpNum = (PFEntry >> 26) & 0x0F; 6930 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 6931 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 6932 6933 enum { 6934 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 6935 OP_VREV, 6936 OP_VDUP0, 6937 OP_VDUP1, 6938 OP_VDUP2, 6939 OP_VDUP3, 6940 OP_VEXT1, 6941 OP_VEXT2, 6942 OP_VEXT3, 6943 OP_VUZPL, // VUZP, left result 6944 OP_VUZPR, // VUZP, right result 6945 OP_VZIPL, // VZIP, left result 6946 OP_VZIPR, // VZIP, right result 6947 OP_VTRNL, // VTRN, left result 6948 OP_VTRNR // VTRN, right result 6949 }; 6950 6951 if (OpNum == OP_COPY) { 6952 if (LHSID == (1*9+2)*9+3) return LHS; 6953 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 6954 return RHS; 6955 } 6956 6957 SDValue OpLHS, OpRHS; 6958 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 6959 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 6960 EVT VT = OpLHS.getValueType(); 6961 6962 switch (OpNum) { 6963 default: llvm_unreachable("Unknown shuffle opcode!"); 6964 case OP_VREV: 6965 // VREV divides the vector in half and swaps within the half. 6966 if (VT.getVectorElementType() == MVT::i32 || 6967 VT.getVectorElementType() == MVT::f32) 6968 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS); 6969 // vrev <4 x i16> -> VREV32 6970 if (VT.getVectorElementType() == MVT::i16) 6971 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS); 6972 // vrev <4 x i8> -> VREV16 6973 assert(VT.getVectorElementType() == MVT::i8); 6974 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS); 6975 case OP_VDUP0: 6976 case OP_VDUP1: 6977 case OP_VDUP2: 6978 case OP_VDUP3: 6979 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, 6980 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32)); 6981 case OP_VEXT1: 6982 case OP_VEXT2: 6983 case OP_VEXT3: 6984 return DAG.getNode(ARMISD::VEXT, dl, VT, 6985 OpLHS, OpRHS, 6986 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32)); 6987 case OP_VUZPL: 6988 case OP_VUZPR: 6989 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT), 6990 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL); 6991 case OP_VZIPL: 6992 case OP_VZIPR: 6993 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT), 6994 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL); 6995 case OP_VTRNL: 6996 case OP_VTRNR: 6997 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT), 6998 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL); 6999 } 7000 } 7001 7002 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, 7003 ArrayRef<int> ShuffleMask, 7004 SelectionDAG &DAG) { 7005 // Check to see if we can use the VTBL instruction. 7006 SDValue V1 = Op.getOperand(0); 7007 SDValue V2 = Op.getOperand(1); 7008 SDLoc DL(Op); 7009 7010 SmallVector<SDValue, 8> VTBLMask; 7011 for (ArrayRef<int>::iterator 7012 I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I) 7013 VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32)); 7014 7015 if (V2.getNode()->isUndef()) 7016 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1, 7017 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask)); 7018 7019 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2, 7020 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask)); 7021 } 7022 7023 static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op, 7024 SelectionDAG &DAG) { 7025 SDLoc DL(Op); 7026 SDValue OpLHS = Op.getOperand(0); 7027 EVT VT = OpLHS.getValueType(); 7028 7029 assert((VT == MVT::v8i16 || VT == MVT::v16i8) && 7030 "Expect an v8i16/v16i8 type"); 7031 OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, OpLHS); 7032 // For a v16i8 type: After the VREV, we have got <8, ...15, 8, ..., 0>. Now, 7033 // extract the first 8 bytes into the top double word and the last 8 bytes 7034 // into the bottom double word. The v8i16 case is similar. 7035 unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4; 7036 return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS, 7037 DAG.getConstant(ExtractNum, DL, MVT::i32)); 7038 } 7039 7040 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { 7041 SDValue V1 = Op.getOperand(0); 7042 SDValue V2 = Op.getOperand(1); 7043 SDLoc dl(Op); 7044 EVT VT = Op.getValueType(); 7045 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode()); 7046 7047 // Convert shuffles that are directly supported on NEON to target-specific 7048 // DAG nodes, instead of keeping them as shuffles and matching them again 7049 // during code selection. This is more efficient and avoids the possibility 7050 // of inconsistencies between legalization and selection. 7051 // FIXME: floating-point vectors should be canonicalized to integer vectors 7052 // of the same time so that they get CSEd properly. 7053 ArrayRef<int> ShuffleMask = SVN->getMask(); 7054 7055 unsigned EltSize = VT.getScalarSizeInBits(); 7056 if (EltSize <= 32) { 7057 if (SVN->isSplat()) { 7058 int Lane = SVN->getSplatIndex(); 7059 // If this is undef splat, generate it via "just" vdup, if possible. 7060 if (Lane == -1) Lane = 0; 7061 7062 // Test if V1 is a SCALAR_TO_VECTOR. 7063 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) { 7064 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); 7065 } 7066 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR 7067 // (and probably will turn into a SCALAR_TO_VECTOR once legalization 7068 // reaches it). 7069 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR && 7070 !isa<ConstantSDNode>(V1.getOperand(0))) { 7071 bool IsScalarToVector = true; 7072 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i) 7073 if (!V1.getOperand(i).isUndef()) { 7074 IsScalarToVector = false; 7075 break; 7076 } 7077 if (IsScalarToVector) 7078 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); 7079 } 7080 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1, 7081 DAG.getConstant(Lane, dl, MVT::i32)); 7082 } 7083 7084 bool ReverseVEXT; 7085 unsigned Imm; 7086 if (isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) { 7087 if (ReverseVEXT) 7088 std::swap(V1, V2); 7089 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2, 7090 DAG.getConstant(Imm, dl, MVT::i32)); 7091 } 7092 7093 if (isVREVMask(ShuffleMask, VT, 64)) 7094 return DAG.getNode(ARMISD::VREV64, dl, VT, V1); 7095 if (isVREVMask(ShuffleMask, VT, 32)) 7096 return DAG.getNode(ARMISD::VREV32, dl, VT, V1); 7097 if (isVREVMask(ShuffleMask, VT, 16)) 7098 return DAG.getNode(ARMISD::VREV16, dl, VT, V1); 7099 7100 if (V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) { 7101 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1, 7102 DAG.getConstant(Imm, dl, MVT::i32)); 7103 } 7104 7105 // Check for Neon shuffles that modify both input vectors in place. 7106 // If both results are used, i.e., if there are two shuffles with the same 7107 // source operands and with masks corresponding to both results of one of 7108 // these operations, DAG memoization will ensure that a single node is 7109 // used for both shuffles. 7110 unsigned WhichResult; 7111 bool isV_UNDEF; 7112 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask( 7113 ShuffleMask, VT, WhichResult, isV_UNDEF)) { 7114 if (isV_UNDEF) 7115 V2 = V1; 7116 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2) 7117 .getValue(WhichResult); 7118 } 7119 7120 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize 7121 // shuffles that produce a result larger than their operands with: 7122 // shuffle(concat(v1, undef), concat(v2, undef)) 7123 // -> 7124 // shuffle(concat(v1, v2), undef) 7125 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine). 7126 // 7127 // This is useful in the general case, but there are special cases where 7128 // native shuffles produce larger results: the two-result ops. 7129 // 7130 // Look through the concat when lowering them: 7131 // shuffle(concat(v1, v2), undef) 7132 // -> 7133 // concat(VZIP(v1, v2):0, :1) 7134 // 7135 if (V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) { 7136 SDValue SubV1 = V1->getOperand(0); 7137 SDValue SubV2 = V1->getOperand(1); 7138 EVT SubVT = SubV1.getValueType(); 7139 7140 // We expect these to have been canonicalized to -1. 7141 assert(llvm::all_of(ShuffleMask, [&](int i) { 7142 return i < (int)VT.getVectorNumElements(); 7143 }) && "Unexpected shuffle index into UNDEF operand!"); 7144 7145 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask( 7146 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) { 7147 if (isV_UNDEF) 7148 SubV2 = SubV1; 7149 assert((WhichResult == 0) && 7150 "In-place shuffle of concat can only have one result!"); 7151 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT), 7152 SubV1, SubV2); 7153 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0), 7154 Res.getValue(1)); 7155 } 7156 } 7157 } 7158 7159 // If the shuffle is not directly supported and it has 4 elements, use 7160 // the PerfectShuffle-generated table to synthesize it from other shuffles. 7161 unsigned NumElts = VT.getVectorNumElements(); 7162 if (NumElts == 4) { 7163 unsigned PFIndexes[4]; 7164 for (unsigned i = 0; i != 4; ++i) { 7165 if (ShuffleMask[i] < 0) 7166 PFIndexes[i] = 8; 7167 else 7168 PFIndexes[i] = ShuffleMask[i]; 7169 } 7170 7171 // Compute the index in the perfect shuffle table. 7172 unsigned PFTableIndex = 7173 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 7174 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 7175 unsigned Cost = (PFEntry >> 30); 7176 7177 if (Cost <= 4) 7178 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 7179 } 7180 7181 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs. 7182 if (EltSize >= 32) { 7183 // Do the expansion with floating-point types, since that is what the VFP 7184 // registers are defined to use, and since i64 is not legal. 7185 EVT EltVT = EVT::getFloatingPointVT(EltSize); 7186 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts); 7187 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1); 7188 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2); 7189 SmallVector<SDValue, 8> Ops; 7190 for (unsigned i = 0; i < NumElts; ++i) { 7191 if (ShuffleMask[i] < 0) 7192 Ops.push_back(DAG.getUNDEF(EltVT)); 7193 else 7194 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 7195 ShuffleMask[i] < (int)NumElts ? V1 : V2, 7196 DAG.getConstant(ShuffleMask[i] & (NumElts-1), 7197 dl, MVT::i32))); 7198 } 7199 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops); 7200 return DAG.getNode(ISD::BITCAST, dl, VT, Val); 7201 } 7202 7203 if ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(ShuffleMask, VT)) 7204 return LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(Op, DAG); 7205 7206 if (VT == MVT::v8i8) 7207 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG)) 7208 return NewOp; 7209 7210 return SDValue(); 7211 } 7212 7213 static SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 7214 // INSERT_VECTOR_ELT is legal only for immediate indexes. 7215 SDValue Lane = Op.getOperand(2); 7216 if (!isa<ConstantSDNode>(Lane)) 7217 return SDValue(); 7218 7219 return Op; 7220 } 7221 7222 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 7223 // EXTRACT_VECTOR_ELT is legal only for immediate indexes. 7224 SDValue Lane = Op.getOperand(1); 7225 if (!isa<ConstantSDNode>(Lane)) 7226 return SDValue(); 7227 7228 SDValue Vec = Op.getOperand(0); 7229 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) { 7230 SDLoc dl(Op); 7231 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane); 7232 } 7233 7234 return Op; 7235 } 7236 7237 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { 7238 // The only time a CONCAT_VECTORS operation can have legal types is when 7239 // two 64-bit vectors are concatenated to a 128-bit vector. 7240 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 && 7241 "unexpected CONCAT_VECTORS"); 7242 SDLoc dl(Op); 7243 SDValue Val = DAG.getUNDEF(MVT::v2f64); 7244 SDValue Op0 = Op.getOperand(0); 7245 SDValue Op1 = Op.getOperand(1); 7246 if (!Op0.isUndef()) 7247 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val, 7248 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0), 7249 DAG.getIntPtrConstant(0, dl)); 7250 if (!Op1.isUndef()) 7251 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val, 7252 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1), 7253 DAG.getIntPtrConstant(1, dl)); 7254 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val); 7255 } 7256 7257 /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each 7258 /// element has been zero/sign-extended, depending on the isSigned parameter, 7259 /// from an integer type half its size. 7260 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, 7261 bool isSigned) { 7262 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32. 7263 EVT VT = N->getValueType(0); 7264 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { 7265 SDNode *BVN = N->getOperand(0).getNode(); 7266 if (BVN->getValueType(0) != MVT::v4i32 || 7267 BVN->getOpcode() != ISD::BUILD_VECTOR) 7268 return false; 7269 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0; 7270 unsigned HiElt = 1 - LoElt; 7271 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt)); 7272 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt)); 7273 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2)); 7274 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2)); 7275 if (!Lo0 || !Hi0 || !Lo1 || !Hi1) 7276 return false; 7277 if (isSigned) { 7278 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 && 7279 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32) 7280 return true; 7281 } else { 7282 if (Hi0->isNullValue() && Hi1->isNullValue()) 7283 return true; 7284 } 7285 return false; 7286 } 7287 7288 if (N->getOpcode() != ISD::BUILD_VECTOR) 7289 return false; 7290 7291 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 7292 SDNode *Elt = N->getOperand(i).getNode(); 7293 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) { 7294 unsigned EltSize = VT.getScalarSizeInBits(); 7295 unsigned HalfSize = EltSize / 2; 7296 if (isSigned) { 7297 if (!isIntN(HalfSize, C->getSExtValue())) 7298 return false; 7299 } else { 7300 if (!isUIntN(HalfSize, C->getZExtValue())) 7301 return false; 7302 } 7303 continue; 7304 } 7305 return false; 7306 } 7307 7308 return true; 7309 } 7310 7311 /// isSignExtended - Check if a node is a vector value that is sign-extended 7312 /// or a constant BUILD_VECTOR with sign-extended elements. 7313 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) { 7314 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N)) 7315 return true; 7316 if (isExtendedBUILD_VECTOR(N, DAG, true)) 7317 return true; 7318 return false; 7319 } 7320 7321 /// isZeroExtended - Check if a node is a vector value that is zero-extended 7322 /// or a constant BUILD_VECTOR with zero-extended elements. 7323 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) { 7324 if (N->getOpcode() == ISD::ZERO_EXTEND || ISD::isZEXTLoad(N)) 7325 return true; 7326 if (isExtendedBUILD_VECTOR(N, DAG, false)) 7327 return true; 7328 return false; 7329 } 7330 7331 static EVT getExtensionTo64Bits(const EVT &OrigVT) { 7332 if (OrigVT.getSizeInBits() >= 64) 7333 return OrigVT; 7334 7335 assert(OrigVT.isSimple() && "Expecting a simple value type"); 7336 7337 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy; 7338 switch (OrigSimpleTy) { 7339 default: llvm_unreachable("Unexpected Vector Type"); 7340 case MVT::v2i8: 7341 case MVT::v2i16: 7342 return MVT::v2i32; 7343 case MVT::v4i8: 7344 return MVT::v4i16; 7345 } 7346 } 7347 7348 /// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total 7349 /// value size to 64 bits. We need a 64-bit D register as an operand to VMULL. 7350 /// We insert the required extension here to get the vector to fill a D register. 7351 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, 7352 const EVT &OrigTy, 7353 const EVT &ExtTy, 7354 unsigned ExtOpcode) { 7355 // The vector originally had a size of OrigTy. It was then extended to ExtTy. 7356 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than 7357 // 64-bits we need to insert a new extension so that it will be 64-bits. 7358 assert(ExtTy.is128BitVector() && "Unexpected extension size"); 7359 if (OrigTy.getSizeInBits() >= 64) 7360 return N; 7361 7362 // Must extend size to at least 64 bits to be used as an operand for VMULL. 7363 EVT NewVT = getExtensionTo64Bits(OrigTy); 7364 7365 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N); 7366 } 7367 7368 /// SkipLoadExtensionForVMULL - return a load of the original vector size that 7369 /// does not do any sign/zero extension. If the original vector is less 7370 /// than 64 bits, an appropriate extension will be added after the load to 7371 /// reach a total size of 64 bits. We have to add the extension separately 7372 /// because ARM does not have a sign/zero extending load for vectors. 7373 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) { 7374 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT()); 7375 7376 // The load already has the right type. 7377 if (ExtendedTy == LD->getMemoryVT()) 7378 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(), 7379 LD->getBasePtr(), LD->getPointerInfo(), 7380 LD->getAlignment(), LD->getMemOperand()->getFlags()); 7381 7382 // We need to create a zextload/sextload. We cannot just create a load 7383 // followed by a zext/zext node because LowerMUL is also run during normal 7384 // operation legalization where we can't create illegal types. 7385 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy, 7386 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(), 7387 LD->getMemoryVT(), LD->getAlignment(), 7388 LD->getMemOperand()->getFlags()); 7389 } 7390 7391 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, 7392 /// extending load, or BUILD_VECTOR with extended elements, return the 7393 /// unextended value. The unextended vector should be 64 bits so that it can 7394 /// be used as an operand to a VMULL instruction. If the original vector size 7395 /// before extension is less than 64 bits we add a an extension to resize 7396 /// the vector to 64 bits. 7397 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) { 7398 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) 7399 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG, 7400 N->getOperand(0)->getValueType(0), 7401 N->getValueType(0), 7402 N->getOpcode()); 7403 7404 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 7405 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) && 7406 "Expected extending load"); 7407 7408 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG); 7409 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1)); 7410 unsigned Opcode = ISD::isSEXTLoad(LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 7411 SDValue extLoad = 7412 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad); 7413 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad); 7414 7415 return newLoad; 7416 } 7417 7418 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will 7419 // have been legalized as a BITCAST from v4i32. 7420 if (N->getOpcode() == ISD::BITCAST) { 7421 SDNode *BVN = N->getOperand(0).getNode(); 7422 assert(BVN->getOpcode() == ISD::BUILD_VECTOR && 7423 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR"); 7424 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0; 7425 return DAG.getBuildVector( 7426 MVT::v2i32, SDLoc(N), 7427 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)}); 7428 } 7429 // Construct a new BUILD_VECTOR with elements truncated to half the size. 7430 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); 7431 EVT VT = N->getValueType(0); 7432 unsigned EltSize = VT.getScalarSizeInBits() / 2; 7433 unsigned NumElts = VT.getVectorNumElements(); 7434 MVT TruncVT = MVT::getIntegerVT(EltSize); 7435 SmallVector<SDValue, 8> Ops; 7436 SDLoc dl(N); 7437 for (unsigned i = 0; i != NumElts; ++i) { 7438 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i)); 7439 const APInt &CInt = C->getAPIntValue(); 7440 // Element types smaller than 32 bits are not legal, so use i32 elements. 7441 // The values are implicitly truncated so sext vs. zext doesn't matter. 7442 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32)); 7443 } 7444 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops); 7445 } 7446 7447 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) { 7448 unsigned Opcode = N->getOpcode(); 7449 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { 7450 SDNode *N0 = N->getOperand(0).getNode(); 7451 SDNode *N1 = N->getOperand(1).getNode(); 7452 return N0->hasOneUse() && N1->hasOneUse() && 7453 isSignExtended(N0, DAG) && isSignExtended(N1, DAG); 7454 } 7455 return false; 7456 } 7457 7458 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) { 7459 unsigned Opcode = N->getOpcode(); 7460 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { 7461 SDNode *N0 = N->getOperand(0).getNode(); 7462 SDNode *N1 = N->getOperand(1).getNode(); 7463 return N0->hasOneUse() && N1->hasOneUse() && 7464 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG); 7465 } 7466 return false; 7467 } 7468 7469 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) { 7470 // Multiplications are only custom-lowered for 128-bit vectors so that 7471 // VMULL can be detected. Otherwise v2i64 multiplications are not legal. 7472 EVT VT = Op.getValueType(); 7473 assert(VT.is128BitVector() && VT.isInteger() && 7474 "unexpected type for custom-lowering ISD::MUL"); 7475 SDNode *N0 = Op.getOperand(0).getNode(); 7476 SDNode *N1 = Op.getOperand(1).getNode(); 7477 unsigned NewOpc = 0; 7478 bool isMLA = false; 7479 bool isN0SExt = isSignExtended(N0, DAG); 7480 bool isN1SExt = isSignExtended(N1, DAG); 7481 if (isN0SExt && isN1SExt) 7482 NewOpc = ARMISD::VMULLs; 7483 else { 7484 bool isN0ZExt = isZeroExtended(N0, DAG); 7485 bool isN1ZExt = isZeroExtended(N1, DAG); 7486 if (isN0ZExt && isN1ZExt) 7487 NewOpc = ARMISD::VMULLu; 7488 else if (isN1SExt || isN1ZExt) { 7489 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these 7490 // into (s/zext A * s/zext C) + (s/zext B * s/zext C) 7491 if (isN1SExt && isAddSubSExt(N0, DAG)) { 7492 NewOpc = ARMISD::VMULLs; 7493 isMLA = true; 7494 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) { 7495 NewOpc = ARMISD::VMULLu; 7496 isMLA = true; 7497 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) { 7498 std::swap(N0, N1); 7499 NewOpc = ARMISD::VMULLu; 7500 isMLA = true; 7501 } 7502 } 7503 7504 if (!NewOpc) { 7505 if (VT == MVT::v2i64) 7506 // Fall through to expand this. It is not legal. 7507 return SDValue(); 7508 else 7509 // Other vector multiplications are legal. 7510 return Op; 7511 } 7512 } 7513 7514 // Legalize to a VMULL instruction. 7515 SDLoc DL(Op); 7516 SDValue Op0; 7517 SDValue Op1 = SkipExtensionForVMULL(N1, DAG); 7518 if (!isMLA) { 7519 Op0 = SkipExtensionForVMULL(N0, DAG); 7520 assert(Op0.getValueType().is64BitVector() && 7521 Op1.getValueType().is64BitVector() && 7522 "unexpected types for extended operands to VMULL"); 7523 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); 7524 } 7525 7526 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during 7527 // isel lowering to take advantage of no-stall back to back vmul + vmla. 7528 // vmull q0, d4, d6 7529 // vmlal q0, d5, d6 7530 // is faster than 7531 // vaddl q0, d4, d5 7532 // vmovl q1, d6 7533 // vmul q0, q0, q1 7534 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG); 7535 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG); 7536 EVT Op1VT = Op1.getValueType(); 7537 return DAG.getNode(N0->getOpcode(), DL, VT, 7538 DAG.getNode(NewOpc, DL, VT, 7539 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), 7540 DAG.getNode(NewOpc, DL, VT, 7541 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); 7542 } 7543 7544 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, 7545 SelectionDAG &DAG) { 7546 // TODO: Should this propagate fast-math-flags? 7547 7548 // Convert to float 7549 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo)); 7550 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo)); 7551 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X); 7552 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y); 7553 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X); 7554 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y); 7555 // Get reciprocal estimate. 7556 // float4 recip = vrecpeq_f32(yf); 7557 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7558 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32), 7559 Y); 7560 // Because char has a smaller range than uchar, we can actually get away 7561 // without any newton steps. This requires that we use a weird bias 7562 // of 0xb000, however (again, this has been exhaustively tested). 7563 // float4 result = as_float4(as_int4(xf*recip) + 0xb000); 7564 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y); 7565 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X); 7566 Y = DAG.getConstant(0xb000, dl, MVT::v4i32); 7567 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y); 7568 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X); 7569 // Convert back to short. 7570 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X); 7571 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X); 7572 return X; 7573 } 7574 7575 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, 7576 SelectionDAG &DAG) { 7577 // TODO: Should this propagate fast-math-flags? 7578 7579 SDValue N2; 7580 // Convert to float. 7581 // float4 yf = vcvt_f32_s32(vmovl_s16(y)); 7582 // float4 xf = vcvt_f32_s32(vmovl_s16(x)); 7583 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0); 7584 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1); 7585 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0); 7586 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1); 7587 7588 // Use reciprocal estimate and one refinement step. 7589 // float4 recip = vrecpeq_f32(yf); 7590 // recip *= vrecpsq_f32(yf, recip); 7591 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7592 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32), 7593 N1); 7594 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7595 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32), 7596 N1, N2); 7597 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); 7598 // Because short has a smaller range than ushort, we can actually get away 7599 // with only a single newton step. This requires that we use a weird bias 7600 // of 89, however (again, this has been exhaustively tested). 7601 // float4 result = as_float4(as_int4(xf*recip) + 0x89); 7602 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2); 7603 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0); 7604 N1 = DAG.getConstant(0x89, dl, MVT::v4i32); 7605 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1); 7606 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0); 7607 // Convert back to integer and return. 7608 // return vmovn_s32(vcvt_s32_f32(result)); 7609 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0); 7610 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0); 7611 return N0; 7612 } 7613 7614 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) { 7615 EVT VT = Op.getValueType(); 7616 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && 7617 "unexpected type for custom-lowering ISD::SDIV"); 7618 7619 SDLoc dl(Op); 7620 SDValue N0 = Op.getOperand(0); 7621 SDValue N1 = Op.getOperand(1); 7622 SDValue N2, N3; 7623 7624 if (VT == MVT::v8i8) { 7625 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0); 7626 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1); 7627 7628 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7629 DAG.getIntPtrConstant(4, dl)); 7630 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7631 DAG.getIntPtrConstant(4, dl)); 7632 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7633 DAG.getIntPtrConstant(0, dl)); 7634 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7635 DAG.getIntPtrConstant(0, dl)); 7636 7637 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16 7638 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16 7639 7640 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2); 7641 N0 = LowerCONCAT_VECTORS(N0, DAG); 7642 7643 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0); 7644 return N0; 7645 } 7646 return LowerSDIV_v4i16(N0, N1, dl, DAG); 7647 } 7648 7649 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) { 7650 // TODO: Should this propagate fast-math-flags? 7651 EVT VT = Op.getValueType(); 7652 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && 7653 "unexpected type for custom-lowering ISD::UDIV"); 7654 7655 SDLoc dl(Op); 7656 SDValue N0 = Op.getOperand(0); 7657 SDValue N1 = Op.getOperand(1); 7658 SDValue N2, N3; 7659 7660 if (VT == MVT::v8i8) { 7661 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0); 7662 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1); 7663 7664 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7665 DAG.getIntPtrConstant(4, dl)); 7666 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7667 DAG.getIntPtrConstant(4, dl)); 7668 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7669 DAG.getIntPtrConstant(0, dl)); 7670 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7671 DAG.getIntPtrConstant(0, dl)); 7672 7673 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16 7674 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16 7675 7676 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2); 7677 N0 = LowerCONCAT_VECTORS(N0, DAG); 7678 7679 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8, 7680 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl, 7681 MVT::i32), 7682 N0); 7683 return N0; 7684 } 7685 7686 // v4i16 sdiv ... Convert to float. 7687 // float4 yf = vcvt_f32_s32(vmovl_u16(y)); 7688 // float4 xf = vcvt_f32_s32(vmovl_u16(x)); 7689 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0); 7690 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1); 7691 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0); 7692 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1); 7693 7694 // Use reciprocal estimate and two refinement steps. 7695 // float4 recip = vrecpeq_f32(yf); 7696 // recip *= vrecpsq_f32(yf, recip); 7697 // recip *= vrecpsq_f32(yf, recip); 7698 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7699 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32), 7700 BN1); 7701 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7702 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32), 7703 BN1, N2); 7704 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); 7705 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7706 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32), 7707 BN1, N2); 7708 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); 7709 // Simply multiplying by the reciprocal estimate can leave us a few ulps 7710 // too low, so we add 2 ulps (exhaustive testing shows that this is enough, 7711 // and that it will never cause us to return an answer too large). 7712 // float4 result = as_float4(as_int4(xf*recip) + 2); 7713 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2); 7714 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0); 7715 N1 = DAG.getConstant(2, dl, MVT::v4i32); 7716 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1); 7717 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0); 7718 // Convert back to integer and return. 7719 // return vmovn_u32(vcvt_s32_f32(result)); 7720 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0); 7721 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0); 7722 return N0; 7723 } 7724 7725 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) { 7726 SDNode *N = Op.getNode(); 7727 EVT VT = N->getValueType(0); 7728 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 7729 7730 SDValue Carry = Op.getOperand(2); 7731 7732 SDLoc DL(Op); 7733 7734 SDValue Result; 7735 if (Op.getOpcode() == ISD::ADDCARRY) { 7736 // This converts the boolean value carry into the carry flag. 7737 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG); 7738 7739 // Do the addition proper using the carry flag we wanted. 7740 Result = DAG.getNode(ARMISD::ADDE, DL, VTs, Op.getOperand(0), 7741 Op.getOperand(1), Carry); 7742 7743 // Now convert the carry flag into a boolean value. 7744 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); 7745 } else { 7746 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we 7747 // have to invert the carry first. 7748 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, 7749 DAG.getConstant(1, DL, MVT::i32), Carry); 7750 // This converts the boolean value carry into the carry flag. 7751 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG); 7752 7753 // Do the subtraction proper using the carry flag we wanted. 7754 Result = DAG.getNode(ARMISD::SUBE, DL, VTs, Op.getOperand(0), 7755 Op.getOperand(1), Carry); 7756 7757 // Now convert the carry flag into a boolean value. 7758 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); 7759 // But the carry returned by ARMISD::SUBE is not a borrow as expected 7760 // by ISD::SUBCARRY, so compute 1 - C. 7761 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, 7762 DAG.getConstant(1, DL, MVT::i32), Carry); 7763 } 7764 7765 // Return both values. 7766 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry); 7767 } 7768 7769 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const { 7770 assert(Subtarget->isTargetDarwin()); 7771 7772 // For iOS, we want to call an alternative entry point: __sincos_stret, 7773 // return values are passed via sret. 7774 SDLoc dl(Op); 7775 SDValue Arg = Op.getOperand(0); 7776 EVT ArgVT = Arg.getValueType(); 7777 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 7778 auto PtrVT = getPointerTy(DAG.getDataLayout()); 7779 7780 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 7781 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7782 7783 // Pair of floats / doubles used to pass the result. 7784 Type *RetTy = StructType::get(ArgTy, ArgTy); 7785 auto &DL = DAG.getDataLayout(); 7786 7787 ArgListTy Args; 7788 bool ShouldUseSRet = Subtarget->isAPCS_ABI(); 7789 SDValue SRet; 7790 if (ShouldUseSRet) { 7791 // Create stack object for sret. 7792 const uint64_t ByteSize = DL.getTypeAllocSize(RetTy); 7793 const unsigned StackAlign = DL.getPrefTypeAlignment(RetTy); 7794 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false); 7795 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL)); 7796 7797 ArgListEntry Entry; 7798 Entry.Node = SRet; 7799 Entry.Ty = RetTy->getPointerTo(); 7800 Entry.IsSExt = false; 7801 Entry.IsZExt = false; 7802 Entry.IsSRet = true; 7803 Args.push_back(Entry); 7804 RetTy = Type::getVoidTy(*DAG.getContext()); 7805 } 7806 7807 ArgListEntry Entry; 7808 Entry.Node = Arg; 7809 Entry.Ty = ArgTy; 7810 Entry.IsSExt = false; 7811 Entry.IsZExt = false; 7812 Args.push_back(Entry); 7813 7814 RTLIB::Libcall LC = 7815 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32; 7816 const char *LibcallName = getLibcallName(LC); 7817 CallingConv::ID CC = getLibcallCallingConv(LC); 7818 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL)); 7819 7820 TargetLowering::CallLoweringInfo CLI(DAG); 7821 CLI.setDebugLoc(dl) 7822 .setChain(DAG.getEntryNode()) 7823 .setCallee(CC, RetTy, Callee, std::move(Args)) 7824 .setDiscardResult(ShouldUseSRet); 7825 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 7826 7827 if (!ShouldUseSRet) 7828 return CallResult.first; 7829 7830 SDValue LoadSin = 7831 DAG.getLoad(ArgVT, dl, CallResult.second, SRet, MachinePointerInfo()); 7832 7833 // Address of cos field. 7834 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet, 7835 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl)); 7836 SDValue LoadCos = 7837 DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add, MachinePointerInfo()); 7838 7839 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT); 7840 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, 7841 LoadSin.getValue(0), LoadCos.getValue(0)); 7842 } 7843 7844 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG, 7845 bool Signed, 7846 SDValue &Chain) const { 7847 EVT VT = Op.getValueType(); 7848 assert((VT == MVT::i32 || VT == MVT::i64) && 7849 "unexpected type for custom lowering DIV"); 7850 SDLoc dl(Op); 7851 7852 const auto &DL = DAG.getDataLayout(); 7853 const auto &TLI = DAG.getTargetLoweringInfo(); 7854 7855 const char *Name = nullptr; 7856 if (Signed) 7857 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64"; 7858 else 7859 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64"; 7860 7861 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL)); 7862 7863 ARMTargetLowering::ArgListTy Args; 7864 7865 for (auto AI : {1, 0}) { 7866 ArgListEntry Arg; 7867 Arg.Node = Op.getOperand(AI); 7868 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext()); 7869 Args.push_back(Arg); 7870 } 7871 7872 CallLoweringInfo CLI(DAG); 7873 CLI.setDebugLoc(dl) 7874 .setChain(Chain) 7875 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()), 7876 ES, std::move(Args)); 7877 7878 return LowerCallTo(CLI).first; 7879 } 7880 7881 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG, 7882 bool Signed) const { 7883 assert(Op.getValueType() == MVT::i32 && 7884 "unexpected type for custom lowering DIV"); 7885 SDLoc dl(Op); 7886 7887 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other, 7888 DAG.getEntryNode(), Op.getOperand(1)); 7889 7890 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK); 7891 } 7892 7893 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) { 7894 SDLoc DL(N); 7895 SDValue Op = N->getOperand(1); 7896 if (N->getValueType(0) == MVT::i32) 7897 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, Op); 7898 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op, 7899 DAG.getConstant(0, DL, MVT::i32)); 7900 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op, 7901 DAG.getConstant(1, DL, MVT::i32)); 7902 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, 7903 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi)); 7904 } 7905 7906 void ARMTargetLowering::ExpandDIV_Windows( 7907 SDValue Op, SelectionDAG &DAG, bool Signed, 7908 SmallVectorImpl<SDValue> &Results) const { 7909 const auto &DL = DAG.getDataLayout(); 7910 const auto &TLI = DAG.getTargetLoweringInfo(); 7911 7912 assert(Op.getValueType() == MVT::i64 && 7913 "unexpected type for custom lowering DIV"); 7914 SDLoc dl(Op); 7915 7916 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode()); 7917 7918 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK); 7919 7920 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result); 7921 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result, 7922 DAG.getConstant(32, dl, TLI.getPointerTy(DL))); 7923 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper); 7924 7925 Results.push_back(Lower); 7926 Results.push_back(Upper); 7927 } 7928 7929 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) { 7930 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering())) 7931 // Acquire/Release load/store is not legal for targets without a dmb or 7932 // equivalent available. 7933 return SDValue(); 7934 7935 // Monotonic load/store is legal for all targets. 7936 return Op; 7937 } 7938 7939 static void ReplaceREADCYCLECOUNTER(SDNode *N, 7940 SmallVectorImpl<SDValue> &Results, 7941 SelectionDAG &DAG, 7942 const ARMSubtarget *Subtarget) { 7943 SDLoc DL(N); 7944 // Under Power Management extensions, the cycle-count is: 7945 // mrc p15, #0, <Rt>, c9, c13, #0 7946 SDValue Ops[] = { N->getOperand(0), // Chain 7947 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32), 7948 DAG.getConstant(15, DL, MVT::i32), 7949 DAG.getConstant(0, DL, MVT::i32), 7950 DAG.getConstant(9, DL, MVT::i32), 7951 DAG.getConstant(13, DL, MVT::i32), 7952 DAG.getConstant(0, DL, MVT::i32) 7953 }; 7954 7955 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, 7956 DAG.getVTList(MVT::i32, MVT::Other), Ops); 7957 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32, 7958 DAG.getConstant(0, DL, MVT::i32))); 7959 Results.push_back(Cycles32.getValue(1)); 7960 } 7961 7962 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) { 7963 SDLoc dl(V.getNode()); 7964 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32); 7965 SDValue VHi = DAG.getAnyExtOrTrunc( 7966 DAG.getNode(ISD::SRL, dl, MVT::i64, V, DAG.getConstant(32, dl, MVT::i32)), 7967 dl, MVT::i32); 7968 bool isBigEndian = DAG.getDataLayout().isBigEndian(); 7969 if (isBigEndian) 7970 std::swap (VLo, VHi); 7971 SDValue RegClass = 7972 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32); 7973 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32); 7974 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32); 7975 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 }; 7976 return SDValue( 7977 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0); 7978 } 7979 7980 static void ReplaceCMP_SWAP_64Results(SDNode *N, 7981 SmallVectorImpl<SDValue> & Results, 7982 SelectionDAG &DAG) { 7983 assert(N->getValueType(0) == MVT::i64 && 7984 "AtomicCmpSwap on types less than 64 should be legal"); 7985 SDValue Ops[] = {N->getOperand(1), 7986 createGPRPairNode(DAG, N->getOperand(2)), 7987 createGPRPairNode(DAG, N->getOperand(3)), 7988 N->getOperand(0)}; 7989 SDNode *CmpSwap = DAG.getMachineNode( 7990 ARM::CMP_SWAP_64, SDLoc(N), 7991 DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops); 7992 7993 MachineFunction &MF = DAG.getMachineFunction(); 7994 MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); 7995 MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); 7996 cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); 7997 7998 bool isBigEndian = DAG.getDataLayout().isBigEndian(); 7999 8000 Results.push_back( 8001 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_1 : ARM::gsub_0, 8002 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0))); 8003 Results.push_back( 8004 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_0 : ARM::gsub_1, 8005 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0))); 8006 Results.push_back(SDValue(CmpSwap, 2)); 8007 } 8008 8009 static SDValue LowerFPOWI(SDValue Op, const ARMSubtarget &Subtarget, 8010 SelectionDAG &DAG) { 8011 const auto &TLI = DAG.getTargetLoweringInfo(); 8012 8013 assert(Subtarget.getTargetTriple().isOSMSVCRT() && 8014 "Custom lowering is MSVCRT specific!"); 8015 8016 SDLoc dl(Op); 8017 SDValue Val = Op.getOperand(0); 8018 MVT Ty = Val->getSimpleValueType(0); 8019 SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, dl, Ty, Op.getOperand(1)); 8020 SDValue Callee = DAG.getExternalSymbol(Ty == MVT::f32 ? "powf" : "pow", 8021 TLI.getPointerTy(DAG.getDataLayout())); 8022 8023 TargetLowering::ArgListTy Args; 8024 TargetLowering::ArgListEntry Entry; 8025 8026 Entry.Node = Val; 8027 Entry.Ty = Val.getValueType().getTypeForEVT(*DAG.getContext()); 8028 Entry.IsZExt = true; 8029 Args.push_back(Entry); 8030 8031 Entry.Node = Exponent; 8032 Entry.Ty = Exponent.getValueType().getTypeForEVT(*DAG.getContext()); 8033 Entry.IsZExt = true; 8034 Args.push_back(Entry); 8035 8036 Type *LCRTy = Val.getValueType().getTypeForEVT(*DAG.getContext()); 8037 8038 // In the in-chain to the call is the entry node If we are emitting a 8039 // tailcall, the chain will be mutated if the node has a non-entry input 8040 // chain. 8041 SDValue InChain = DAG.getEntryNode(); 8042 SDValue TCChain = InChain; 8043 8044 const Function &F = DAG.getMachineFunction().getFunction(); 8045 bool IsTC = TLI.isInTailCallPosition(DAG, Op.getNode(), TCChain) && 8046 F.getReturnType() == LCRTy; 8047 if (IsTC) 8048 InChain = TCChain; 8049 8050 TargetLowering::CallLoweringInfo CLI(DAG); 8051 CLI.setDebugLoc(dl) 8052 .setChain(InChain) 8053 .setCallee(CallingConv::ARM_AAPCS_VFP, LCRTy, Callee, std::move(Args)) 8054 .setTailCall(IsTC); 8055 std::pair<SDValue, SDValue> CI = TLI.LowerCallTo(CLI); 8056 8057 // Return the chain (the DAG root) if it is a tail call 8058 return !CI.second.getNode() ? DAG.getRoot() : CI.first; 8059 } 8060 8061 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 8062 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump()); 8063 switch (Op.getOpcode()) { 8064 default: llvm_unreachable("Don't know how to custom lower this!"); 8065 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG); 8066 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 8067 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 8068 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 8069 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 8070 case ISD::SELECT: return LowerSELECT(Op, DAG); 8071 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 8072 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 8073 case ISD::BR_CC: return LowerBR_CC(Op, DAG); 8074 case ISD::BR_JT: return LowerBR_JT(Op, DAG); 8075 case ISD::VASTART: return LowerVASTART(Op, DAG); 8076 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget); 8077 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget); 8078 case ISD::SINT_TO_FP: 8079 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 8080 case ISD::FP_TO_SINT: 8081 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); 8082 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 8083 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 8084 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 8085 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG); 8086 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG); 8087 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG); 8088 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG, 8089 Subtarget); 8090 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget); 8091 case ISD::SHL: 8092 case ISD::SRL: 8093 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget); 8094 case ISD::SREM: return LowerREM(Op.getNode(), DAG); 8095 case ISD::UREM: return LowerREM(Op.getNode(), DAG); 8096 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG); 8097 case ISD::SRL_PARTS: 8098 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG); 8099 case ISD::CTTZ: 8100 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget); 8101 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget); 8102 case ISD::SETCC: return LowerVSETCC(Op, DAG); 8103 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG); 8104 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget); 8105 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget); 8106 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 8107 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 8108 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 8109 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 8110 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 8111 case ISD::MUL: return LowerMUL(Op, DAG); 8112 case ISD::SDIV: 8113 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) 8114 return LowerDIV_Windows(Op, DAG, /* Signed */ true); 8115 return LowerSDIV(Op, DAG); 8116 case ISD::UDIV: 8117 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) 8118 return LowerDIV_Windows(Op, DAG, /* Signed */ false); 8119 return LowerUDIV(Op, DAG); 8120 case ISD::ADDCARRY: 8121 case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG); 8122 case ISD::SADDO: 8123 case ISD::SSUBO: 8124 return LowerSignedALUO(Op, DAG); 8125 case ISD::UADDO: 8126 case ISD::USUBO: 8127 return LowerUnsignedALUO(Op, DAG); 8128 case ISD::ATOMIC_LOAD: 8129 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG); 8130 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG); 8131 case ISD::SDIVREM: 8132 case ISD::UDIVREM: return LowerDivRem(Op, DAG); 8133 case ISD::DYNAMIC_STACKALLOC: 8134 if (Subtarget->isTargetWindows()) 8135 return LowerDYNAMIC_STACKALLOC(Op, DAG); 8136 llvm_unreachable("Don't know how to custom lower this!"); 8137 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG); 8138 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); 8139 case ISD::FPOWI: return LowerFPOWI(Op, *Subtarget, DAG); 8140 case ARMISD::WIN__DBZCHK: return SDValue(); 8141 } 8142 } 8143 8144 static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results, 8145 SelectionDAG &DAG) { 8146 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 8147 unsigned Opc = 0; 8148 if (IntNo == Intrinsic::arm_smlald) 8149 Opc = ARMISD::SMLALD; 8150 else if (IntNo == Intrinsic::arm_smlaldx) 8151 Opc = ARMISD::SMLALDX; 8152 else if (IntNo == Intrinsic::arm_smlsld) 8153 Opc = ARMISD::SMLSLD; 8154 else if (IntNo == Intrinsic::arm_smlsldx) 8155 Opc = ARMISD::SMLSLDX; 8156 else 8157 return; 8158 8159 SDLoc dl(N); 8160 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 8161 N->getOperand(3), 8162 DAG.getConstant(0, dl, MVT::i32)); 8163 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 8164 N->getOperand(3), 8165 DAG.getConstant(1, dl, MVT::i32)); 8166 8167 SDValue LongMul = DAG.getNode(Opc, dl, 8168 DAG.getVTList(MVT::i32, MVT::i32), 8169 N->getOperand(1), N->getOperand(2), 8170 Lo, Hi); 8171 Results.push_back(LongMul.getValue(0)); 8172 Results.push_back(LongMul.getValue(1)); 8173 } 8174 8175 /// ReplaceNodeResults - Replace the results of node with an illegal result 8176 /// type with new values built out of custom code. 8177 void ARMTargetLowering::ReplaceNodeResults(SDNode *N, 8178 SmallVectorImpl<SDValue> &Results, 8179 SelectionDAG &DAG) const { 8180 SDValue Res; 8181 switch (N->getOpcode()) { 8182 default: 8183 llvm_unreachable("Don't know how to custom expand this!"); 8184 case ISD::READ_REGISTER: 8185 ExpandREAD_REGISTER(N, Results, DAG); 8186 break; 8187 case ISD::BITCAST: 8188 Res = ExpandBITCAST(N, DAG, Subtarget); 8189 break; 8190 case ISD::SRL: 8191 case ISD::SRA: 8192 Res = Expand64BitShift(N, DAG, Subtarget); 8193 break; 8194 case ISD::SREM: 8195 case ISD::UREM: 8196 Res = LowerREM(N, DAG); 8197 break; 8198 case ISD::SDIVREM: 8199 case ISD::UDIVREM: 8200 Res = LowerDivRem(SDValue(N, 0), DAG); 8201 assert(Res.getNumOperands() == 2 && "DivRem needs two values"); 8202 Results.push_back(Res.getValue(0)); 8203 Results.push_back(Res.getValue(1)); 8204 return; 8205 case ISD::READCYCLECOUNTER: 8206 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget); 8207 return; 8208 case ISD::UDIV: 8209 case ISD::SDIV: 8210 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows"); 8211 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV, 8212 Results); 8213 case ISD::ATOMIC_CMP_SWAP: 8214 ReplaceCMP_SWAP_64Results(N, Results, DAG); 8215 return; 8216 case ISD::INTRINSIC_WO_CHAIN: 8217 return ReplaceLongIntrinsic(N, Results, DAG); 8218 } 8219 if (Res.getNode()) 8220 Results.push_back(Res); 8221 } 8222 8223 //===----------------------------------------------------------------------===// 8224 // ARM Scheduler Hooks 8225 //===----------------------------------------------------------------------===// 8226 8227 /// SetupEntryBlockForSjLj - Insert code into the entry block that creates and 8228 /// registers the function context. 8229 void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI, 8230 MachineBasicBlock *MBB, 8231 MachineBasicBlock *DispatchBB, 8232 int FI) const { 8233 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && 8234 "ROPI/RWPI not currently supported with SjLj"); 8235 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 8236 DebugLoc dl = MI.getDebugLoc(); 8237 MachineFunction *MF = MBB->getParent(); 8238 MachineRegisterInfo *MRI = &MF->getRegInfo(); 8239 MachineConstantPool *MCP = MF->getConstantPool(); 8240 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>(); 8241 const Function &F = MF->getFunction(); 8242 8243 bool isThumb = Subtarget->isThumb(); 8244 bool isThumb2 = Subtarget->isThumb2(); 8245 8246 unsigned PCLabelId = AFI->createPICLabelUId(); 8247 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8; 8248 ARMConstantPoolValue *CPV = 8249 ARMConstantPoolMBB::Create(F.getContext(), DispatchBB, PCLabelId, PCAdj); 8250 unsigned CPI = MCP->getConstantPoolIndex(CPV, 4); 8251 8252 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass 8253 : &ARM::GPRRegClass; 8254 8255 // Grab constant pool and fixed stack memory operands. 8256 MachineMemOperand *CPMMO = 8257 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF), 8258 MachineMemOperand::MOLoad, 4, 4); 8259 8260 MachineMemOperand *FIMMOSt = 8261 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI), 8262 MachineMemOperand::MOStore, 4, 4); 8263 8264 // Load the address of the dispatch MBB into the jump buffer. 8265 if (isThumb2) { 8266 // Incoming value: jbuf 8267 // ldr.n r5, LCPI1_1 8268 // orr r5, r5, #1 8269 // add r5, pc 8270 // str r5, [$jbuf, #+4] ; &jbuf[1] 8271 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8272 BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1) 8273 .addConstantPoolIndex(CPI) 8274 .addMemOperand(CPMMO) 8275 .add(predOps(ARMCC::AL)); 8276 // Set the low bit because of thumb mode. 8277 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8278 BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2) 8279 .addReg(NewVReg1, RegState::Kill) 8280 .addImm(0x01) 8281 .add(predOps(ARMCC::AL)) 8282 .add(condCodeOp()); 8283 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8284 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3) 8285 .addReg(NewVReg2, RegState::Kill) 8286 .addImm(PCLabelId); 8287 BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12)) 8288 .addReg(NewVReg3, RegState::Kill) 8289 .addFrameIndex(FI) 8290 .addImm(36) // &jbuf[1] :: pc 8291 .addMemOperand(FIMMOSt) 8292 .add(predOps(ARMCC::AL)); 8293 } else if (isThumb) { 8294 // Incoming value: jbuf 8295 // ldr.n r1, LCPI1_4 8296 // add r1, pc 8297 // mov r2, #1 8298 // orrs r1, r2 8299 // add r2, $jbuf, #+4 ; &jbuf[1] 8300 // str r1, [r2] 8301 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8302 BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1) 8303 .addConstantPoolIndex(CPI) 8304 .addMemOperand(CPMMO) 8305 .add(predOps(ARMCC::AL)); 8306 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8307 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2) 8308 .addReg(NewVReg1, RegState::Kill) 8309 .addImm(PCLabelId); 8310 // Set the low bit because of thumb mode. 8311 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8312 BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3) 8313 .addReg(ARM::CPSR, RegState::Define) 8314 .addImm(1) 8315 .add(predOps(ARMCC::AL)); 8316 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8317 BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4) 8318 .addReg(ARM::CPSR, RegState::Define) 8319 .addReg(NewVReg2, RegState::Kill) 8320 .addReg(NewVReg3, RegState::Kill) 8321 .add(predOps(ARMCC::AL)); 8322 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); 8323 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5) 8324 .addFrameIndex(FI) 8325 .addImm(36); // &jbuf[1] :: pc 8326 BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi)) 8327 .addReg(NewVReg4, RegState::Kill) 8328 .addReg(NewVReg5, RegState::Kill) 8329 .addImm(0) 8330 .addMemOperand(FIMMOSt) 8331 .add(predOps(ARMCC::AL)); 8332 } else { 8333 // Incoming value: jbuf 8334 // ldr r1, LCPI1_1 8335 // add r1, pc, r1 8336 // str r1, [$jbuf, #+4] ; &jbuf[1] 8337 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8338 BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1) 8339 .addConstantPoolIndex(CPI) 8340 .addImm(0) 8341 .addMemOperand(CPMMO) 8342 .add(predOps(ARMCC::AL)); 8343 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8344 BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2) 8345 .addReg(NewVReg1, RegState::Kill) 8346 .addImm(PCLabelId) 8347 .add(predOps(ARMCC::AL)); 8348 BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12)) 8349 .addReg(NewVReg2, RegState::Kill) 8350 .addFrameIndex(FI) 8351 .addImm(36) // &jbuf[1] :: pc 8352 .addMemOperand(FIMMOSt) 8353 .add(predOps(ARMCC::AL)); 8354 } 8355 } 8356 8357 void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI, 8358 MachineBasicBlock *MBB) const { 8359 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 8360 DebugLoc dl = MI.getDebugLoc(); 8361 MachineFunction *MF = MBB->getParent(); 8362 MachineRegisterInfo *MRI = &MF->getRegInfo(); 8363 MachineFrameInfo &MFI = MF->getFrameInfo(); 8364 int FI = MFI.getFunctionContextIndex(); 8365 8366 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass 8367 : &ARM::GPRnopcRegClass; 8368 8369 // Get a mapping of the call site numbers to all of the landing pads they're 8370 // associated with. 8371 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad; 8372 unsigned MaxCSNum = 0; 8373 for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E; 8374 ++BB) { 8375 if (!BB->isEHPad()) continue; 8376 8377 // FIXME: We should assert that the EH_LABEL is the first MI in the landing 8378 // pad. 8379 for (MachineBasicBlock::iterator 8380 II = BB->begin(), IE = BB->end(); II != IE; ++II) { 8381 if (!II->isEHLabel()) continue; 8382 8383 MCSymbol *Sym = II->getOperand(0).getMCSymbol(); 8384 if (!MF->hasCallSiteLandingPad(Sym)) continue; 8385 8386 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym); 8387 for (SmallVectorImpl<unsigned>::iterator 8388 CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end(); 8389 CSI != CSE; ++CSI) { 8390 CallSiteNumToLPad[*CSI].push_back(&*BB); 8391 MaxCSNum = std::max(MaxCSNum, *CSI); 8392 } 8393 break; 8394 } 8395 } 8396 8397 // Get an ordered list of the machine basic blocks for the jump table. 8398 std::vector<MachineBasicBlock*> LPadList; 8399 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs; 8400 LPadList.reserve(CallSiteNumToLPad.size()); 8401 for (unsigned I = 1; I <= MaxCSNum; ++I) { 8402 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I]; 8403 for (SmallVectorImpl<MachineBasicBlock*>::iterator 8404 II = MBBList.begin(), IE = MBBList.end(); II != IE; ++II) { 8405 LPadList.push_back(*II); 8406 InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end()); 8407 } 8408 } 8409 8410 assert(!LPadList.empty() && 8411 "No landing pad destinations for the dispatch jump table!"); 8412 8413 // Create the jump table and associated information. 8414 MachineJumpTableInfo *JTI = 8415 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline); 8416 unsigned MJTI = JTI->createJumpTableIndex(LPadList); 8417 8418 // Create the MBBs for the dispatch code. 8419 8420 // Shove the dispatch's address into the return slot in the function context. 8421 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock(); 8422 DispatchBB->setIsEHPad(); 8423 8424 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); 8425 unsigned trap_opcode; 8426 if (Subtarget->isThumb()) 8427 trap_opcode = ARM::tTRAP; 8428 else 8429 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP; 8430 8431 BuildMI(TrapBB, dl, TII->get(trap_opcode)); 8432 DispatchBB->addSuccessor(TrapBB); 8433 8434 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock(); 8435 DispatchBB->addSuccessor(DispContBB); 8436 8437 // Insert and MBBs. 8438 MF->insert(MF->end(), DispatchBB); 8439 MF->insert(MF->end(), DispContBB); 8440 MF->insert(MF->end(), TrapBB); 8441 8442 // Insert code into the entry block that creates and registers the function 8443 // context. 8444 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI); 8445 8446 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand( 8447 MachinePointerInfo::getFixedStack(*MF, FI), 8448 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, 4); 8449 8450 MachineInstrBuilder MIB; 8451 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup)); 8452 8453 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII); 8454 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo(); 8455 8456 // Add a register mask with no preserved registers. This results in all 8457 // registers being marked as clobbered. This can't work if the dispatch block 8458 // is in a Thumb1 function and is linked with ARM code which uses the FP 8459 // registers, as there is no way to preserve the FP registers in Thumb1 mode. 8460 MIB.addRegMask(RI.getSjLjDispatchPreservedMask(*MF)); 8461 8462 bool IsPositionIndependent = isPositionIndependent(); 8463 unsigned NumLPads = LPadList.size(); 8464 if (Subtarget->isThumb2()) { 8465 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8466 BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1) 8467 .addFrameIndex(FI) 8468 .addImm(4) 8469 .addMemOperand(FIMMOLd) 8470 .add(predOps(ARMCC::AL)); 8471 8472 if (NumLPads < 256) { 8473 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri)) 8474 .addReg(NewVReg1) 8475 .addImm(LPadList.size()) 8476 .add(predOps(ARMCC::AL)); 8477 } else { 8478 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8479 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1) 8480 .addImm(NumLPads & 0xFFFF) 8481 .add(predOps(ARMCC::AL)); 8482 8483 unsigned VReg2 = VReg1; 8484 if ((NumLPads & 0xFFFF0000) != 0) { 8485 VReg2 = MRI->createVirtualRegister(TRC); 8486 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2) 8487 .addReg(VReg1) 8488 .addImm(NumLPads >> 16) 8489 .add(predOps(ARMCC::AL)); 8490 } 8491 8492 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr)) 8493 .addReg(NewVReg1) 8494 .addReg(VReg2) 8495 .add(predOps(ARMCC::AL)); 8496 } 8497 8498 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc)) 8499 .addMBB(TrapBB) 8500 .addImm(ARMCC::HI) 8501 .addReg(ARM::CPSR); 8502 8503 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8504 BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT), NewVReg3) 8505 .addJumpTableIndex(MJTI) 8506 .add(predOps(ARMCC::AL)); 8507 8508 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8509 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4) 8510 .addReg(NewVReg3, RegState::Kill) 8511 .addReg(NewVReg1) 8512 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2)) 8513 .add(predOps(ARMCC::AL)) 8514 .add(condCodeOp()); 8515 8516 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT)) 8517 .addReg(NewVReg4, RegState::Kill) 8518 .addReg(NewVReg1) 8519 .addJumpTableIndex(MJTI); 8520 } else if (Subtarget->isThumb()) { 8521 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8522 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1) 8523 .addFrameIndex(FI) 8524 .addImm(1) 8525 .addMemOperand(FIMMOLd) 8526 .add(predOps(ARMCC::AL)); 8527 8528 if (NumLPads < 256) { 8529 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8)) 8530 .addReg(NewVReg1) 8531 .addImm(NumLPads) 8532 .add(predOps(ARMCC::AL)); 8533 } else { 8534 MachineConstantPool *ConstantPool = MF->getConstantPool(); 8535 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); 8536 const Constant *C = ConstantInt::get(Int32Ty, NumLPads); 8537 8538 // MachineConstantPool wants an explicit alignment. 8539 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); 8540 if (Align == 0) 8541 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); 8542 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); 8543 8544 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8545 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci)) 8546 .addReg(VReg1, RegState::Define) 8547 .addConstantPoolIndex(Idx) 8548 .add(predOps(ARMCC::AL)); 8549 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr)) 8550 .addReg(NewVReg1) 8551 .addReg(VReg1) 8552 .add(predOps(ARMCC::AL)); 8553 } 8554 8555 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc)) 8556 .addMBB(TrapBB) 8557 .addImm(ARMCC::HI) 8558 .addReg(ARM::CPSR); 8559 8560 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8561 BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2) 8562 .addReg(ARM::CPSR, RegState::Define) 8563 .addReg(NewVReg1) 8564 .addImm(2) 8565 .add(predOps(ARMCC::AL)); 8566 8567 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8568 BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3) 8569 .addJumpTableIndex(MJTI) 8570 .add(predOps(ARMCC::AL)); 8571 8572 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8573 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4) 8574 .addReg(ARM::CPSR, RegState::Define) 8575 .addReg(NewVReg2, RegState::Kill) 8576 .addReg(NewVReg3) 8577 .add(predOps(ARMCC::AL)); 8578 8579 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand( 8580 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4); 8581 8582 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); 8583 BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5) 8584 .addReg(NewVReg4, RegState::Kill) 8585 .addImm(0) 8586 .addMemOperand(JTMMOLd) 8587 .add(predOps(ARMCC::AL)); 8588 8589 unsigned NewVReg6 = NewVReg5; 8590 if (IsPositionIndependent) { 8591 NewVReg6 = MRI->createVirtualRegister(TRC); 8592 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6) 8593 .addReg(ARM::CPSR, RegState::Define) 8594 .addReg(NewVReg5, RegState::Kill) 8595 .addReg(NewVReg3) 8596 .add(predOps(ARMCC::AL)); 8597 } 8598 8599 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr)) 8600 .addReg(NewVReg6, RegState::Kill) 8601 .addJumpTableIndex(MJTI); 8602 } else { 8603 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8604 BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1) 8605 .addFrameIndex(FI) 8606 .addImm(4) 8607 .addMemOperand(FIMMOLd) 8608 .add(predOps(ARMCC::AL)); 8609 8610 if (NumLPads < 256) { 8611 BuildMI(DispatchBB, dl, TII->get(ARM::CMPri)) 8612 .addReg(NewVReg1) 8613 .addImm(NumLPads) 8614 .add(predOps(ARMCC::AL)); 8615 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) { 8616 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8617 BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1) 8618 .addImm(NumLPads & 0xFFFF) 8619 .add(predOps(ARMCC::AL)); 8620 8621 unsigned VReg2 = VReg1; 8622 if ((NumLPads & 0xFFFF0000) != 0) { 8623 VReg2 = MRI->createVirtualRegister(TRC); 8624 BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2) 8625 .addReg(VReg1) 8626 .addImm(NumLPads >> 16) 8627 .add(predOps(ARMCC::AL)); 8628 } 8629 8630 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) 8631 .addReg(NewVReg1) 8632 .addReg(VReg2) 8633 .add(predOps(ARMCC::AL)); 8634 } else { 8635 MachineConstantPool *ConstantPool = MF->getConstantPool(); 8636 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); 8637 const Constant *C = ConstantInt::get(Int32Ty, NumLPads); 8638 8639 // MachineConstantPool wants an explicit alignment. 8640 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); 8641 if (Align == 0) 8642 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); 8643 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); 8644 8645 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8646 BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp)) 8647 .addReg(VReg1, RegState::Define) 8648 .addConstantPoolIndex(Idx) 8649 .addImm(0) 8650 .add(predOps(ARMCC::AL)); 8651 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) 8652 .addReg(NewVReg1) 8653 .addReg(VReg1, RegState::Kill) 8654 .add(predOps(ARMCC::AL)); 8655 } 8656 8657 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc)) 8658 .addMBB(TrapBB) 8659 .addImm(ARMCC::HI) 8660 .addReg(ARM::CPSR); 8661 8662 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8663 BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3) 8664 .addReg(NewVReg1) 8665 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2)) 8666 .add(predOps(ARMCC::AL)) 8667 .add(condCodeOp()); 8668 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8669 BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4) 8670 .addJumpTableIndex(MJTI) 8671 .add(predOps(ARMCC::AL)); 8672 8673 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand( 8674 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4); 8675 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); 8676 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5) 8677 .addReg(NewVReg3, RegState::Kill) 8678 .addReg(NewVReg4) 8679 .addImm(0) 8680 .addMemOperand(JTMMOLd) 8681 .add(predOps(ARMCC::AL)); 8682 8683 if (IsPositionIndependent) { 8684 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd)) 8685 .addReg(NewVReg5, RegState::Kill) 8686 .addReg(NewVReg4) 8687 .addJumpTableIndex(MJTI); 8688 } else { 8689 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr)) 8690 .addReg(NewVReg5, RegState::Kill) 8691 .addJumpTableIndex(MJTI); 8692 } 8693 } 8694 8695 // Add the jump table entries as successors to the MBB. 8696 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs; 8697 for (std::vector<MachineBasicBlock*>::iterator 8698 I = LPadList.begin(), E = LPadList.end(); I != E; ++I) { 8699 MachineBasicBlock *CurMBB = *I; 8700 if (SeenMBBs.insert(CurMBB).second) 8701 DispContBB->addSuccessor(CurMBB); 8702 } 8703 8704 // N.B. the order the invoke BBs are processed in doesn't matter here. 8705 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF); 8706 SmallVector<MachineBasicBlock*, 64> MBBLPads; 8707 for (MachineBasicBlock *BB : InvokeBBs) { 8708 8709 // Remove the landing pad successor from the invoke block and replace it 8710 // with the new dispatch block. 8711 SmallVector<MachineBasicBlock*, 4> Successors(BB->succ_begin(), 8712 BB->succ_end()); 8713 while (!Successors.empty()) { 8714 MachineBasicBlock *SMBB = Successors.pop_back_val(); 8715 if (SMBB->isEHPad()) { 8716 BB->removeSuccessor(SMBB); 8717 MBBLPads.push_back(SMBB); 8718 } 8719 } 8720 8721 BB->addSuccessor(DispatchBB, BranchProbability::getZero()); 8722 BB->normalizeSuccProbs(); 8723 8724 // Find the invoke call and mark all of the callee-saved registers as 8725 // 'implicit defined' so that they're spilled. This prevents code from 8726 // moving instructions to before the EH block, where they will never be 8727 // executed. 8728 for (MachineBasicBlock::reverse_iterator 8729 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) { 8730 if (!II->isCall()) continue; 8731 8732 DenseMap<unsigned, bool> DefRegs; 8733 for (MachineInstr::mop_iterator 8734 OI = II->operands_begin(), OE = II->operands_end(); 8735 OI != OE; ++OI) { 8736 if (!OI->isReg()) continue; 8737 DefRegs[OI->getReg()] = true; 8738 } 8739 8740 MachineInstrBuilder MIB(*MF, &*II); 8741 8742 for (unsigned i = 0; SavedRegs[i] != 0; ++i) { 8743 unsigned Reg = SavedRegs[i]; 8744 if (Subtarget->isThumb2() && 8745 !ARM::tGPRRegClass.contains(Reg) && 8746 !ARM::hGPRRegClass.contains(Reg)) 8747 continue; 8748 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg)) 8749 continue; 8750 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg)) 8751 continue; 8752 if (!DefRegs[Reg]) 8753 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead); 8754 } 8755 8756 break; 8757 } 8758 } 8759 8760 // Mark all former landing pads as non-landing pads. The dispatch is the only 8761 // landing pad now. 8762 for (SmallVectorImpl<MachineBasicBlock*>::iterator 8763 I = MBBLPads.begin(), E = MBBLPads.end(); I != E; ++I) 8764 (*I)->setIsEHPad(false); 8765 8766 // The instruction is gone now. 8767 MI.eraseFromParent(); 8768 } 8769 8770 static 8771 MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) { 8772 for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(), 8773 E = MBB->succ_end(); I != E; ++I) 8774 if (*I != Succ) 8775 return *I; 8776 llvm_unreachable("Expecting a BB with two successors!"); 8777 } 8778 8779 /// Return the load opcode for a given load size. If load size >= 8, 8780 /// neon opcode will be returned. 8781 static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) { 8782 if (LdSize >= 8) 8783 return LdSize == 16 ? ARM::VLD1q32wb_fixed 8784 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0; 8785 if (IsThumb1) 8786 return LdSize == 4 ? ARM::tLDRi 8787 : LdSize == 2 ? ARM::tLDRHi 8788 : LdSize == 1 ? ARM::tLDRBi : 0; 8789 if (IsThumb2) 8790 return LdSize == 4 ? ARM::t2LDR_POST 8791 : LdSize == 2 ? ARM::t2LDRH_POST 8792 : LdSize == 1 ? ARM::t2LDRB_POST : 0; 8793 return LdSize == 4 ? ARM::LDR_POST_IMM 8794 : LdSize == 2 ? ARM::LDRH_POST 8795 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0; 8796 } 8797 8798 /// Return the store opcode for a given store size. If store size >= 8, 8799 /// neon opcode will be returned. 8800 static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) { 8801 if (StSize >= 8) 8802 return StSize == 16 ? ARM::VST1q32wb_fixed 8803 : StSize == 8 ? ARM::VST1d32wb_fixed : 0; 8804 if (IsThumb1) 8805 return StSize == 4 ? ARM::tSTRi 8806 : StSize == 2 ? ARM::tSTRHi 8807 : StSize == 1 ? ARM::tSTRBi : 0; 8808 if (IsThumb2) 8809 return StSize == 4 ? ARM::t2STR_POST 8810 : StSize == 2 ? ARM::t2STRH_POST 8811 : StSize == 1 ? ARM::t2STRB_POST : 0; 8812 return StSize == 4 ? ARM::STR_POST_IMM 8813 : StSize == 2 ? ARM::STRH_POST 8814 : StSize == 1 ? ARM::STRB_POST_IMM : 0; 8815 } 8816 8817 /// Emit a post-increment load operation with given size. The instructions 8818 /// will be added to BB at Pos. 8819 static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, 8820 const TargetInstrInfo *TII, const DebugLoc &dl, 8821 unsigned LdSize, unsigned Data, unsigned AddrIn, 8822 unsigned AddrOut, bool IsThumb1, bool IsThumb2) { 8823 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2); 8824 assert(LdOpc != 0 && "Should have a load opcode"); 8825 if (LdSize >= 8) { 8826 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8827 .addReg(AddrOut, RegState::Define) 8828 .addReg(AddrIn) 8829 .addImm(0) 8830 .add(predOps(ARMCC::AL)); 8831 } else if (IsThumb1) { 8832 // load + update AddrIn 8833 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8834 .addReg(AddrIn) 8835 .addImm(0) 8836 .add(predOps(ARMCC::AL)); 8837 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut) 8838 .add(t1CondCodeOp()) 8839 .addReg(AddrIn) 8840 .addImm(LdSize) 8841 .add(predOps(ARMCC::AL)); 8842 } else if (IsThumb2) { 8843 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8844 .addReg(AddrOut, RegState::Define) 8845 .addReg(AddrIn) 8846 .addImm(LdSize) 8847 .add(predOps(ARMCC::AL)); 8848 } else { // arm 8849 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8850 .addReg(AddrOut, RegState::Define) 8851 .addReg(AddrIn) 8852 .addReg(0) 8853 .addImm(LdSize) 8854 .add(predOps(ARMCC::AL)); 8855 } 8856 } 8857 8858 /// Emit a post-increment store operation with given size. The instructions 8859 /// will be added to BB at Pos. 8860 static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, 8861 const TargetInstrInfo *TII, const DebugLoc &dl, 8862 unsigned StSize, unsigned Data, unsigned AddrIn, 8863 unsigned AddrOut, bool IsThumb1, bool IsThumb2) { 8864 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2); 8865 assert(StOpc != 0 && "Should have a store opcode"); 8866 if (StSize >= 8) { 8867 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) 8868 .addReg(AddrIn) 8869 .addImm(0) 8870 .addReg(Data) 8871 .add(predOps(ARMCC::AL)); 8872 } else if (IsThumb1) { 8873 // store + update AddrIn 8874 BuildMI(*BB, Pos, dl, TII->get(StOpc)) 8875 .addReg(Data) 8876 .addReg(AddrIn) 8877 .addImm(0) 8878 .add(predOps(ARMCC::AL)); 8879 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut) 8880 .add(t1CondCodeOp()) 8881 .addReg(AddrIn) 8882 .addImm(StSize) 8883 .add(predOps(ARMCC::AL)); 8884 } else if (IsThumb2) { 8885 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) 8886 .addReg(Data) 8887 .addReg(AddrIn) 8888 .addImm(StSize) 8889 .add(predOps(ARMCC::AL)); 8890 } else { // arm 8891 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) 8892 .addReg(Data) 8893 .addReg(AddrIn) 8894 .addReg(0) 8895 .addImm(StSize) 8896 .add(predOps(ARMCC::AL)); 8897 } 8898 } 8899 8900 MachineBasicBlock * 8901 ARMTargetLowering::EmitStructByval(MachineInstr &MI, 8902 MachineBasicBlock *BB) const { 8903 // This pseudo instruction has 3 operands: dst, src, size 8904 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold(). 8905 // Otherwise, we will generate unrolled scalar copies. 8906 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 8907 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 8908 MachineFunction::iterator It = ++BB->getIterator(); 8909 8910 unsigned dest = MI.getOperand(0).getReg(); 8911 unsigned src = MI.getOperand(1).getReg(); 8912 unsigned SizeVal = MI.getOperand(2).getImm(); 8913 unsigned Align = MI.getOperand(3).getImm(); 8914 DebugLoc dl = MI.getDebugLoc(); 8915 8916 MachineFunction *MF = BB->getParent(); 8917 MachineRegisterInfo &MRI = MF->getRegInfo(); 8918 unsigned UnitSize = 0; 8919 const TargetRegisterClass *TRC = nullptr; 8920 const TargetRegisterClass *VecTRC = nullptr; 8921 8922 bool IsThumb1 = Subtarget->isThumb1Only(); 8923 bool IsThumb2 = Subtarget->isThumb2(); 8924 bool IsThumb = Subtarget->isThumb(); 8925 8926 if (Align & 1) { 8927 UnitSize = 1; 8928 } else if (Align & 2) { 8929 UnitSize = 2; 8930 } else { 8931 // Check whether we can use NEON instructions. 8932 if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) && 8933 Subtarget->hasNEON()) { 8934 if ((Align % 16 == 0) && SizeVal >= 16) 8935 UnitSize = 16; 8936 else if ((Align % 8 == 0) && SizeVal >= 8) 8937 UnitSize = 8; 8938 } 8939 // Can't use NEON instructions. 8940 if (UnitSize == 0) 8941 UnitSize = 4; 8942 } 8943 8944 // Select the correct opcode and register class for unit size load/store 8945 bool IsNeon = UnitSize >= 8; 8946 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass; 8947 if (IsNeon) 8948 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass 8949 : UnitSize == 8 ? &ARM::DPRRegClass 8950 : nullptr; 8951 8952 unsigned BytesLeft = SizeVal % UnitSize; 8953 unsigned LoopSize = SizeVal - BytesLeft; 8954 8955 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) { 8956 // Use LDR and STR to copy. 8957 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize) 8958 // [destOut] = STR_POST(scratch, destIn, UnitSize) 8959 unsigned srcIn = src; 8960 unsigned destIn = dest; 8961 for (unsigned i = 0; i < LoopSize; i+=UnitSize) { 8962 unsigned srcOut = MRI.createVirtualRegister(TRC); 8963 unsigned destOut = MRI.createVirtualRegister(TRC); 8964 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC); 8965 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut, 8966 IsThumb1, IsThumb2); 8967 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut, 8968 IsThumb1, IsThumb2); 8969 srcIn = srcOut; 8970 destIn = destOut; 8971 } 8972 8973 // Handle the leftover bytes with LDRB and STRB. 8974 // [scratch, srcOut] = LDRB_POST(srcIn, 1) 8975 // [destOut] = STRB_POST(scratch, destIn, 1) 8976 for (unsigned i = 0; i < BytesLeft; i++) { 8977 unsigned srcOut = MRI.createVirtualRegister(TRC); 8978 unsigned destOut = MRI.createVirtualRegister(TRC); 8979 unsigned scratch = MRI.createVirtualRegister(TRC); 8980 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut, 8981 IsThumb1, IsThumb2); 8982 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut, 8983 IsThumb1, IsThumb2); 8984 srcIn = srcOut; 8985 destIn = destOut; 8986 } 8987 MI.eraseFromParent(); // The instruction is gone now. 8988 return BB; 8989 } 8990 8991 // Expand the pseudo op to a loop. 8992 // thisMBB: 8993 // ... 8994 // movw varEnd, # --> with thumb2 8995 // movt varEnd, # 8996 // ldrcp varEnd, idx --> without thumb2 8997 // fallthrough --> loopMBB 8998 // loopMBB: 8999 // PHI varPhi, varEnd, varLoop 9000 // PHI srcPhi, src, srcLoop 9001 // PHI destPhi, dst, destLoop 9002 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize) 9003 // [destLoop] = STR_POST(scratch, destPhi, UnitSize) 9004 // subs varLoop, varPhi, #UnitSize 9005 // bne loopMBB 9006 // fallthrough --> exitMBB 9007 // exitMBB: 9008 // epilogue to handle left-over bytes 9009 // [scratch, srcOut] = LDRB_POST(srcLoop, 1) 9010 // [destOut] = STRB_POST(scratch, destLoop, 1) 9011 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); 9012 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); 9013 MF->insert(It, loopMBB); 9014 MF->insert(It, exitMBB); 9015 9016 // Transfer the remainder of BB and its successor edges to exitMBB. 9017 exitMBB->splice(exitMBB->begin(), BB, 9018 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9019 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 9020 9021 // Load an immediate to varEnd. 9022 unsigned varEnd = MRI.createVirtualRegister(TRC); 9023 if (Subtarget->useMovt(*MF)) { 9024 unsigned Vtmp = varEnd; 9025 if ((LoopSize & 0xFFFF0000) != 0) 9026 Vtmp = MRI.createVirtualRegister(TRC); 9027 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVi16 : ARM::MOVi16), Vtmp) 9028 .addImm(LoopSize & 0xFFFF) 9029 .add(predOps(ARMCC::AL)); 9030 9031 if ((LoopSize & 0xFFFF0000) != 0) 9032 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVTi16 : ARM::MOVTi16), varEnd) 9033 .addReg(Vtmp) 9034 .addImm(LoopSize >> 16) 9035 .add(predOps(ARMCC::AL)); 9036 } else { 9037 MachineConstantPool *ConstantPool = MF->getConstantPool(); 9038 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); 9039 const Constant *C = ConstantInt::get(Int32Ty, LoopSize); 9040 9041 // MachineConstantPool wants an explicit alignment. 9042 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); 9043 if (Align == 0) 9044 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); 9045 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); 9046 9047 if (IsThumb) 9048 BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci)) 9049 .addReg(varEnd, RegState::Define) 9050 .addConstantPoolIndex(Idx) 9051 .add(predOps(ARMCC::AL)); 9052 else 9053 BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp)) 9054 .addReg(varEnd, RegState::Define) 9055 .addConstantPoolIndex(Idx) 9056 .addImm(0) 9057 .add(predOps(ARMCC::AL)); 9058 } 9059 BB->addSuccessor(loopMBB); 9060 9061 // Generate the loop body: 9062 // varPhi = PHI(varLoop, varEnd) 9063 // srcPhi = PHI(srcLoop, src) 9064 // destPhi = PHI(destLoop, dst) 9065 MachineBasicBlock *entryBB = BB; 9066 BB = loopMBB; 9067 unsigned varLoop = MRI.createVirtualRegister(TRC); 9068 unsigned varPhi = MRI.createVirtualRegister(TRC); 9069 unsigned srcLoop = MRI.createVirtualRegister(TRC); 9070 unsigned srcPhi = MRI.createVirtualRegister(TRC); 9071 unsigned destLoop = MRI.createVirtualRegister(TRC); 9072 unsigned destPhi = MRI.createVirtualRegister(TRC); 9073 9074 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi) 9075 .addReg(varLoop).addMBB(loopMBB) 9076 .addReg(varEnd).addMBB(entryBB); 9077 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi) 9078 .addReg(srcLoop).addMBB(loopMBB) 9079 .addReg(src).addMBB(entryBB); 9080 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi) 9081 .addReg(destLoop).addMBB(loopMBB) 9082 .addReg(dest).addMBB(entryBB); 9083 9084 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize) 9085 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz) 9086 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC); 9087 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop, 9088 IsThumb1, IsThumb2); 9089 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop, 9090 IsThumb1, IsThumb2); 9091 9092 // Decrement loop variable by UnitSize. 9093 if (IsThumb1) { 9094 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop) 9095 .add(t1CondCodeOp()) 9096 .addReg(varPhi) 9097 .addImm(UnitSize) 9098 .add(predOps(ARMCC::AL)); 9099 } else { 9100 MachineInstrBuilder MIB = 9101 BuildMI(*BB, BB->end(), dl, 9102 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop); 9103 MIB.addReg(varPhi) 9104 .addImm(UnitSize) 9105 .add(predOps(ARMCC::AL)) 9106 .add(condCodeOp()); 9107 MIB->getOperand(5).setReg(ARM::CPSR); 9108 MIB->getOperand(5).setIsDef(true); 9109 } 9110 BuildMI(*BB, BB->end(), dl, 9111 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc)) 9112 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR); 9113 9114 // loopMBB can loop back to loopMBB or fall through to exitMBB. 9115 BB->addSuccessor(loopMBB); 9116 BB->addSuccessor(exitMBB); 9117 9118 // Add epilogue to handle BytesLeft. 9119 BB = exitMBB; 9120 auto StartOfExit = exitMBB->begin(); 9121 9122 // [scratch, srcOut] = LDRB_POST(srcLoop, 1) 9123 // [destOut] = STRB_POST(scratch, destLoop, 1) 9124 unsigned srcIn = srcLoop; 9125 unsigned destIn = destLoop; 9126 for (unsigned i = 0; i < BytesLeft; i++) { 9127 unsigned srcOut = MRI.createVirtualRegister(TRC); 9128 unsigned destOut = MRI.createVirtualRegister(TRC); 9129 unsigned scratch = MRI.createVirtualRegister(TRC); 9130 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut, 9131 IsThumb1, IsThumb2); 9132 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut, 9133 IsThumb1, IsThumb2); 9134 srcIn = srcOut; 9135 destIn = destOut; 9136 } 9137 9138 MI.eraseFromParent(); // The instruction is gone now. 9139 return BB; 9140 } 9141 9142 MachineBasicBlock * 9143 ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI, 9144 MachineBasicBlock *MBB) const { 9145 const TargetMachine &TM = getTargetMachine(); 9146 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); 9147 DebugLoc DL = MI.getDebugLoc(); 9148 9149 assert(Subtarget->isTargetWindows() && 9150 "__chkstk is only supported on Windows"); 9151 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode"); 9152 9153 // __chkstk takes the number of words to allocate on the stack in R4, and 9154 // returns the stack adjustment in number of bytes in R4. This will not 9155 // clober any other registers (other than the obvious lr). 9156 // 9157 // Although, technically, IP should be considered a register which may be 9158 // clobbered, the call itself will not touch it. Windows on ARM is a pure 9159 // thumb-2 environment, so there is no interworking required. As a result, we 9160 // do not expect a veneer to be emitted by the linker, clobbering IP. 9161 // 9162 // Each module receives its own copy of __chkstk, so no import thunk is 9163 // required, again, ensuring that IP is not clobbered. 9164 // 9165 // Finally, although some linkers may theoretically provide a trampoline for 9166 // out of range calls (which is quite common due to a 32M range limitation of 9167 // branches for Thumb), we can generate the long-call version via 9168 // -mcmodel=large, alleviating the need for the trampoline which may clobber 9169 // IP. 9170 9171 switch (TM.getCodeModel()) { 9172 case CodeModel::Small: 9173 case CodeModel::Medium: 9174 case CodeModel::Kernel: 9175 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL)) 9176 .add(predOps(ARMCC::AL)) 9177 .addExternalSymbol("__chkstk") 9178 .addReg(ARM::R4, RegState::Implicit | RegState::Kill) 9179 .addReg(ARM::R4, RegState::Implicit | RegState::Define) 9180 .addReg(ARM::R12, 9181 RegState::Implicit | RegState::Define | RegState::Dead) 9182 .addReg(ARM::CPSR, 9183 RegState::Implicit | RegState::Define | RegState::Dead); 9184 break; 9185 case CodeModel::Large: { 9186 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 9187 unsigned Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass); 9188 9189 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg) 9190 .addExternalSymbol("__chkstk"); 9191 BuildMI(*MBB, MI, DL, TII.get(ARM::tBLXr)) 9192 .add(predOps(ARMCC::AL)) 9193 .addReg(Reg, RegState::Kill) 9194 .addReg(ARM::R4, RegState::Implicit | RegState::Kill) 9195 .addReg(ARM::R4, RegState::Implicit | RegState::Define) 9196 .addReg(ARM::R12, 9197 RegState::Implicit | RegState::Define | RegState::Dead) 9198 .addReg(ARM::CPSR, 9199 RegState::Implicit | RegState::Define | RegState::Dead); 9200 break; 9201 } 9202 } 9203 9204 BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr), ARM::SP) 9205 .addReg(ARM::SP, RegState::Kill) 9206 .addReg(ARM::R4, RegState::Kill) 9207 .setMIFlags(MachineInstr::FrameSetup) 9208 .add(predOps(ARMCC::AL)) 9209 .add(condCodeOp()); 9210 9211 MI.eraseFromParent(); 9212 return MBB; 9213 } 9214 9215 MachineBasicBlock * 9216 ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI, 9217 MachineBasicBlock *MBB) const { 9218 DebugLoc DL = MI.getDebugLoc(); 9219 MachineFunction *MF = MBB->getParent(); 9220 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 9221 9222 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock(); 9223 MF->insert(++MBB->getIterator(), ContBB); 9224 ContBB->splice(ContBB->begin(), MBB, 9225 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 9226 ContBB->transferSuccessorsAndUpdatePHIs(MBB); 9227 MBB->addSuccessor(ContBB); 9228 9229 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); 9230 BuildMI(TrapBB, DL, TII->get(ARM::t__brkdiv0)); 9231 MF->push_back(TrapBB); 9232 MBB->addSuccessor(TrapBB); 9233 9234 BuildMI(*MBB, MI, DL, TII->get(ARM::tCMPi8)) 9235 .addReg(MI.getOperand(0).getReg()) 9236 .addImm(0) 9237 .add(predOps(ARMCC::AL)); 9238 BuildMI(*MBB, MI, DL, TII->get(ARM::t2Bcc)) 9239 .addMBB(TrapBB) 9240 .addImm(ARMCC::EQ) 9241 .addReg(ARM::CPSR); 9242 9243 MI.eraseFromParent(); 9244 return ContBB; 9245 } 9246 9247 MachineBasicBlock * 9248 ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 9249 MachineBasicBlock *BB) const { 9250 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 9251 DebugLoc dl = MI.getDebugLoc(); 9252 bool isThumb2 = Subtarget->isThumb2(); 9253 switch (MI.getOpcode()) { 9254 default: { 9255 MI.print(errs()); 9256 llvm_unreachable("Unexpected instr type to insert"); 9257 } 9258 9259 // Thumb1 post-indexed loads are really just single-register LDMs. 9260 case ARM::tLDR_postidx: { 9261 MachineOperand Def(MI.getOperand(1)); 9262 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD)) 9263 .add(Def) // Rn_wb 9264 .add(MI.getOperand(2)) // Rn 9265 .add(MI.getOperand(3)) // PredImm 9266 .add(MI.getOperand(4)) // PredReg 9267 .add(MI.getOperand(0)); // Rt 9268 MI.eraseFromParent(); 9269 return BB; 9270 } 9271 9272 // The Thumb2 pre-indexed stores have the same MI operands, they just 9273 // define them differently in the .td files from the isel patterns, so 9274 // they need pseudos. 9275 case ARM::t2STR_preidx: 9276 MI.setDesc(TII->get(ARM::t2STR_PRE)); 9277 return BB; 9278 case ARM::t2STRB_preidx: 9279 MI.setDesc(TII->get(ARM::t2STRB_PRE)); 9280 return BB; 9281 case ARM::t2STRH_preidx: 9282 MI.setDesc(TII->get(ARM::t2STRH_PRE)); 9283 return BB; 9284 9285 case ARM::STRi_preidx: 9286 case ARM::STRBi_preidx: { 9287 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM 9288 : ARM::STRB_PRE_IMM; 9289 // Decode the offset. 9290 unsigned Offset = MI.getOperand(4).getImm(); 9291 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub; 9292 Offset = ARM_AM::getAM2Offset(Offset); 9293 if (isSub) 9294 Offset = -Offset; 9295 9296 MachineMemOperand *MMO = *MI.memoperands_begin(); 9297 BuildMI(*BB, MI, dl, TII->get(NewOpc)) 9298 .add(MI.getOperand(0)) // Rn_wb 9299 .add(MI.getOperand(1)) // Rt 9300 .add(MI.getOperand(2)) // Rn 9301 .addImm(Offset) // offset (skip GPR==zero_reg) 9302 .add(MI.getOperand(5)) // pred 9303 .add(MI.getOperand(6)) 9304 .addMemOperand(MMO); 9305 MI.eraseFromParent(); 9306 return BB; 9307 } 9308 case ARM::STRr_preidx: 9309 case ARM::STRBr_preidx: 9310 case ARM::STRH_preidx: { 9311 unsigned NewOpc; 9312 switch (MI.getOpcode()) { 9313 default: llvm_unreachable("unexpected opcode!"); 9314 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break; 9315 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break; 9316 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break; 9317 } 9318 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc)); 9319 for (unsigned i = 0; i < MI.getNumOperands(); ++i) 9320 MIB.add(MI.getOperand(i)); 9321 MI.eraseFromParent(); 9322 return BB; 9323 } 9324 9325 case ARM::tMOVCCr_pseudo: { 9326 // To "insert" a SELECT_CC instruction, we actually have to insert the 9327 // diamond control-flow pattern. The incoming instruction knows the 9328 // destination vreg to set, the condition code register to branch on, the 9329 // true/false values to select between, and a branch opcode to use. 9330 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 9331 MachineFunction::iterator It = ++BB->getIterator(); 9332 9333 // thisMBB: 9334 // ... 9335 // TrueVal = ... 9336 // cmpTY ccX, r1, r2 9337 // bCC copy1MBB 9338 // fallthrough --> copy0MBB 9339 MachineBasicBlock *thisMBB = BB; 9340 MachineFunction *F = BB->getParent(); 9341 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 9342 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 9343 F->insert(It, copy0MBB); 9344 F->insert(It, sinkMBB); 9345 9346 // Transfer the remainder of BB and its successor edges to sinkMBB. 9347 sinkMBB->splice(sinkMBB->begin(), BB, 9348 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9349 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 9350 9351 BB->addSuccessor(copy0MBB); 9352 BB->addSuccessor(sinkMBB); 9353 9354 BuildMI(BB, dl, TII->get(ARM::tBcc)) 9355 .addMBB(sinkMBB) 9356 .addImm(MI.getOperand(3).getImm()) 9357 .addReg(MI.getOperand(4).getReg()); 9358 9359 // copy0MBB: 9360 // %FalseValue = ... 9361 // # fallthrough to sinkMBB 9362 BB = copy0MBB; 9363 9364 // Update machine-CFG edges 9365 BB->addSuccessor(sinkMBB); 9366 9367 // sinkMBB: 9368 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 9369 // ... 9370 BB = sinkMBB; 9371 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg()) 9372 .addReg(MI.getOperand(1).getReg()) 9373 .addMBB(copy0MBB) 9374 .addReg(MI.getOperand(2).getReg()) 9375 .addMBB(thisMBB); 9376 9377 MI.eraseFromParent(); // The pseudo instruction is gone now. 9378 return BB; 9379 } 9380 9381 case ARM::BCCi64: 9382 case ARM::BCCZi64: { 9383 // If there is an unconditional branch to the other successor, remove it. 9384 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9385 9386 // Compare both parts that make up the double comparison separately for 9387 // equality. 9388 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64; 9389 9390 unsigned LHS1 = MI.getOperand(1).getReg(); 9391 unsigned LHS2 = MI.getOperand(2).getReg(); 9392 if (RHSisZero) { 9393 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) 9394 .addReg(LHS1) 9395 .addImm(0) 9396 .add(predOps(ARMCC::AL)); 9397 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) 9398 .addReg(LHS2).addImm(0) 9399 .addImm(ARMCC::EQ).addReg(ARM::CPSR); 9400 } else { 9401 unsigned RHS1 = MI.getOperand(3).getReg(); 9402 unsigned RHS2 = MI.getOperand(4).getReg(); 9403 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) 9404 .addReg(LHS1) 9405 .addReg(RHS1) 9406 .add(predOps(ARMCC::AL)); 9407 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) 9408 .addReg(LHS2).addReg(RHS2) 9409 .addImm(ARMCC::EQ).addReg(ARM::CPSR); 9410 } 9411 9412 MachineBasicBlock *destMBB = MI.getOperand(RHSisZero ? 3 : 5).getMBB(); 9413 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB); 9414 if (MI.getOperand(0).getImm() == ARMCC::NE) 9415 std::swap(destMBB, exitMBB); 9416 9417 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)) 9418 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR); 9419 if (isThumb2) 9420 BuildMI(BB, dl, TII->get(ARM::t2B)) 9421 .addMBB(exitMBB) 9422 .add(predOps(ARMCC::AL)); 9423 else 9424 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB); 9425 9426 MI.eraseFromParent(); // The pseudo instruction is gone now. 9427 return BB; 9428 } 9429 9430 case ARM::Int_eh_sjlj_setjmp: 9431 case ARM::Int_eh_sjlj_setjmp_nofp: 9432 case ARM::tInt_eh_sjlj_setjmp: 9433 case ARM::t2Int_eh_sjlj_setjmp: 9434 case ARM::t2Int_eh_sjlj_setjmp_nofp: 9435 return BB; 9436 9437 case ARM::Int_eh_sjlj_setup_dispatch: 9438 EmitSjLjDispatchBlock(MI, BB); 9439 return BB; 9440 9441 case ARM::ABS: 9442 case ARM::t2ABS: { 9443 // To insert an ABS instruction, we have to insert the 9444 // diamond control-flow pattern. The incoming instruction knows the 9445 // source vreg to test against 0, the destination vreg to set, 9446 // the condition code register to branch on, the 9447 // true/false values to select between, and a branch opcode to use. 9448 // It transforms 9449 // V1 = ABS V0 9450 // into 9451 // V2 = MOVS V0 9452 // BCC (branch to SinkBB if V0 >= 0) 9453 // RSBBB: V3 = RSBri V2, 0 (compute ABS if V2 < 0) 9454 // SinkBB: V1 = PHI(V2, V3) 9455 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 9456 MachineFunction::iterator BBI = ++BB->getIterator(); 9457 MachineFunction *Fn = BB->getParent(); 9458 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB); 9459 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB); 9460 Fn->insert(BBI, RSBBB); 9461 Fn->insert(BBI, SinkBB); 9462 9463 unsigned int ABSSrcReg = MI.getOperand(1).getReg(); 9464 unsigned int ABSDstReg = MI.getOperand(0).getReg(); 9465 bool ABSSrcKIll = MI.getOperand(1).isKill(); 9466 bool isThumb2 = Subtarget->isThumb2(); 9467 MachineRegisterInfo &MRI = Fn->getRegInfo(); 9468 // In Thumb mode S must not be specified if source register is the SP or 9469 // PC and if destination register is the SP, so restrict register class 9470 unsigned NewRsbDstReg = 9471 MRI.createVirtualRegister(isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass); 9472 9473 // Transfer the remainder of BB and its successor edges to sinkMBB. 9474 SinkBB->splice(SinkBB->begin(), BB, 9475 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9476 SinkBB->transferSuccessorsAndUpdatePHIs(BB); 9477 9478 BB->addSuccessor(RSBBB); 9479 BB->addSuccessor(SinkBB); 9480 9481 // fall through to SinkMBB 9482 RSBBB->addSuccessor(SinkBB); 9483 9484 // insert a cmp at the end of BB 9485 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) 9486 .addReg(ABSSrcReg) 9487 .addImm(0) 9488 .add(predOps(ARMCC::AL)); 9489 9490 // insert a bcc with opposite CC to ARMCC::MI at the end of BB 9491 BuildMI(BB, dl, 9492 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB) 9493 .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR); 9494 9495 // insert rsbri in RSBBB 9496 // Note: BCC and rsbri will be converted into predicated rsbmi 9497 // by if-conversion pass 9498 BuildMI(*RSBBB, RSBBB->begin(), dl, 9499 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg) 9500 .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0) 9501 .addImm(0) 9502 .add(predOps(ARMCC::AL)) 9503 .add(condCodeOp()); 9504 9505 // insert PHI in SinkBB, 9506 // reuse ABSDstReg to not change uses of ABS instruction 9507 BuildMI(*SinkBB, SinkBB->begin(), dl, 9508 TII->get(ARM::PHI), ABSDstReg) 9509 .addReg(NewRsbDstReg).addMBB(RSBBB) 9510 .addReg(ABSSrcReg).addMBB(BB); 9511 9512 // remove ABS instruction 9513 MI.eraseFromParent(); 9514 9515 // return last added BB 9516 return SinkBB; 9517 } 9518 case ARM::COPY_STRUCT_BYVAL_I32: 9519 ++NumLoopByVals; 9520 return EmitStructByval(MI, BB); 9521 case ARM::WIN__CHKSTK: 9522 return EmitLowered__chkstk(MI, BB); 9523 case ARM::WIN__DBZCHK: 9524 return EmitLowered__dbzchk(MI, BB); 9525 } 9526 } 9527 9528 /// Attaches vregs to MEMCPY that it will use as scratch registers 9529 /// when it is expanded into LDM/STM. This is done as a post-isel lowering 9530 /// instead of as a custom inserter because we need the use list from the SDNode. 9531 static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, 9532 MachineInstr &MI, const SDNode *Node) { 9533 bool isThumb1 = Subtarget->isThumb1Only(); 9534 9535 DebugLoc DL = MI.getDebugLoc(); 9536 MachineFunction *MF = MI.getParent()->getParent(); 9537 MachineRegisterInfo &MRI = MF->getRegInfo(); 9538 MachineInstrBuilder MIB(*MF, MI); 9539 9540 // If the new dst/src is unused mark it as dead. 9541 if (!Node->hasAnyUseOfValue(0)) { 9542 MI.getOperand(0).setIsDead(true); 9543 } 9544 if (!Node->hasAnyUseOfValue(1)) { 9545 MI.getOperand(1).setIsDead(true); 9546 } 9547 9548 // The MEMCPY both defines and kills the scratch registers. 9549 for (unsigned I = 0; I != MI.getOperand(4).getImm(); ++I) { 9550 unsigned TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass 9551 : &ARM::GPRRegClass); 9552 MIB.addReg(TmpReg, RegState::Define|RegState::Dead); 9553 } 9554 } 9555 9556 void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, 9557 SDNode *Node) const { 9558 if (MI.getOpcode() == ARM::MEMCPY) { 9559 attachMEMCPYScratchRegs(Subtarget, MI, Node); 9560 return; 9561 } 9562 9563 const MCInstrDesc *MCID = &MI.getDesc(); 9564 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB, 9565 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional 9566 // operand is still set to noreg. If needed, set the optional operand's 9567 // register to CPSR, and remove the redundant implicit def. 9568 // 9569 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR). 9570 9571 // Rename pseudo opcodes. 9572 unsigned NewOpc = convertAddSubFlagsOpcode(MI.getOpcode()); 9573 unsigned ccOutIdx; 9574 if (NewOpc) { 9575 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo(); 9576 MCID = &TII->get(NewOpc); 9577 9578 assert(MCID->getNumOperands() == 9579 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() 9580 && "converted opcode should be the same except for cc_out" 9581 " (and, on Thumb1, pred)"); 9582 9583 MI.setDesc(*MCID); 9584 9585 // Add the optional cc_out operand 9586 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/true)); 9587 9588 // On Thumb1, move all input operands to the end, then add the predicate 9589 if (Subtarget->isThumb1Only()) { 9590 for (unsigned c = MCID->getNumOperands() - 4; c--;) { 9591 MI.addOperand(MI.getOperand(1)); 9592 MI.RemoveOperand(1); 9593 } 9594 9595 // Restore the ties 9596 for (unsigned i = MI.getNumOperands(); i--;) { 9597 const MachineOperand& op = MI.getOperand(i); 9598 if (op.isReg() && op.isUse()) { 9599 int DefIdx = MCID->getOperandConstraint(i, MCOI::TIED_TO); 9600 if (DefIdx != -1) 9601 MI.tieOperands(DefIdx, i); 9602 } 9603 } 9604 9605 MI.addOperand(MachineOperand::CreateImm(ARMCC::AL)); 9606 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/false)); 9607 ccOutIdx = 1; 9608 } else 9609 ccOutIdx = MCID->getNumOperands() - 1; 9610 } else 9611 ccOutIdx = MCID->getNumOperands() - 1; 9612 9613 // Any ARM instruction that sets the 's' bit should specify an optional 9614 // "cc_out" operand in the last operand position. 9615 if (!MI.hasOptionalDef() || !MCID->OpInfo[ccOutIdx].isOptionalDef()) { 9616 assert(!NewOpc && "Optional cc_out operand required"); 9617 return; 9618 } 9619 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it 9620 // since we already have an optional CPSR def. 9621 bool definesCPSR = false; 9622 bool deadCPSR = false; 9623 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e; 9624 ++i) { 9625 const MachineOperand &MO = MI.getOperand(i); 9626 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) { 9627 definesCPSR = true; 9628 if (MO.isDead()) 9629 deadCPSR = true; 9630 MI.RemoveOperand(i); 9631 break; 9632 } 9633 } 9634 if (!definesCPSR) { 9635 assert(!NewOpc && "Optional cc_out operand required"); 9636 return; 9637 } 9638 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag"); 9639 if (deadCPSR) { 9640 assert(!MI.getOperand(ccOutIdx).getReg() && 9641 "expect uninitialized optional cc_out operand"); 9642 // Thumb1 instructions must have the S bit even if the CPSR is dead. 9643 if (!Subtarget->isThumb1Only()) 9644 return; 9645 } 9646 9647 // If this instruction was defined with an optional CPSR def and its dag node 9648 // had a live implicit CPSR def, then activate the optional CPSR def. 9649 MachineOperand &MO = MI.getOperand(ccOutIdx); 9650 MO.setReg(ARM::CPSR); 9651 MO.setIsDef(true); 9652 } 9653 9654 //===----------------------------------------------------------------------===// 9655 // ARM Optimization Hooks 9656 //===----------------------------------------------------------------------===// 9657 9658 // Helper function that checks if N is a null or all ones constant. 9659 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) { 9660 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N); 9661 } 9662 9663 // Return true if N is conditionally 0 or all ones. 9664 // Detects these expressions where cc is an i1 value: 9665 // 9666 // (select cc 0, y) [AllOnes=0] 9667 // (select cc y, 0) [AllOnes=0] 9668 // (zext cc) [AllOnes=0] 9669 // (sext cc) [AllOnes=0/1] 9670 // (select cc -1, y) [AllOnes=1] 9671 // (select cc y, -1) [AllOnes=1] 9672 // 9673 // Invert is set when N is the null/all ones constant when CC is false. 9674 // OtherOp is set to the alternative value of N. 9675 static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, 9676 SDValue &CC, bool &Invert, 9677 SDValue &OtherOp, 9678 SelectionDAG &DAG) { 9679 switch (N->getOpcode()) { 9680 default: return false; 9681 case ISD::SELECT: { 9682 CC = N->getOperand(0); 9683 SDValue N1 = N->getOperand(1); 9684 SDValue N2 = N->getOperand(2); 9685 if (isZeroOrAllOnes(N1, AllOnes)) { 9686 Invert = false; 9687 OtherOp = N2; 9688 return true; 9689 } 9690 if (isZeroOrAllOnes(N2, AllOnes)) { 9691 Invert = true; 9692 OtherOp = N1; 9693 return true; 9694 } 9695 return false; 9696 } 9697 case ISD::ZERO_EXTEND: 9698 // (zext cc) can never be the all ones value. 9699 if (AllOnes) 9700 return false; 9701 LLVM_FALLTHROUGH; 9702 case ISD::SIGN_EXTEND: { 9703 SDLoc dl(N); 9704 EVT VT = N->getValueType(0); 9705 CC = N->getOperand(0); 9706 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC) 9707 return false; 9708 Invert = !AllOnes; 9709 if (AllOnes) 9710 // When looking for an AllOnes constant, N is an sext, and the 'other' 9711 // value is 0. 9712 OtherOp = DAG.getConstant(0, dl, VT); 9713 else if (N->getOpcode() == ISD::ZERO_EXTEND) 9714 // When looking for a 0 constant, N can be zext or sext. 9715 OtherOp = DAG.getConstant(1, dl, VT); 9716 else 9717 OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, 9718 VT); 9719 return true; 9720 } 9721 } 9722 } 9723 9724 // Combine a constant select operand into its use: 9725 // 9726 // (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) 9727 // (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) 9728 // (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1] 9729 // (or (select cc, 0, c), x) -> (select cc, x, (or, x, c)) 9730 // (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c)) 9731 // 9732 // The transform is rejected if the select doesn't have a constant operand that 9733 // is null, or all ones when AllOnes is set. 9734 // 9735 // Also recognize sext/zext from i1: 9736 // 9737 // (add (zext cc), x) -> (select cc (add x, 1), x) 9738 // (add (sext cc), x) -> (select cc (add x, -1), x) 9739 // 9740 // These transformations eventually create predicated instructions. 9741 // 9742 // @param N The node to transform. 9743 // @param Slct The N operand that is a select. 9744 // @param OtherOp The other N operand (x above). 9745 // @param DCI Context. 9746 // @param AllOnes Require the select constant to be all ones instead of null. 9747 // @returns The new node, or SDValue() on failure. 9748 static 9749 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, 9750 TargetLowering::DAGCombinerInfo &DCI, 9751 bool AllOnes = false) { 9752 SelectionDAG &DAG = DCI.DAG; 9753 EVT VT = N->getValueType(0); 9754 SDValue NonConstantVal; 9755 SDValue CCOp; 9756 bool SwapSelectOps; 9757 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps, 9758 NonConstantVal, DAG)) 9759 return SDValue(); 9760 9761 // Slct is now know to be the desired identity constant when CC is true. 9762 SDValue TrueVal = OtherOp; 9763 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, 9764 OtherOp, NonConstantVal); 9765 // Unless SwapSelectOps says CC should be false. 9766 if (SwapSelectOps) 9767 std::swap(TrueVal, FalseVal); 9768 9769 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, 9770 CCOp, TrueVal, FalseVal); 9771 } 9772 9773 // Attempt combineSelectAndUse on each operand of a commutative operator N. 9774 static 9775 SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, 9776 TargetLowering::DAGCombinerInfo &DCI) { 9777 SDValue N0 = N->getOperand(0); 9778 SDValue N1 = N->getOperand(1); 9779 if (N0.getNode()->hasOneUse()) 9780 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes)) 9781 return Result; 9782 if (N1.getNode()->hasOneUse()) 9783 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes)) 9784 return Result; 9785 return SDValue(); 9786 } 9787 9788 static bool IsVUZPShuffleNode(SDNode *N) { 9789 // VUZP shuffle node. 9790 if (N->getOpcode() == ARMISD::VUZP) 9791 return true; 9792 9793 // "VUZP" on i32 is an alias for VTRN. 9794 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32) 9795 return true; 9796 9797 return false; 9798 } 9799 9800 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, 9801 TargetLowering::DAGCombinerInfo &DCI, 9802 const ARMSubtarget *Subtarget) { 9803 // Look for ADD(VUZP.0, VUZP.1). 9804 if (!IsVUZPShuffleNode(N0.getNode()) || N0.getNode() != N1.getNode() || 9805 N0 == N1) 9806 return SDValue(); 9807 9808 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD. 9809 if (!N->getValueType(0).is64BitVector()) 9810 return SDValue(); 9811 9812 // Generate vpadd. 9813 SelectionDAG &DAG = DCI.DAG; 9814 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9815 SDLoc dl(N); 9816 SDNode *Unzip = N0.getNode(); 9817 EVT VT = N->getValueType(0); 9818 9819 SmallVector<SDValue, 8> Ops; 9820 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl, 9821 TLI.getPointerTy(DAG.getDataLayout()))); 9822 Ops.push_back(Unzip->getOperand(0)); 9823 Ops.push_back(Unzip->getOperand(1)); 9824 9825 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); 9826 } 9827 9828 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, 9829 TargetLowering::DAGCombinerInfo &DCI, 9830 const ARMSubtarget *Subtarget) { 9831 // Check for two extended operands. 9832 if (!(N0.getOpcode() == ISD::SIGN_EXTEND && 9833 N1.getOpcode() == ISD::SIGN_EXTEND) && 9834 !(N0.getOpcode() == ISD::ZERO_EXTEND && 9835 N1.getOpcode() == ISD::ZERO_EXTEND)) 9836 return SDValue(); 9837 9838 SDValue N00 = N0.getOperand(0); 9839 SDValue N10 = N1.getOperand(0); 9840 9841 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1)) 9842 if (!IsVUZPShuffleNode(N00.getNode()) || N00.getNode() != N10.getNode() || 9843 N00 == N10) 9844 return SDValue(); 9845 9846 // We only recognize Q register paddl here; this can't be reached until 9847 // after type legalization. 9848 if (!N00.getValueType().is64BitVector() || 9849 !N0.getValueType().is128BitVector()) 9850 return SDValue(); 9851 9852 // Generate vpaddl. 9853 SelectionDAG &DAG = DCI.DAG; 9854 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9855 SDLoc dl(N); 9856 EVT VT = N->getValueType(0); 9857 9858 SmallVector<SDValue, 8> Ops; 9859 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension. 9860 unsigned Opcode; 9861 if (N0.getOpcode() == ISD::SIGN_EXTEND) 9862 Opcode = Intrinsic::arm_neon_vpaddls; 9863 else 9864 Opcode = Intrinsic::arm_neon_vpaddlu; 9865 Ops.push_back(DAG.getConstant(Opcode, dl, 9866 TLI.getPointerTy(DAG.getDataLayout()))); 9867 EVT ElemTy = N00.getValueType().getVectorElementType(); 9868 unsigned NumElts = VT.getVectorNumElements(); 9869 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2); 9870 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT, 9871 N00.getOperand(0), N00.getOperand(1)); 9872 Ops.push_back(Concat); 9873 9874 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); 9875 } 9876 9877 // FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in 9878 // an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is 9879 // much easier to match. 9880 static SDValue 9881 AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, 9882 TargetLowering::DAGCombinerInfo &DCI, 9883 const ARMSubtarget *Subtarget) { 9884 // Only perform optimization if after legalize, and if NEON is available. We 9885 // also expected both operands to be BUILD_VECTORs. 9886 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON() 9887 || N0.getOpcode() != ISD::BUILD_VECTOR 9888 || N1.getOpcode() != ISD::BUILD_VECTOR) 9889 return SDValue(); 9890 9891 // Check output type since VPADDL operand elements can only be 8, 16, or 32. 9892 EVT VT = N->getValueType(0); 9893 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64) 9894 return SDValue(); 9895 9896 // Check that the vector operands are of the right form. 9897 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR 9898 // operands, where N is the size of the formed vector. 9899 // Each EXTRACT_VECTOR should have the same input vector and odd or even 9900 // index such that we have a pair wise add pattern. 9901 9902 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing. 9903 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 9904 return SDValue(); 9905 SDValue Vec = N0->getOperand(0)->getOperand(0); 9906 SDNode *V = Vec.getNode(); 9907 unsigned nextIndex = 0; 9908 9909 // For each operands to the ADD which are BUILD_VECTORs, 9910 // check to see if each of their operands are an EXTRACT_VECTOR with 9911 // the same vector and appropriate index. 9912 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) { 9913 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT 9914 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 9915 9916 SDValue ExtVec0 = N0->getOperand(i); 9917 SDValue ExtVec1 = N1->getOperand(i); 9918 9919 // First operand is the vector, verify its the same. 9920 if (V != ExtVec0->getOperand(0).getNode() || 9921 V != ExtVec1->getOperand(0).getNode()) 9922 return SDValue(); 9923 9924 // Second is the constant, verify its correct. 9925 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1)); 9926 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1)); 9927 9928 // For the constant, we want to see all the even or all the odd. 9929 if (!C0 || !C1 || C0->getZExtValue() != nextIndex 9930 || C1->getZExtValue() != nextIndex+1) 9931 return SDValue(); 9932 9933 // Increment index. 9934 nextIndex+=2; 9935 } else 9936 return SDValue(); 9937 } 9938 9939 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure 9940 // we're using the entire input vector, otherwise there's a size/legality 9941 // mismatch somewhere. 9942 if (nextIndex != Vec.getValueType().getVectorNumElements() || 9943 Vec.getValueType().getVectorElementType() == VT.getVectorElementType()) 9944 return SDValue(); 9945 9946 // Create VPADDL node. 9947 SelectionDAG &DAG = DCI.DAG; 9948 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9949 9950 SDLoc dl(N); 9951 9952 // Build operand list. 9953 SmallVector<SDValue, 8> Ops; 9954 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl, 9955 TLI.getPointerTy(DAG.getDataLayout()))); 9956 9957 // Input is the vector. 9958 Ops.push_back(Vec); 9959 9960 // Get widened type and narrowed type. 9961 MVT widenType; 9962 unsigned numElem = VT.getVectorNumElements(); 9963 9964 EVT inputLaneType = Vec.getValueType().getVectorElementType(); 9965 switch (inputLaneType.getSimpleVT().SimpleTy) { 9966 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break; 9967 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break; 9968 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break; 9969 default: 9970 llvm_unreachable("Invalid vector element type for padd optimization."); 9971 } 9972 9973 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops); 9974 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE; 9975 return DAG.getNode(ExtOp, dl, VT, tmp); 9976 } 9977 9978 static SDValue findMUL_LOHI(SDValue V) { 9979 if (V->getOpcode() == ISD::UMUL_LOHI || 9980 V->getOpcode() == ISD::SMUL_LOHI) 9981 return V; 9982 return SDValue(); 9983 } 9984 9985 static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, 9986 TargetLowering::DAGCombinerInfo &DCI, 9987 const ARMSubtarget *Subtarget) { 9988 if (Subtarget->isThumb()) { 9989 if (!Subtarget->hasDSP()) 9990 return SDValue(); 9991 } else if (!Subtarget->hasV5TEOps()) 9992 return SDValue(); 9993 9994 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and 9995 // accumulates the product into a 64-bit value. The 16-bit values will 9996 // be sign extended somehow or SRA'd into 32-bit values 9997 // (addc (adde (mul 16bit, 16bit), lo), hi) 9998 SDValue Mul = AddcNode->getOperand(0); 9999 SDValue Lo = AddcNode->getOperand(1); 10000 if (Mul.getOpcode() != ISD::MUL) { 10001 Lo = AddcNode->getOperand(0); 10002 Mul = AddcNode->getOperand(1); 10003 if (Mul.getOpcode() != ISD::MUL) 10004 return SDValue(); 10005 } 10006 10007 SDValue SRA = AddeNode->getOperand(0); 10008 SDValue Hi = AddeNode->getOperand(1); 10009 if (SRA.getOpcode() != ISD::SRA) { 10010 SRA = AddeNode->getOperand(1); 10011 Hi = AddeNode->getOperand(0); 10012 if (SRA.getOpcode() != ISD::SRA) 10013 return SDValue(); 10014 } 10015 if (auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) { 10016 if (Const->getZExtValue() != 31) 10017 return SDValue(); 10018 } else 10019 return SDValue(); 10020 10021 if (SRA.getOperand(0) != Mul) 10022 return SDValue(); 10023 10024 SelectionDAG &DAG = DCI.DAG; 10025 SDLoc dl(AddcNode); 10026 unsigned Opcode = 0; 10027 SDValue Op0; 10028 SDValue Op1; 10029 10030 if (isS16(Mul.getOperand(0), DAG) && isS16(Mul.getOperand(1), DAG)) { 10031 Opcode = ARMISD::SMLALBB; 10032 Op0 = Mul.getOperand(0); 10033 Op1 = Mul.getOperand(1); 10034 } else if (isS16(Mul.getOperand(0), DAG) && isSRA16(Mul.getOperand(1))) { 10035 Opcode = ARMISD::SMLALBT; 10036 Op0 = Mul.getOperand(0); 10037 Op1 = Mul.getOperand(1).getOperand(0); 10038 } else if (isSRA16(Mul.getOperand(0)) && isS16(Mul.getOperand(1), DAG)) { 10039 Opcode = ARMISD::SMLALTB; 10040 Op0 = Mul.getOperand(0).getOperand(0); 10041 Op1 = Mul.getOperand(1); 10042 } else if (isSRA16(Mul.getOperand(0)) && isSRA16(Mul.getOperand(1))) { 10043 Opcode = ARMISD::SMLALTT; 10044 Op0 = Mul->getOperand(0).getOperand(0); 10045 Op1 = Mul->getOperand(1).getOperand(0); 10046 } 10047 10048 if (!Op0 || !Op1) 10049 return SDValue(); 10050 10051 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32), 10052 Op0, Op1, Lo, Hi); 10053 // Replace the ADDs' nodes uses by the MLA node's values. 10054 SDValue HiMLALResult(SMLAL.getNode(), 1); 10055 SDValue LoMLALResult(SMLAL.getNode(), 0); 10056 10057 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult); 10058 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult); 10059 10060 // Return original node to notify the driver to stop replacing. 10061 SDValue resNode(AddcNode, 0); 10062 return resNode; 10063 } 10064 10065 static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, 10066 TargetLowering::DAGCombinerInfo &DCI, 10067 const ARMSubtarget *Subtarget) { 10068 // Look for multiply add opportunities. 10069 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where 10070 // each add nodes consumes a value from ISD::UMUL_LOHI and there is 10071 // a glue link from the first add to the second add. 10072 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by 10073 // a S/UMLAL instruction. 10074 // UMUL_LOHI 10075 // / :lo \ :hi 10076 // V \ [no multiline comment] 10077 // loAdd -> ADDC | 10078 // \ :carry / 10079 // V V 10080 // ADDE <- hiAdd 10081 // 10082 // In the special case where only the higher part of a signed result is used 10083 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts 10084 // a constant with the exact value of 0x80000000, we recognize we are dealing 10085 // with a "rounded multiply and add" (or subtract) and transform it into 10086 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively. 10087 10088 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE || 10089 AddeSubeNode->getOpcode() == ARMISD::SUBE) && 10090 "Expect an ADDE or SUBE"); 10091 10092 assert(AddeSubeNode->getNumOperands() == 3 && 10093 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 && 10094 "ADDE node has the wrong inputs"); 10095 10096 // Check that we are chained to the right ADDC or SUBC node. 10097 SDNode *AddcSubcNode = AddeSubeNode->getOperand(2).getNode(); 10098 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE && 10099 AddcSubcNode->getOpcode() != ARMISD::ADDC) || 10100 (AddeSubeNode->getOpcode() == ARMISD::SUBE && 10101 AddcSubcNode->getOpcode() != ARMISD::SUBC)) 10102 return SDValue(); 10103 10104 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0); 10105 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1); 10106 10107 // Check if the two operands are from the same mul_lohi node. 10108 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode()) 10109 return SDValue(); 10110 10111 assert(AddcSubcNode->getNumValues() == 2 && 10112 AddcSubcNode->getValueType(0) == MVT::i32 && 10113 "Expect ADDC with two result values. First: i32"); 10114 10115 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it 10116 // maybe a SMLAL which multiplies two 16-bit values. 10117 if (AddeSubeNode->getOpcode() == ARMISD::ADDE && 10118 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI && 10119 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI && 10120 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI && 10121 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI) 10122 return AddCombineTo64BitSMLAL16(AddcSubcNode, AddeSubeNode, DCI, Subtarget); 10123 10124 // Check for the triangle shape. 10125 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0); 10126 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1); 10127 10128 // Make sure that the ADDE/SUBE operands are not coming from the same node. 10129 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode()) 10130 return SDValue(); 10131 10132 // Find the MUL_LOHI node walking up ADDE/SUBE's operands. 10133 bool IsLeftOperandMUL = false; 10134 SDValue MULOp = findMUL_LOHI(AddeSubeOp0); 10135 if (MULOp == SDValue()) 10136 MULOp = findMUL_LOHI(AddeSubeOp1); 10137 else 10138 IsLeftOperandMUL = true; 10139 if (MULOp == SDValue()) 10140 return SDValue(); 10141 10142 // Figure out the right opcode. 10143 unsigned Opc = MULOp->getOpcode(); 10144 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL; 10145 10146 // Figure out the high and low input values to the MLAL node. 10147 SDValue *HiAddSub = nullptr; 10148 SDValue *LoMul = nullptr; 10149 SDValue *LowAddSub = nullptr; 10150 10151 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI. 10152 if ((AddeSubeOp0 != MULOp.getValue(1)) && (AddeSubeOp1 != MULOp.getValue(1))) 10153 return SDValue(); 10154 10155 if (IsLeftOperandMUL) 10156 HiAddSub = &AddeSubeOp1; 10157 else 10158 HiAddSub = &AddeSubeOp0; 10159 10160 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node 10161 // whose low result is fed to the ADDC/SUBC we are checking. 10162 10163 if (AddcSubcOp0 == MULOp.getValue(0)) { 10164 LoMul = &AddcSubcOp0; 10165 LowAddSub = &AddcSubcOp1; 10166 } 10167 if (AddcSubcOp1 == MULOp.getValue(0)) { 10168 LoMul = &AddcSubcOp1; 10169 LowAddSub = &AddcSubcOp0; 10170 } 10171 10172 if (!LoMul) 10173 return SDValue(); 10174 10175 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC 10176 // the replacement below will create a cycle. 10177 if (AddcSubcNode == HiAddSub->getNode() || 10178 AddcSubcNode->isPredecessorOf(HiAddSub->getNode())) 10179 return SDValue(); 10180 10181 // Create the merged node. 10182 SelectionDAG &DAG = DCI.DAG; 10183 10184 // Start building operand list. 10185 SmallVector<SDValue, 8> Ops; 10186 Ops.push_back(LoMul->getOperand(0)); 10187 Ops.push_back(LoMul->getOperand(1)); 10188 10189 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be 10190 // the case, we must be doing signed multiplication and only use the higher 10191 // part of the result of the MLAL, furthermore the LowAddSub must be a constant 10192 // addition or subtraction with the value of 0x800000. 10193 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() && 10194 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(1) && 10195 LowAddSub->getNode()->getOpcode() == ISD::Constant && 10196 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() == 10197 0x80000000) { 10198 Ops.push_back(*HiAddSub); 10199 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) { 10200 FinalOpc = ARMISD::SMMLSR; 10201 } else { 10202 FinalOpc = ARMISD::SMMLAR; 10203 } 10204 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops); 10205 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode); 10206 10207 return SDValue(AddeSubeNode, 0); 10208 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC) 10209 // SMMLS is generated during instruction selection and the rest of this 10210 // function can not handle the case where AddcSubcNode is a SUBC. 10211 return SDValue(); 10212 10213 // Finish building the operand list for {U/S}MLAL 10214 Ops.push_back(*LowAddSub); 10215 Ops.push_back(*HiAddSub); 10216 10217 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), 10218 DAG.getVTList(MVT::i32, MVT::i32), Ops); 10219 10220 // Replace the ADDs' nodes uses by the MLA node's values. 10221 SDValue HiMLALResult(MLALNode.getNode(), 1); 10222 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult); 10223 10224 SDValue LoMLALResult(MLALNode.getNode(), 0); 10225 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult); 10226 10227 // Return original node to notify the driver to stop replacing. 10228 return SDValue(AddeSubeNode, 0); 10229 } 10230 10231 static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, 10232 TargetLowering::DAGCombinerInfo &DCI, 10233 const ARMSubtarget *Subtarget) { 10234 // UMAAL is similar to UMLAL except that it adds two unsigned values. 10235 // While trying to combine for the other MLAL nodes, first search for the 10236 // chance to use UMAAL. Check if Addc uses a node which has already 10237 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde 10238 // as the addend, and it's handled in PerformUMLALCombine. 10239 10240 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) 10241 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget); 10242 10243 // Check that we have a glued ADDC node. 10244 SDNode* AddcNode = AddeNode->getOperand(2).getNode(); 10245 if (AddcNode->getOpcode() != ARMISD::ADDC) 10246 return SDValue(); 10247 10248 // Find the converted UMAAL or quit if it doesn't exist. 10249 SDNode *UmlalNode = nullptr; 10250 SDValue AddHi; 10251 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) { 10252 UmlalNode = AddcNode->getOperand(0).getNode(); 10253 AddHi = AddcNode->getOperand(1); 10254 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) { 10255 UmlalNode = AddcNode->getOperand(1).getNode(); 10256 AddHi = AddcNode->getOperand(0); 10257 } else { 10258 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget); 10259 } 10260 10261 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as 10262 // the ADDC as well as Zero. 10263 if (!isNullConstant(UmlalNode->getOperand(3))) 10264 return SDValue(); 10265 10266 if ((isNullConstant(AddeNode->getOperand(0)) && 10267 AddeNode->getOperand(1).getNode() == UmlalNode) || 10268 (AddeNode->getOperand(0).getNode() == UmlalNode && 10269 isNullConstant(AddeNode->getOperand(1)))) { 10270 SelectionDAG &DAG = DCI.DAG; 10271 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1), 10272 UmlalNode->getOperand(2), AddHi }; 10273 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode), 10274 DAG.getVTList(MVT::i32, MVT::i32), Ops); 10275 10276 // Replace the ADDs' nodes uses by the UMAAL node's values. 10277 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1)); 10278 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0)); 10279 10280 // Return original node to notify the driver to stop replacing. 10281 return SDValue(AddeNode, 0); 10282 } 10283 return SDValue(); 10284 } 10285 10286 static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, 10287 const ARMSubtarget *Subtarget) { 10288 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) 10289 return SDValue(); 10290 10291 // Check that we have a pair of ADDC and ADDE as operands. 10292 // Both addends of the ADDE must be zero. 10293 SDNode* AddcNode = N->getOperand(2).getNode(); 10294 SDNode* AddeNode = N->getOperand(3).getNode(); 10295 if ((AddcNode->getOpcode() == ARMISD::ADDC) && 10296 (AddeNode->getOpcode() == ARMISD::ADDE) && 10297 isNullConstant(AddeNode->getOperand(0)) && 10298 isNullConstant(AddeNode->getOperand(1)) && 10299 (AddeNode->getOperand(2).getNode() == AddcNode)) 10300 return DAG.getNode(ARMISD::UMAAL, SDLoc(N), 10301 DAG.getVTList(MVT::i32, MVT::i32), 10302 {N->getOperand(0), N->getOperand(1), 10303 AddcNode->getOperand(0), AddcNode->getOperand(1)}); 10304 else 10305 return SDValue(); 10306 } 10307 10308 static SDValue PerformAddcSubcCombine(SDNode *N, 10309 TargetLowering::DAGCombinerInfo &DCI, 10310 const ARMSubtarget *Subtarget) { 10311 SelectionDAG &DAG(DCI.DAG); 10312 10313 if (N->getOpcode() == ARMISD::SUBC) { 10314 // (SUBC (ADDE 0, 0, C), 1) -> C 10315 SDValue LHS = N->getOperand(0); 10316 SDValue RHS = N->getOperand(1); 10317 if (LHS->getOpcode() == ARMISD::ADDE && 10318 isNullConstant(LHS->getOperand(0)) && 10319 isNullConstant(LHS->getOperand(1)) && isOneConstant(RHS)) { 10320 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2)); 10321 } 10322 } 10323 10324 if (Subtarget->isThumb1Only()) { 10325 SDValue RHS = N->getOperand(1); 10326 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { 10327 int32_t imm = C->getSExtValue(); 10328 if (imm < 0 && imm > std::numeric_limits<int>::min()) { 10329 SDLoc DL(N); 10330 RHS = DAG.getConstant(-imm, DL, MVT::i32); 10331 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC 10332 : ARMISD::ADDC; 10333 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS); 10334 } 10335 } 10336 } 10337 10338 return SDValue(); 10339 } 10340 10341 static SDValue PerformAddeSubeCombine(SDNode *N, 10342 TargetLowering::DAGCombinerInfo &DCI, 10343 const ARMSubtarget *Subtarget) { 10344 if (Subtarget->isThumb1Only()) { 10345 SelectionDAG &DAG = DCI.DAG; 10346 SDValue RHS = N->getOperand(1); 10347 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { 10348 int64_t imm = C->getSExtValue(); 10349 if (imm < 0) { 10350 SDLoc DL(N); 10351 10352 // The with-carry-in form matches bitwise not instead of the negation. 10353 // Effectively, the inverse interpretation of the carry flag already 10354 // accounts for part of the negation. 10355 RHS = DAG.getConstant(~imm, DL, MVT::i32); 10356 10357 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE 10358 : ARMISD::ADDE; 10359 return DAG.getNode(Opcode, DL, N->getVTList(), 10360 N->getOperand(0), RHS, N->getOperand(2)); 10361 } 10362 } 10363 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) { 10364 return AddCombineTo64bitMLAL(N, DCI, Subtarget); 10365 } 10366 return SDValue(); 10367 } 10368 10369 /// PerformADDECombine - Target-specific dag combine transform from 10370 /// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or 10371 /// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL 10372 static SDValue PerformADDECombine(SDNode *N, 10373 TargetLowering::DAGCombinerInfo &DCI, 10374 const ARMSubtarget *Subtarget) { 10375 // Only ARM and Thumb2 support UMLAL/SMLAL. 10376 if (Subtarget->isThumb1Only()) 10377 return PerformAddeSubeCombine(N, DCI, Subtarget); 10378 10379 // Only perform the checks after legalize when the pattern is available. 10380 if (DCI.isBeforeLegalize()) return SDValue(); 10381 10382 return AddCombineTo64bitUMAAL(N, DCI, Subtarget); 10383 } 10384 10385 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with 10386 /// operands N0 and N1. This is a helper for PerformADDCombine that is 10387 /// called with the default operands, and if that fails, with commuted 10388 /// operands. 10389 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, 10390 TargetLowering::DAGCombinerInfo &DCI, 10391 const ARMSubtarget *Subtarget){ 10392 // Attempt to create vpadd for this add. 10393 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget)) 10394 return Result; 10395 10396 // Attempt to create vpaddl for this add. 10397 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget)) 10398 return Result; 10399 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI, 10400 Subtarget)) 10401 return Result; 10402 10403 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) 10404 if (N0.getNode()->hasOneUse()) 10405 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI)) 10406 return Result; 10407 return SDValue(); 10408 } 10409 10410 static SDValue PerformSHLSimplify(SDNode *N, 10411 TargetLowering::DAGCombinerInfo &DCI, 10412 const ARMSubtarget *ST) { 10413 // Allow the generic combiner to identify potential bswaps. 10414 if (DCI.isBeforeLegalize()) 10415 return SDValue(); 10416 10417 // DAG combiner will fold: 10418 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2) 10419 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2 10420 // Other code patterns that can be also be modified have the following form: 10421 // b + ((a << 1) | 510) 10422 // b + ((a << 1) & 510) 10423 // b + ((a << 1) ^ 510) 10424 // b + ((a << 1) + 510) 10425 10426 // Many instructions can perform the shift for free, but it requires both 10427 // the operands to be registers. If c1 << c2 is too large, a mov immediate 10428 // instruction will needed. So, unfold back to the original pattern if: 10429 // - if c1 and c2 are small enough that they don't require mov imms. 10430 // - the user(s) of the node can perform an shl 10431 10432 // No shifted operands for 16-bit instructions. 10433 if (ST->isThumb() && ST->isThumb1Only()) 10434 return SDValue(); 10435 10436 // Check that all the users could perform the shl themselves. 10437 for (auto U : N->uses()) { 10438 switch(U->getOpcode()) { 10439 default: 10440 return SDValue(); 10441 case ISD::SUB: 10442 case ISD::ADD: 10443 case ISD::AND: 10444 case ISD::OR: 10445 case ISD::XOR: 10446 case ISD::SETCC: 10447 case ARMISD::CMP: 10448 // Check that the user isn't already using a constant because there 10449 // aren't any instructions that support an immediate operand and a 10450 // shifted operand. 10451 if (isa<ConstantSDNode>(U->getOperand(0)) || 10452 isa<ConstantSDNode>(U->getOperand(1))) 10453 return SDValue(); 10454 10455 // Check that it's not already using a shift. 10456 if (U->getOperand(0).getOpcode() == ISD::SHL || 10457 U->getOperand(1).getOpcode() == ISD::SHL) 10458 return SDValue(); 10459 break; 10460 } 10461 } 10462 10463 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR && 10464 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND) 10465 return SDValue(); 10466 10467 if (N->getOperand(0).getOpcode() != ISD::SHL) 10468 return SDValue(); 10469 10470 SDValue SHL = N->getOperand(0); 10471 10472 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10473 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1)); 10474 if (!C1ShlC2 || !C2) 10475 return SDValue(); 10476 10477 APInt C2Int = C2->getAPIntValue(); 10478 APInt C1Int = C1ShlC2->getAPIntValue(); 10479 10480 // Check that performing a lshr will not lose any information. 10481 APInt Mask = APInt::getHighBitsSet(C2Int.getBitWidth(), 10482 C2Int.getBitWidth() - C2->getZExtValue()); 10483 if ((C1Int & Mask) != C1Int) 10484 return SDValue(); 10485 10486 // Shift the first constant. 10487 C1Int.lshrInPlace(C2Int); 10488 10489 // The immediates are encoded as an 8-bit value that can be rotated. 10490 auto LargeImm = [](const APInt &Imm) { 10491 unsigned Zeros = Imm.countLeadingZeros() + Imm.countTrailingZeros(); 10492 return Imm.getBitWidth() - Zeros > 8; 10493 }; 10494 10495 if (LargeImm(C1Int) || LargeImm(C2Int)) 10496 return SDValue(); 10497 10498 SelectionDAG &DAG = DCI.DAG; 10499 SDLoc dl(N); 10500 SDValue X = SHL.getOperand(0); 10501 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X, 10502 DAG.getConstant(C1Int, dl, MVT::i32)); 10503 // Shift left to compensate for the lshr of C1Int. 10504 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1)); 10505 10506 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump(); 10507 SHL.dump(); N->dump()); 10508 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump()); 10509 10510 DAG.ReplaceAllUsesWith(SDValue(N, 0), Res); 10511 return SDValue(N, 0); 10512 } 10513 10514 10515 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD. 10516 /// 10517 static SDValue PerformADDCombine(SDNode *N, 10518 TargetLowering::DAGCombinerInfo &DCI, 10519 const ARMSubtarget *Subtarget) { 10520 SDValue N0 = N->getOperand(0); 10521 SDValue N1 = N->getOperand(1); 10522 10523 // Only works one way, because it needs an immediate operand. 10524 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget)) 10525 return Result; 10526 10527 // First try with the default operand order. 10528 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget)) 10529 return Result; 10530 10531 // If that didn't work, try again with the operands commuted. 10532 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget); 10533 } 10534 10535 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB. 10536 /// 10537 static SDValue PerformSUBCombine(SDNode *N, 10538 TargetLowering::DAGCombinerInfo &DCI) { 10539 SDValue N0 = N->getOperand(0); 10540 SDValue N1 = N->getOperand(1); 10541 10542 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) 10543 if (N1.getNode()->hasOneUse()) 10544 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI)) 10545 return Result; 10546 10547 return SDValue(); 10548 } 10549 10550 /// PerformVMULCombine 10551 /// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the 10552 /// special multiplier accumulator forwarding. 10553 /// vmul d3, d0, d2 10554 /// vmla d3, d1, d2 10555 /// is faster than 10556 /// vadd d3, d0, d1 10557 /// vmul d3, d3, d2 10558 // However, for (A + B) * (A + B), 10559 // vadd d2, d0, d1 10560 // vmul d3, d0, d2 10561 // vmla d3, d1, d2 10562 // is slower than 10563 // vadd d2, d0, d1 10564 // vmul d3, d2, d2 10565 static SDValue PerformVMULCombine(SDNode *N, 10566 TargetLowering::DAGCombinerInfo &DCI, 10567 const ARMSubtarget *Subtarget) { 10568 if (!Subtarget->hasVMLxForwarding()) 10569 return SDValue(); 10570 10571 SelectionDAG &DAG = DCI.DAG; 10572 SDValue N0 = N->getOperand(0); 10573 SDValue N1 = N->getOperand(1); 10574 unsigned Opcode = N0.getOpcode(); 10575 if (Opcode != ISD::ADD && Opcode != ISD::SUB && 10576 Opcode != ISD::FADD && Opcode != ISD::FSUB) { 10577 Opcode = N1.getOpcode(); 10578 if (Opcode != ISD::ADD && Opcode != ISD::SUB && 10579 Opcode != ISD::FADD && Opcode != ISD::FSUB) 10580 return SDValue(); 10581 std::swap(N0, N1); 10582 } 10583 10584 if (N0 == N1) 10585 return SDValue(); 10586 10587 EVT VT = N->getValueType(0); 10588 SDLoc DL(N); 10589 SDValue N00 = N0->getOperand(0); 10590 SDValue N01 = N0->getOperand(1); 10591 return DAG.getNode(Opcode, DL, VT, 10592 DAG.getNode(ISD::MUL, DL, VT, N00, N1), 10593 DAG.getNode(ISD::MUL, DL, VT, N01, N1)); 10594 } 10595 10596 static SDValue PerformMULCombine(SDNode *N, 10597 TargetLowering::DAGCombinerInfo &DCI, 10598 const ARMSubtarget *Subtarget) { 10599 SelectionDAG &DAG = DCI.DAG; 10600 10601 if (Subtarget->isThumb1Only()) 10602 return SDValue(); 10603 10604 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 10605 return SDValue(); 10606 10607 EVT VT = N->getValueType(0); 10608 if (VT.is64BitVector() || VT.is128BitVector()) 10609 return PerformVMULCombine(N, DCI, Subtarget); 10610 if (VT != MVT::i32) 10611 return SDValue(); 10612 10613 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10614 if (!C) 10615 return SDValue(); 10616 10617 int64_t MulAmt = C->getSExtValue(); 10618 unsigned ShiftAmt = countTrailingZeros<uint64_t>(MulAmt); 10619 10620 ShiftAmt = ShiftAmt & (32 - 1); 10621 SDValue V = N->getOperand(0); 10622 SDLoc DL(N); 10623 10624 SDValue Res; 10625 MulAmt >>= ShiftAmt; 10626 10627 if (MulAmt >= 0) { 10628 if (isPowerOf2_32(MulAmt - 1)) { 10629 // (mul x, 2^N + 1) => (add (shl x, N), x) 10630 Res = DAG.getNode(ISD::ADD, DL, VT, 10631 V, 10632 DAG.getNode(ISD::SHL, DL, VT, 10633 V, 10634 DAG.getConstant(Log2_32(MulAmt - 1), DL, 10635 MVT::i32))); 10636 } else if (isPowerOf2_32(MulAmt + 1)) { 10637 // (mul x, 2^N - 1) => (sub (shl x, N), x) 10638 Res = DAG.getNode(ISD::SUB, DL, VT, 10639 DAG.getNode(ISD::SHL, DL, VT, 10640 V, 10641 DAG.getConstant(Log2_32(MulAmt + 1), DL, 10642 MVT::i32)), 10643 V); 10644 } else 10645 return SDValue(); 10646 } else { 10647 uint64_t MulAmtAbs = -MulAmt; 10648 if (isPowerOf2_32(MulAmtAbs + 1)) { 10649 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) 10650 Res = DAG.getNode(ISD::SUB, DL, VT, 10651 V, 10652 DAG.getNode(ISD::SHL, DL, VT, 10653 V, 10654 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL, 10655 MVT::i32))); 10656 } else if (isPowerOf2_32(MulAmtAbs - 1)) { 10657 // (mul x, -(2^N + 1)) => - (add (shl x, N), x) 10658 Res = DAG.getNode(ISD::ADD, DL, VT, 10659 V, 10660 DAG.getNode(ISD::SHL, DL, VT, 10661 V, 10662 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL, 10663 MVT::i32))); 10664 Res = DAG.getNode(ISD::SUB, DL, VT, 10665 DAG.getConstant(0, DL, MVT::i32), Res); 10666 } else 10667 return SDValue(); 10668 } 10669 10670 if (ShiftAmt != 0) 10671 Res = DAG.getNode(ISD::SHL, DL, VT, 10672 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32)); 10673 10674 // Do not add new nodes to DAG combiner worklist. 10675 DCI.CombineTo(N, Res, false); 10676 return SDValue(); 10677 } 10678 10679 static SDValue CombineANDShift(SDNode *N, 10680 TargetLowering::DAGCombinerInfo &DCI, 10681 const ARMSubtarget *Subtarget) { 10682 // Allow DAGCombine to pattern-match before we touch the canonical form. 10683 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 10684 return SDValue(); 10685 10686 if (N->getValueType(0) != MVT::i32) 10687 return SDValue(); 10688 10689 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10690 if (!N1C) 10691 return SDValue(); 10692 10693 uint32_t C1 = (uint32_t)N1C->getZExtValue(); 10694 // Don't transform uxtb/uxth. 10695 if (C1 == 255 || C1 == 65535) 10696 return SDValue(); 10697 10698 SDNode *N0 = N->getOperand(0).getNode(); 10699 if (!N0->hasOneUse()) 10700 return SDValue(); 10701 10702 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL) 10703 return SDValue(); 10704 10705 bool LeftShift = N0->getOpcode() == ISD::SHL; 10706 10707 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 10708 if (!N01C) 10709 return SDValue(); 10710 10711 uint32_t C2 = (uint32_t)N01C->getZExtValue(); 10712 if (!C2 || C2 >= 32) 10713 return SDValue(); 10714 10715 SelectionDAG &DAG = DCI.DAG; 10716 SDLoc DL(N); 10717 10718 // We have a pattern of the form "(and (shl x, c2) c1)" or 10719 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to 10720 // transform to a pair of shifts, to save materializing c1. 10721 10722 // First pattern: right shift, and c1+1 is a power of two. 10723 // FIXME: Also check reversed pattern (left shift, and ~c1+1 is a power 10724 // of two). 10725 // FIXME: Use demanded bits? 10726 if (!LeftShift && isMask_32(C1)) { 1072