1 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines the interfaces that ARM uses to lower LLVM code into a 11 // selection DAG. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "ARMISelLowering.h" 16 #include "ARMBaseInstrInfo.h" 17 #include "ARMBaseRegisterInfo.h" 18 #include "ARMCallingConv.h" 19 #include "ARMConstantPoolValue.h" 20 #include "ARMMachineFunctionInfo.h" 21 #include "ARMPerfectShuffle.h" 22 #include "ARMRegisterInfo.h" 23 #include "ARMSelectionDAGInfo.h" 24 #include "ARMSubtarget.h" 25 #include "MCTargetDesc/ARMAddressingModes.h" 26 #include "MCTargetDesc/ARMBaseInfo.h" 27 #include "Utils/ARMBaseInfo.h" 28 #include "llvm/ADT/APFloat.h" 29 #include "llvm/ADT/APInt.h" 30 #include "llvm/ADT/ArrayRef.h" 31 #include "llvm/ADT/BitVector.h" 32 #include "llvm/ADT/DenseMap.h" 33 #include "llvm/ADT/STLExtras.h" 34 #include "llvm/ADT/SmallPtrSet.h" 35 #include "llvm/ADT/SmallVector.h" 36 #include "llvm/ADT/Statistic.h" 37 #include "llvm/ADT/StringExtras.h" 38 #include "llvm/ADT/StringRef.h" 39 #include "llvm/ADT/StringSwitch.h" 40 #include "llvm/ADT/Triple.h" 41 #include "llvm/ADT/Twine.h" 42 #include "llvm/Analysis/VectorUtils.h" 43 #include "llvm/CodeGen/CallingConvLower.h" 44 #include "llvm/CodeGen/ISDOpcodes.h" 45 #include "llvm/CodeGen/IntrinsicLowering.h" 46 #include "llvm/CodeGen/MachineBasicBlock.h" 47 #include "llvm/CodeGen/MachineConstantPool.h" 48 #include "llvm/CodeGen/MachineFrameInfo.h" 49 #include "llvm/CodeGen/MachineFunction.h" 50 #include "llvm/CodeGen/MachineInstr.h" 51 #include "llvm/CodeGen/MachineInstrBuilder.h" 52 #include "llvm/CodeGen/MachineJumpTableInfo.h" 53 #include "llvm/CodeGen/MachineMemOperand.h" 54 #include "llvm/CodeGen/MachineOperand.h" 55 #include "llvm/CodeGen/MachineRegisterInfo.h" 56 #include "llvm/CodeGen/RuntimeLibcalls.h" 57 #include "llvm/CodeGen/SelectionDAG.h" 58 #include "llvm/CodeGen/SelectionDAGNodes.h" 59 #include "llvm/CodeGen/TargetInstrInfo.h" 60 #include "llvm/CodeGen/TargetLowering.h" 61 #include "llvm/CodeGen/TargetOpcodes.h" 62 #include "llvm/CodeGen/TargetRegisterInfo.h" 63 #include "llvm/CodeGen/TargetSubtargetInfo.h" 64 #include "llvm/CodeGen/ValueTypes.h" 65 #include "llvm/IR/Attributes.h" 66 #include "llvm/IR/CallingConv.h" 67 #include "llvm/IR/Constant.h" 68 #include "llvm/IR/Constants.h" 69 #include "llvm/IR/DataLayout.h" 70 #include "llvm/IR/DebugLoc.h" 71 #include "llvm/IR/DerivedTypes.h" 72 #include "llvm/IR/Function.h" 73 #include "llvm/IR/GlobalAlias.h" 74 #include "llvm/IR/GlobalValue.h" 75 #include "llvm/IR/GlobalVariable.h" 76 #include "llvm/IR/IRBuilder.h" 77 #include "llvm/IR/InlineAsm.h" 78 #include "llvm/IR/Instruction.h" 79 #include "llvm/IR/Instructions.h" 80 #include "llvm/IR/IntrinsicInst.h" 81 #include "llvm/IR/Intrinsics.h" 82 #include "llvm/IR/Module.h" 83 #include "llvm/IR/Type.h" 84 #include "llvm/IR/User.h" 85 #include "llvm/IR/Value.h" 86 #include "llvm/MC/MCInstrDesc.h" 87 #include "llvm/MC/MCInstrItineraries.h" 88 #include "llvm/MC/MCRegisterInfo.h" 89 #include "llvm/MC/MCSchedule.h" 90 #include "llvm/Support/AtomicOrdering.h" 91 #include "llvm/Support/BranchProbability.h" 92 #include "llvm/Support/Casting.h" 93 #include "llvm/Support/CodeGen.h" 94 #include "llvm/Support/CommandLine.h" 95 #include "llvm/Support/Compiler.h" 96 #include "llvm/Support/Debug.h" 97 #include "llvm/Support/ErrorHandling.h" 98 #include "llvm/Support/KnownBits.h" 99 #include "llvm/Support/MachineValueType.h" 100 #include "llvm/Support/MathExtras.h" 101 #include "llvm/Support/raw_ostream.h" 102 #include "llvm/Target/TargetMachine.h" 103 #include "llvm/Target/TargetOptions.h" 104 #include <algorithm> 105 #include <cassert> 106 #include <cstdint> 107 #include <cstdlib> 108 #include <iterator> 109 #include <limits> 110 #include <string> 111 #include <tuple> 112 #include <utility> 113 #include <vector> 114 115 using namespace llvm; 116 117 #define DEBUG_TYPE "arm-isel" 118 119 STATISTIC(NumTailCalls, "Number of tail calls"); 120 STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt"); 121 STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments"); 122 STATISTIC(NumConstpoolPromoted, 123 "Number of constants with their storage promoted into constant pools"); 124 125 static cl::opt<bool> 126 ARMInterworking("arm-interworking", cl::Hidden, 127 cl::desc("Enable / disable ARM interworking (for debugging only)"), 128 cl::init(true)); 129 130 static cl::opt<bool> EnableConstpoolPromotion( 131 "arm-promote-constant", cl::Hidden, 132 cl::desc("Enable / disable promotion of unnamed_addr constants into " 133 "constant pools"), 134 cl::init(false)); // FIXME: set to true by default once PR32780 is fixed 135 static cl::opt<unsigned> ConstpoolPromotionMaxSize( 136 "arm-promote-constant-max-size", cl::Hidden, 137 cl::desc("Maximum size of constant to promote into a constant pool"), 138 cl::init(64)); 139 static cl::opt<unsigned> ConstpoolPromotionMaxTotal( 140 "arm-promote-constant-max-total", cl::Hidden, 141 cl::desc("Maximum size of ALL constants to promote into a constant pool"), 142 cl::init(128)); 143 144 // The APCS parameter registers. 145 static const MCPhysReg GPRArgRegs[] = { 146 ARM::R0, ARM::R1, ARM::R2, ARM::R3 147 }; 148 149 void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT, 150 MVT PromotedBitwiseVT) { 151 if (VT != PromotedLdStVT) { 152 setOperationAction(ISD::LOAD, VT, Promote); 153 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT); 154 155 setOperationAction(ISD::STORE, VT, Promote); 156 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT); 157 } 158 159 MVT ElemTy = VT.getVectorElementType(); 160 if (ElemTy != MVT::f64) 161 setOperationAction(ISD::SETCC, VT, Custom); 162 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); 163 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 164 if (ElemTy == MVT::i32) { 165 setOperationAction(ISD::SINT_TO_FP, VT, Custom); 166 setOperationAction(ISD::UINT_TO_FP, VT, Custom); 167 setOperationAction(ISD::FP_TO_SINT, VT, Custom); 168 setOperationAction(ISD::FP_TO_UINT, VT, Custom); 169 } else { 170 setOperationAction(ISD::SINT_TO_FP, VT, Expand); 171 setOperationAction(ISD::UINT_TO_FP, VT, Expand); 172 setOperationAction(ISD::FP_TO_SINT, VT, Expand); 173 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 174 } 175 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 176 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 177 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); 178 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); 179 setOperationAction(ISD::SELECT, VT, Expand); 180 setOperationAction(ISD::SELECT_CC, VT, Expand); 181 setOperationAction(ISD::VSELECT, VT, Expand); 182 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 183 if (VT.isInteger()) { 184 setOperationAction(ISD::SHL, VT, Custom); 185 setOperationAction(ISD::SRA, VT, Custom); 186 setOperationAction(ISD::SRL, VT, Custom); 187 } 188 189 // Promote all bit-wise operations. 190 if (VT.isInteger() && VT != PromotedBitwiseVT) { 191 setOperationAction(ISD::AND, VT, Promote); 192 AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT); 193 setOperationAction(ISD::OR, VT, Promote); 194 AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT); 195 setOperationAction(ISD::XOR, VT, Promote); 196 AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT); 197 } 198 199 // Neon does not support vector divide/remainder operations. 200 setOperationAction(ISD::SDIV, VT, Expand); 201 setOperationAction(ISD::UDIV, VT, Expand); 202 setOperationAction(ISD::FDIV, VT, Expand); 203 setOperationAction(ISD::SREM, VT, Expand); 204 setOperationAction(ISD::UREM, VT, Expand); 205 setOperationAction(ISD::FREM, VT, Expand); 206 207 if (!VT.isFloatingPoint() && 208 VT != MVT::v2i64 && VT != MVT::v1i64) 209 for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) 210 setOperationAction(Opcode, VT, Legal); 211 } 212 213 void ARMTargetLowering::addDRTypeForNEON(MVT VT) { 214 addRegisterClass(VT, &ARM::DPRRegClass); 215 addTypeForNEON(VT, MVT::f64, MVT::v2i32); 216 } 217 218 void ARMTargetLowering::addQRTypeForNEON(MVT VT) { 219 addRegisterClass(VT, &ARM::DPairRegClass); 220 addTypeForNEON(VT, MVT::v2f64, MVT::v4i32); 221 } 222 223 ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, 224 const ARMSubtarget &STI) 225 : TargetLowering(TM), Subtarget(&STI) { 226 RegInfo = Subtarget->getRegisterInfo(); 227 Itins = Subtarget->getInstrItineraryData(); 228 229 setBooleanContents(ZeroOrOneBooleanContent); 230 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 231 232 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() && 233 !Subtarget->isTargetWatchOS()) { 234 bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard; 235 for (int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID) 236 setLibcallCallingConv(static_cast<RTLIB::Libcall>(LCID), 237 IsHFTarget ? CallingConv::ARM_AAPCS_VFP 238 : CallingConv::ARM_AAPCS); 239 } 240 241 if (Subtarget->isTargetMachO()) { 242 // Uses VFP for Thumb libfuncs if available. 243 if (Subtarget->isThumb() && Subtarget->hasVFP2() && 244 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) { 245 static const struct { 246 const RTLIB::Libcall Op; 247 const char * const Name; 248 const ISD::CondCode Cond; 249 } LibraryCalls[] = { 250 // Single-precision floating-point arithmetic. 251 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID }, 252 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID }, 253 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID }, 254 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID }, 255 256 // Double-precision floating-point arithmetic. 257 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID }, 258 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID }, 259 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID }, 260 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID }, 261 262 // Single-precision comparisons. 263 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE }, 264 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE }, 265 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE }, 266 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE }, 267 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE }, 268 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE }, 269 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE }, 270 { RTLIB::O_F32, "__unordsf2vfp", ISD::SETEQ }, 271 272 // Double-precision comparisons. 273 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE }, 274 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE }, 275 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE }, 276 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE }, 277 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE }, 278 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE }, 279 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE }, 280 { RTLIB::O_F64, "__unorddf2vfp", ISD::SETEQ }, 281 282 // Floating-point to integer conversions. 283 // i64 conversions are done via library routines even when generating VFP 284 // instructions, so use the same ones. 285 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID }, 286 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID }, 287 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID }, 288 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID }, 289 290 // Conversions between floating types. 291 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID }, 292 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID }, 293 294 // Integer to floating-point conversions. 295 // i64 conversions are done via library routines even when generating VFP 296 // instructions, so use the same ones. 297 // FIXME: There appears to be some naming inconsistency in ARM libgcc: 298 // e.g., __floatunsidf vs. __floatunssidfvfp. 299 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID }, 300 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID }, 301 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID }, 302 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID }, 303 }; 304 305 for (const auto &LC : LibraryCalls) { 306 setLibcallName(LC.Op, LC.Name); 307 if (LC.Cond != ISD::SETCC_INVALID) 308 setCmpLibcallCC(LC.Op, LC.Cond); 309 } 310 } 311 } 312 313 // These libcalls are not available in 32-bit. 314 setLibcallName(RTLIB::SHL_I128, nullptr); 315 setLibcallName(RTLIB::SRL_I128, nullptr); 316 setLibcallName(RTLIB::SRA_I128, nullptr); 317 318 // RTLIB 319 if (Subtarget->isAAPCS_ABI() && 320 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() || 321 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) { 322 static const struct { 323 const RTLIB::Libcall Op; 324 const char * const Name; 325 const CallingConv::ID CC; 326 const ISD::CondCode Cond; 327 } LibraryCalls[] = { 328 // Double-precision floating-point arithmetic helper functions 329 // RTABI chapter 4.1.2, Table 2 330 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 331 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 332 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 333 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 334 335 // Double-precision floating-point comparison helper functions 336 // RTABI chapter 4.1.2, Table 3 337 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE }, 338 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ }, 339 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE }, 340 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE }, 341 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE }, 342 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE }, 343 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE }, 344 { RTLIB::O_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ }, 345 346 // Single-precision floating-point arithmetic helper functions 347 // RTABI chapter 4.1.2, Table 4 348 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 349 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 350 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 351 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 352 353 // Single-precision floating-point comparison helper functions 354 // RTABI chapter 4.1.2, Table 5 355 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE }, 356 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ }, 357 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE }, 358 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE }, 359 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE }, 360 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE }, 361 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE }, 362 { RTLIB::O_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ }, 363 364 // Floating-point to integer conversions. 365 // RTABI chapter 4.1.2, Table 6 366 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 367 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 368 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 369 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 370 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 371 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 372 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 373 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 374 375 // Conversions between floating types. 376 // RTABI chapter 4.1.2, Table 7 377 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 378 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 379 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 380 381 // Integer to floating-point conversions. 382 // RTABI chapter 4.1.2, Table 8 383 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 384 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 385 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 386 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 387 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 388 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 389 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 390 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 391 392 // Long long helper functions 393 // RTABI chapter 4.2, Table 9 394 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 395 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 396 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 397 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 398 399 // Integer division functions 400 // RTABI chapter 4.3.1 401 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 402 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 403 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 404 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 405 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 406 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 407 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 408 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 409 }; 410 411 for (const auto &LC : LibraryCalls) { 412 setLibcallName(LC.Op, LC.Name); 413 setLibcallCallingConv(LC.Op, LC.CC); 414 if (LC.Cond != ISD::SETCC_INVALID) 415 setCmpLibcallCC(LC.Op, LC.Cond); 416 } 417 418 // EABI dependent RTLIB 419 if (TM.Options.EABIVersion == EABI::EABI4 || 420 TM.Options.EABIVersion == EABI::EABI5) { 421 static const struct { 422 const RTLIB::Libcall Op; 423 const char *const Name; 424 const CallingConv::ID CC; 425 const ISD::CondCode Cond; 426 } MemOpsLibraryCalls[] = { 427 // Memory operations 428 // RTABI chapter 4.3.4 429 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 430 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 431 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, 432 }; 433 434 for (const auto &LC : MemOpsLibraryCalls) { 435 setLibcallName(LC.Op, LC.Name); 436 setLibcallCallingConv(LC.Op, LC.CC); 437 if (LC.Cond != ISD::SETCC_INVALID) 438 setCmpLibcallCC(LC.Op, LC.Cond); 439 } 440 } 441 } 442 443 if (Subtarget->isTargetWindows()) { 444 static const struct { 445 const RTLIB::Libcall Op; 446 const char * const Name; 447 const CallingConv::ID CC; 448 } LibraryCalls[] = { 449 { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP }, 450 { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP }, 451 { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP }, 452 { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP }, 453 { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP }, 454 { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP }, 455 { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP }, 456 { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP }, 457 }; 458 459 for (const auto &LC : LibraryCalls) { 460 setLibcallName(LC.Op, LC.Name); 461 setLibcallCallingConv(LC.Op, LC.CC); 462 } 463 } 464 465 // Use divmod compiler-rt calls for iOS 5.0 and later. 466 if (Subtarget->isTargetMachO() && 467 !(Subtarget->isTargetIOS() && 468 Subtarget->getTargetTriple().isOSVersionLT(5, 0))) { 469 setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4"); 470 setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4"); 471 } 472 473 // The half <-> float conversion functions are always soft-float on 474 // non-watchos platforms, but are needed for some targets which use a 475 // hard-float calling convention by default. 476 if (!Subtarget->isTargetWatchABI()) { 477 if (Subtarget->isAAPCS_ABI()) { 478 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS); 479 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS); 480 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS); 481 } else { 482 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS); 483 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS); 484 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS); 485 } 486 } 487 488 // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have 489 // a __gnu_ prefix (which is the default). 490 if (Subtarget->isTargetAEABI()) { 491 static const struct { 492 const RTLIB::Libcall Op; 493 const char * const Name; 494 const CallingConv::ID CC; 495 } LibraryCalls[] = { 496 { RTLIB::FPROUND_F32_F16, "__aeabi_f2h", CallingConv::ARM_AAPCS }, 497 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS }, 498 { RTLIB::FPEXT_F16_F32, "__aeabi_h2f", CallingConv::ARM_AAPCS }, 499 }; 500 501 for (const auto &LC : LibraryCalls) { 502 setLibcallName(LC.Op, LC.Name); 503 setLibcallCallingConv(LC.Op, LC.CC); 504 } 505 } 506 507 if (Subtarget->isThumb1Only()) 508 addRegisterClass(MVT::i32, &ARM::tGPRRegClass); 509 else 510 addRegisterClass(MVT::i32, &ARM::GPRRegClass); 511 512 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && 513 !Subtarget->isThumb1Only()) { 514 addRegisterClass(MVT::f32, &ARM::SPRRegClass); 515 addRegisterClass(MVT::f64, &ARM::DPRRegClass); 516 } 517 518 if (Subtarget->hasFullFP16()) { 519 addRegisterClass(MVT::f16, &ARM::HPRRegClass); 520 setOperationAction(ISD::BITCAST, MVT::i16, Custom); 521 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 522 setOperationAction(ISD::BITCAST, MVT::f16, Custom); 523 524 setOperationAction(ISD::FMINNUM, MVT::f16, Legal); 525 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal); 526 } 527 528 for (MVT VT : MVT::vector_valuetypes()) { 529 for (MVT InnerVT : MVT::vector_valuetypes()) { 530 setTruncStoreAction(VT, InnerVT, Expand); 531 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 532 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 533 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 534 } 535 536 setOperationAction(ISD::MULHS, VT, Expand); 537 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 538 setOperationAction(ISD::MULHU, VT, Expand); 539 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 540 541 setOperationAction(ISD::BSWAP, VT, Expand); 542 } 543 544 setOperationAction(ISD::ConstantFP, MVT::f32, Custom); 545 setOperationAction(ISD::ConstantFP, MVT::f64, Custom); 546 547 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom); 548 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom); 549 550 if (Subtarget->hasNEON()) { 551 addDRTypeForNEON(MVT::v2f32); 552 addDRTypeForNEON(MVT::v8i8); 553 addDRTypeForNEON(MVT::v4i16); 554 addDRTypeForNEON(MVT::v2i32); 555 addDRTypeForNEON(MVT::v1i64); 556 557 addQRTypeForNEON(MVT::v4f32); 558 addQRTypeForNEON(MVT::v2f64); 559 addQRTypeForNEON(MVT::v16i8); 560 addQRTypeForNEON(MVT::v8i16); 561 addQRTypeForNEON(MVT::v4i32); 562 addQRTypeForNEON(MVT::v2i64); 563 564 if (Subtarget->hasFullFP16()) { 565 addQRTypeForNEON(MVT::v8f16); 566 addDRTypeForNEON(MVT::v4f16); 567 } 568 569 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but 570 // neither Neon nor VFP support any arithmetic operations on it. 571 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively 572 // supported for v4f32. 573 setOperationAction(ISD::FADD, MVT::v2f64, Expand); 574 setOperationAction(ISD::FSUB, MVT::v2f64, Expand); 575 setOperationAction(ISD::FMUL, MVT::v2f64, Expand); 576 // FIXME: Code duplication: FDIV and FREM are expanded always, see 577 // ARMTargetLowering::addTypeForNEON method for details. 578 setOperationAction(ISD::FDIV, MVT::v2f64, Expand); 579 setOperationAction(ISD::FREM, MVT::v2f64, Expand); 580 // FIXME: Create unittest. 581 // In another words, find a way when "copysign" appears in DAG with vector 582 // operands. 583 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand); 584 // FIXME: Code duplication: SETCC has custom operation action, see 585 // ARMTargetLowering::addTypeForNEON method for details. 586 setOperationAction(ISD::SETCC, MVT::v2f64, Expand); 587 // FIXME: Create unittest for FNEG and for FABS. 588 setOperationAction(ISD::FNEG, MVT::v2f64, Expand); 589 setOperationAction(ISD::FABS, MVT::v2f64, Expand); 590 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand); 591 setOperationAction(ISD::FSIN, MVT::v2f64, Expand); 592 setOperationAction(ISD::FCOS, MVT::v2f64, Expand); 593 setOperationAction(ISD::FPOW, MVT::v2f64, Expand); 594 setOperationAction(ISD::FLOG, MVT::v2f64, Expand); 595 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand); 596 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand); 597 setOperationAction(ISD::FEXP, MVT::v2f64, Expand); 598 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand); 599 // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR. 600 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand); 601 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand); 602 setOperationAction(ISD::FRINT, MVT::v2f64, Expand); 603 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand); 604 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand); 605 setOperationAction(ISD::FMA, MVT::v2f64, Expand); 606 607 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); 608 setOperationAction(ISD::FSIN, MVT::v4f32, Expand); 609 setOperationAction(ISD::FCOS, MVT::v4f32, Expand); 610 setOperationAction(ISD::FPOW, MVT::v4f32, Expand); 611 setOperationAction(ISD::FLOG, MVT::v4f32, Expand); 612 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand); 613 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand); 614 setOperationAction(ISD::FEXP, MVT::v4f32, Expand); 615 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand); 616 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand); 617 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand); 618 setOperationAction(ISD::FRINT, MVT::v4f32, Expand); 619 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); 620 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand); 621 622 // Mark v2f32 intrinsics. 623 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand); 624 setOperationAction(ISD::FSIN, MVT::v2f32, Expand); 625 setOperationAction(ISD::FCOS, MVT::v2f32, Expand); 626 setOperationAction(ISD::FPOW, MVT::v2f32, Expand); 627 setOperationAction(ISD::FLOG, MVT::v2f32, Expand); 628 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand); 629 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand); 630 setOperationAction(ISD::FEXP, MVT::v2f32, Expand); 631 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand); 632 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand); 633 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand); 634 setOperationAction(ISD::FRINT, MVT::v2f32, Expand); 635 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand); 636 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand); 637 638 // Neon does not support some operations on v1i64 and v2i64 types. 639 setOperationAction(ISD::MUL, MVT::v1i64, Expand); 640 // Custom handling for some quad-vector types to detect VMULL. 641 setOperationAction(ISD::MUL, MVT::v8i16, Custom); 642 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 643 setOperationAction(ISD::MUL, MVT::v2i64, Custom); 644 // Custom handling for some vector types to avoid expensive expansions 645 setOperationAction(ISD::SDIV, MVT::v4i16, Custom); 646 setOperationAction(ISD::SDIV, MVT::v8i8, Custom); 647 setOperationAction(ISD::UDIV, MVT::v4i16, Custom); 648 setOperationAction(ISD::UDIV, MVT::v8i8, Custom); 649 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with 650 // a destination type that is wider than the source, and nor does 651 // it have a FP_TO_[SU]INT instruction with a narrower destination than 652 // source. 653 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); 654 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); 655 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom); 656 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom); 657 658 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); 659 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand); 660 661 // NEON does not have single instruction CTPOP for vectors with element 662 // types wider than 8-bits. However, custom lowering can leverage the 663 // v8i8/v16i8 vcnt instruction. 664 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom); 665 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); 666 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom); 667 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom); 668 setOperationAction(ISD::CTPOP, MVT::v1i64, Expand); 669 setOperationAction(ISD::CTPOP, MVT::v2i64, Expand); 670 671 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); 672 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); 673 674 // NEON does not have single instruction CTTZ for vectors. 675 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom); 676 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom); 677 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom); 678 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom); 679 680 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom); 681 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom); 682 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom); 683 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom); 684 685 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom); 686 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom); 687 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom); 688 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom); 689 690 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom); 691 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom); 692 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); 693 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom); 694 695 // NEON only has FMA instructions as of VFP4. 696 if (!Subtarget->hasVFP4()) { 697 setOperationAction(ISD::FMA, MVT::v2f32, Expand); 698 setOperationAction(ISD::FMA, MVT::v4f32, Expand); 699 } 700 701 setTargetDAGCombine(ISD::INTRINSIC_VOID); 702 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 703 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 704 setTargetDAGCombine(ISD::SHL); 705 setTargetDAGCombine(ISD::SRL); 706 setTargetDAGCombine(ISD::SRA); 707 setTargetDAGCombine(ISD::SIGN_EXTEND); 708 setTargetDAGCombine(ISD::ZERO_EXTEND); 709 setTargetDAGCombine(ISD::ANY_EXTEND); 710 setTargetDAGCombine(ISD::BUILD_VECTOR); 711 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 712 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 713 setTargetDAGCombine(ISD::STORE); 714 setTargetDAGCombine(ISD::FP_TO_SINT); 715 setTargetDAGCombine(ISD::FP_TO_UINT); 716 setTargetDAGCombine(ISD::FDIV); 717 setTargetDAGCombine(ISD::LOAD); 718 719 // It is legal to extload from v4i8 to v4i16 or v4i32. 720 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16, 721 MVT::v2i32}) { 722 for (MVT VT : MVT::integer_vector_valuetypes()) { 723 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal); 724 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal); 725 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal); 726 } 727 } 728 } 729 730 if (Subtarget->isFPOnlySP()) { 731 // When targeting a floating-point unit with only single-precision 732 // operations, f64 is legal for the few double-precision instructions which 733 // are present However, no double-precision operations other than moves, 734 // loads and stores are provided by the hardware. 735 setOperationAction(ISD::FADD, MVT::f64, Expand); 736 setOperationAction(ISD::FSUB, MVT::f64, Expand); 737 setOperationAction(ISD::FMUL, MVT::f64, Expand); 738 setOperationAction(ISD::FMA, MVT::f64, Expand); 739 setOperationAction(ISD::FDIV, MVT::f64, Expand); 740 setOperationAction(ISD::FREM, MVT::f64, Expand); 741 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 742 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand); 743 setOperationAction(ISD::FNEG, MVT::f64, Expand); 744 setOperationAction(ISD::FABS, MVT::f64, Expand); 745 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 746 setOperationAction(ISD::FSIN, MVT::f64, Expand); 747 setOperationAction(ISD::FCOS, MVT::f64, Expand); 748 setOperationAction(ISD::FPOW, MVT::f64, Expand); 749 setOperationAction(ISD::FLOG, MVT::f64, Expand); 750 setOperationAction(ISD::FLOG2, MVT::f64, Expand); 751 setOperationAction(ISD::FLOG10, MVT::f64, Expand); 752 setOperationAction(ISD::FEXP, MVT::f64, Expand); 753 setOperationAction(ISD::FEXP2, MVT::f64, Expand); 754 setOperationAction(ISD::FCEIL, MVT::f64, Expand); 755 setOperationAction(ISD::FTRUNC, MVT::f64, Expand); 756 setOperationAction(ISD::FRINT, MVT::f64, Expand); 757 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); 758 setOperationAction(ISD::FFLOOR, MVT::f64, Expand); 759 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 760 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 761 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 762 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 763 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom); 764 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom); 765 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); 766 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom); 767 } 768 769 computeRegisterProperties(Subtarget->getRegisterInfo()); 770 771 // ARM does not have floating-point extending loads. 772 for (MVT VT : MVT::fp_valuetypes()) { 773 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); 774 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); 775 } 776 777 // ... or truncating stores 778 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 779 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 780 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 781 782 // ARM does not have i1 sign extending load. 783 for (MVT VT : MVT::integer_valuetypes()) 784 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 785 786 // ARM supports all 4 flavors of integer indexed load / store. 787 if (!Subtarget->isThumb1Only()) { 788 for (unsigned im = (unsigned)ISD::PRE_INC; 789 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { 790 setIndexedLoadAction(im, MVT::i1, Legal); 791 setIndexedLoadAction(im, MVT::i8, Legal); 792 setIndexedLoadAction(im, MVT::i16, Legal); 793 setIndexedLoadAction(im, MVT::i32, Legal); 794 setIndexedStoreAction(im, MVT::i1, Legal); 795 setIndexedStoreAction(im, MVT::i8, Legal); 796 setIndexedStoreAction(im, MVT::i16, Legal); 797 setIndexedStoreAction(im, MVT::i32, Legal); 798 } 799 } else { 800 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}. 801 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal); 802 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal); 803 } 804 805 setOperationAction(ISD::SADDO, MVT::i32, Custom); 806 setOperationAction(ISD::UADDO, MVT::i32, Custom); 807 setOperationAction(ISD::SSUBO, MVT::i32, Custom); 808 setOperationAction(ISD::USUBO, MVT::i32, Custom); 809 810 setOperationAction(ISD::ADDCARRY, MVT::i32, Custom); 811 setOperationAction(ISD::SUBCARRY, MVT::i32, Custom); 812 813 // i64 operation support. 814 setOperationAction(ISD::MUL, MVT::i64, Expand); 815 setOperationAction(ISD::MULHU, MVT::i32, Expand); 816 if (Subtarget->isThumb1Only()) { 817 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 818 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 819 } 820 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops() 821 || (Subtarget->isThumb2() && !Subtarget->hasDSP())) 822 setOperationAction(ISD::MULHS, MVT::i32, Expand); 823 824 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 825 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 826 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 827 setOperationAction(ISD::SRL, MVT::i64, Custom); 828 setOperationAction(ISD::SRA, MVT::i64, Custom); 829 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); 830 831 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1. 832 if (Subtarget->isThumb1Only()) { 833 setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand); 834 setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand); 835 setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand); 836 } 837 838 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) 839 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 840 841 // ARM does not have ROTL. 842 setOperationAction(ISD::ROTL, MVT::i32, Expand); 843 for (MVT VT : MVT::vector_valuetypes()) { 844 setOperationAction(ISD::ROTL, VT, Expand); 845 setOperationAction(ISD::ROTR, VT, Expand); 846 } 847 setOperationAction(ISD::CTTZ, MVT::i32, Custom); 848 setOperationAction(ISD::CTPOP, MVT::i32, Expand); 849 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) 850 setOperationAction(ISD::CTLZ, MVT::i32, Expand); 851 852 // @llvm.readcyclecounter requires the Performance Monitors extension. 853 // Default to the 0 expansion on unsupported platforms. 854 // FIXME: Technically there are older ARM CPUs that have 855 // implementation-specific ways of obtaining this information. 856 if (Subtarget->hasPerfMon()) 857 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); 858 859 // Only ARMv6 has BSWAP. 860 if (!Subtarget->hasV6Ops()) 861 setOperationAction(ISD::BSWAP, MVT::i32, Expand); 862 863 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode() 864 : Subtarget->hasDivideInARMMode(); 865 if (!hasDivide) { 866 // These are expanded into libcalls if the cpu doesn't have HW divider. 867 setOperationAction(ISD::SDIV, MVT::i32, LibCall); 868 setOperationAction(ISD::UDIV, MVT::i32, LibCall); 869 } 870 871 if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) { 872 setOperationAction(ISD::SDIV, MVT::i32, Custom); 873 setOperationAction(ISD::UDIV, MVT::i32, Custom); 874 875 setOperationAction(ISD::SDIV, MVT::i64, Custom); 876 setOperationAction(ISD::UDIV, MVT::i64, Custom); 877 } 878 879 setOperationAction(ISD::SREM, MVT::i32, Expand); 880 setOperationAction(ISD::UREM, MVT::i32, Expand); 881 882 // Register based DivRem for AEABI (RTABI 4.2) 883 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || 884 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || 885 Subtarget->isTargetWindows()) { 886 setOperationAction(ISD::SREM, MVT::i64, Custom); 887 setOperationAction(ISD::UREM, MVT::i64, Custom); 888 HasStandaloneRem = false; 889 890 if (Subtarget->isTargetWindows()) { 891 const struct { 892 const RTLIB::Libcall Op; 893 const char * const Name; 894 const CallingConv::ID CC; 895 } LibraryCalls[] = { 896 { RTLIB::SDIVREM_I8, "__rt_sdiv", CallingConv::ARM_AAPCS }, 897 { RTLIB::SDIVREM_I16, "__rt_sdiv", CallingConv::ARM_AAPCS }, 898 { RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS }, 899 { RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS }, 900 901 { RTLIB::UDIVREM_I8, "__rt_udiv", CallingConv::ARM_AAPCS }, 902 { RTLIB::UDIVREM_I16, "__rt_udiv", CallingConv::ARM_AAPCS }, 903 { RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS }, 904 { RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS }, 905 }; 906 907 for (const auto &LC : LibraryCalls) { 908 setLibcallName(LC.Op, LC.Name); 909 setLibcallCallingConv(LC.Op, LC.CC); 910 } 911 } else { 912 const struct { 913 const RTLIB::Libcall Op; 914 const char * const Name; 915 const CallingConv::ID CC; 916 } LibraryCalls[] = { 917 { RTLIB::SDIVREM_I8, "__aeabi_idivmod", CallingConv::ARM_AAPCS }, 918 { RTLIB::SDIVREM_I16, "__aeabi_idivmod", CallingConv::ARM_AAPCS }, 919 { RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS }, 920 { RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS }, 921 922 { RTLIB::UDIVREM_I8, "__aeabi_uidivmod", CallingConv::ARM_AAPCS }, 923 { RTLIB::UDIVREM_I16, "__aeabi_uidivmod", CallingConv::ARM_AAPCS }, 924 { RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS }, 925 { RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS }, 926 }; 927 928 for (const auto &LC : LibraryCalls) { 929 setLibcallName(LC.Op, LC.Name); 930 setLibcallCallingConv(LC.Op, LC.CC); 931 } 932 } 933 934 setOperationAction(ISD::SDIVREM, MVT::i32, Custom); 935 setOperationAction(ISD::UDIVREM, MVT::i32, Custom); 936 setOperationAction(ISD::SDIVREM, MVT::i64, Custom); 937 setOperationAction(ISD::UDIVREM, MVT::i64, Custom); 938 } else { 939 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 940 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 941 } 942 943 if (Subtarget->isTargetWindows() && Subtarget->getTargetTriple().isOSMSVCRT()) 944 for (auto &VT : {MVT::f32, MVT::f64}) 945 setOperationAction(ISD::FPOWI, VT, Custom); 946 947 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 948 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 949 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 950 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 951 952 setOperationAction(ISD::TRAP, MVT::Other, Legal); 953 954 // Use the default implementation. 955 setOperationAction(ISD::VASTART, MVT::Other, Custom); 956 setOperationAction(ISD::VAARG, MVT::Other, Expand); 957 setOperationAction(ISD::VACOPY, MVT::Other, Expand); 958 setOperationAction(ISD::VAEND, MVT::Other, Expand); 959 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 960 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 961 962 if (Subtarget->isTargetWindows()) 963 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 964 else 965 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 966 967 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use 968 // the default expansion. 969 InsertFencesForAtomic = false; 970 if (Subtarget->hasAnyDataBarrier() && 971 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) { 972 // ATOMIC_FENCE needs custom lowering; the others should have been expanded 973 // to ldrex/strex loops already. 974 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); 975 if (!Subtarget->isThumb() || !Subtarget->isMClass()) 976 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); 977 978 // On v8, we have particularly efficient implementations of atomic fences 979 // if they can be combined with nearby atomic loads and stores. 980 if (!Subtarget->hasV8Ops() || getTargetMachine().getOptLevel() == 0) { 981 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc. 982 InsertFencesForAtomic = true; 983 } 984 } else { 985 // If there's anything we can use as a barrier, go through custom lowering 986 // for ATOMIC_FENCE. 987 // If target has DMB in thumb, Fences can be inserted. 988 if (Subtarget->hasDataBarrier()) 989 InsertFencesForAtomic = true; 990 991 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, 992 Subtarget->hasAnyDataBarrier() ? Custom : Expand); 993 994 // Set them all for expansion, which will force libcalls. 995 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand); 996 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Expand); 997 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Expand); 998 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand); 999 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Expand); 1000 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Expand); 1001 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Expand); 1002 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand); 1003 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand); 1004 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand); 1005 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand); 1006 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand); 1007 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the 1008 // Unordered/Monotonic case. 1009 if (!InsertFencesForAtomic) { 1010 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom); 1011 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom); 1012 } 1013 } 1014 1015 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 1016 1017 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes. 1018 if (!Subtarget->hasV6Ops()) { 1019 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 1020 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 1021 } 1022 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 1023 1024 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && 1025 !Subtarget->isThumb1Only()) { 1026 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR 1027 // iff target supports vfp2. 1028 setOperationAction(ISD::BITCAST, MVT::i64, Custom); 1029 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 1030 } 1031 1032 // We want to custom lower some of our intrinsics. 1033 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 1034 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 1035 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 1036 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom); 1037 if (Subtarget->useSjLjEH()) 1038 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume"); 1039 1040 setOperationAction(ISD::SETCC, MVT::i32, Expand); 1041 setOperationAction(ISD::SETCC, MVT::f32, Expand); 1042 setOperationAction(ISD::SETCC, MVT::f64, Expand); 1043 setOperationAction(ISD::SELECT, MVT::i32, Custom); 1044 setOperationAction(ISD::SELECT, MVT::f32, Custom); 1045 setOperationAction(ISD::SELECT, MVT::f64, Custom); 1046 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 1047 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 1048 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 1049 if (Subtarget->hasFullFP16()) { 1050 setOperationAction(ISD::SETCC, MVT::f16, Expand); 1051 setOperationAction(ISD::SELECT, MVT::f16, Custom); 1052 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom); 1053 } 1054 1055 setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom); 1056 1057 setOperationAction(ISD::BRCOND, MVT::Other, Custom); 1058 setOperationAction(ISD::BR_CC, MVT::i32, Custom); 1059 if (Subtarget->hasFullFP16()) 1060 setOperationAction(ISD::BR_CC, MVT::f16, Custom); 1061 setOperationAction(ISD::BR_CC, MVT::f32, Custom); 1062 setOperationAction(ISD::BR_CC, MVT::f64, Custom); 1063 setOperationAction(ISD::BR_JT, MVT::Other, Custom); 1064 1065 // We don't support sin/cos/fmod/copysign/pow 1066 setOperationAction(ISD::FSIN, MVT::f64, Expand); 1067 setOperationAction(ISD::FSIN, MVT::f32, Expand); 1068 setOperationAction(ISD::FCOS, MVT::f32, Expand); 1069 setOperationAction(ISD::FCOS, MVT::f64, Expand); 1070 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 1071 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 1072 setOperationAction(ISD::FREM, MVT::f64, Expand); 1073 setOperationAction(ISD::FREM, MVT::f32, Expand); 1074 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && 1075 !Subtarget->isThumb1Only()) { 1076 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 1077 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 1078 } 1079 setOperationAction(ISD::FPOW, MVT::f64, Expand); 1080 setOperationAction(ISD::FPOW, MVT::f32, Expand); 1081 1082 if (!Subtarget->hasVFP4()) { 1083 setOperationAction(ISD::FMA, MVT::f64, Expand); 1084 setOperationAction(ISD::FMA, MVT::f32, Expand); 1085 } 1086 1087 // Various VFP goodness 1088 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) { 1089 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded. 1090 if (!Subtarget->hasFPARMv8() || Subtarget->isFPOnlySP()) { 1091 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); 1092 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); 1093 } 1094 1095 // fp16 is a special v7 extension that adds f16 <-> f32 conversions. 1096 if (!Subtarget->hasFP16()) { 1097 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); 1098 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); 1099 } 1100 } 1101 1102 // Use __sincos_stret if available. 1103 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr && 1104 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) { 1105 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); 1106 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); 1107 } 1108 1109 // FP-ARMv8 implements a lot of rounding-like FP operations. 1110 if (Subtarget->hasFPARMv8()) { 1111 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 1112 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 1113 setOperationAction(ISD::FROUND, MVT::f32, Legal); 1114 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 1115 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); 1116 setOperationAction(ISD::FRINT, MVT::f32, Legal); 1117 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); 1118 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); 1119 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal); 1120 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal); 1121 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); 1122 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); 1123 1124 if (!Subtarget->isFPOnlySP()) { 1125 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 1126 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 1127 setOperationAction(ISD::FROUND, MVT::f64, Legal); 1128 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 1129 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); 1130 setOperationAction(ISD::FRINT, MVT::f64, Legal); 1131 setOperationAction(ISD::FMINNUM, MVT::f64, Legal); 1132 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); 1133 } 1134 } 1135 1136 if (Subtarget->hasNEON()) { 1137 // vmin and vmax aren't available in a scalar form, so we use 1138 // a NEON instruction with an undef lane instead. 1139 setOperationAction(ISD::FMINNAN, MVT::f16, Legal); 1140 setOperationAction(ISD::FMAXNAN, MVT::f16, Legal); 1141 setOperationAction(ISD::FMINNAN, MVT::f32, Legal); 1142 setOperationAction(ISD::FMAXNAN, MVT::f32, Legal); 1143 setOperationAction(ISD::FMINNAN, MVT::v2f32, Legal); 1144 setOperationAction(ISD::FMAXNAN, MVT::v2f32, Legal); 1145 setOperationAction(ISD::FMINNAN, MVT::v4f32, Legal); 1146 setOperationAction(ISD::FMAXNAN, MVT::v4f32, Legal); 1147 } 1148 1149 // We have target-specific dag combine patterns for the following nodes: 1150 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine 1151 setTargetDAGCombine(ISD::ADD); 1152 setTargetDAGCombine(ISD::SUB); 1153 setTargetDAGCombine(ISD::MUL); 1154 setTargetDAGCombine(ISD::AND); 1155 setTargetDAGCombine(ISD::OR); 1156 setTargetDAGCombine(ISD::XOR); 1157 1158 if (Subtarget->hasV6Ops()) 1159 setTargetDAGCombine(ISD::SRL); 1160 1161 setStackPointerRegisterToSaveRestore(ARM::SP); 1162 1163 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() || 1164 !Subtarget->hasVFP2()) 1165 setSchedulingPreference(Sched::RegPressure); 1166 else 1167 setSchedulingPreference(Sched::Hybrid); 1168 1169 //// temporary - rewrite interface to use type 1170 MaxStoresPerMemset = 8; 1171 MaxStoresPerMemsetOptSize = 4; 1172 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores 1173 MaxStoresPerMemcpyOptSize = 2; 1174 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores 1175 MaxStoresPerMemmoveOptSize = 2; 1176 1177 // On ARM arguments smaller than 4 bytes are extended, so all arguments 1178 // are at least 4 bytes aligned. 1179 setMinStackArgumentAlignment(4); 1180 1181 // Prefer likely predicted branches to selects on out-of-order cores. 1182 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder(); 1183 1184 setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2); 1185 } 1186 1187 bool ARMTargetLowering::useSoftFloat() const { 1188 return Subtarget->useSoftFloat(); 1189 } 1190 1191 // FIXME: It might make sense to define the representative register class as the 1192 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is 1193 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently, 1194 // SPR's representative would be DPR_VFP2. This should work well if register 1195 // pressure tracking were modified such that a register use would increment the 1196 // pressure of the register class's representative and all of it's super 1197 // classes' representatives transitively. We have not implemented this because 1198 // of the difficulty prior to coalescing of modeling operand register classes 1199 // due to the common occurrence of cross class copies and subregister insertions 1200 // and extractions. 1201 std::pair<const TargetRegisterClass *, uint8_t> 1202 ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, 1203 MVT VT) const { 1204 const TargetRegisterClass *RRC = nullptr; 1205 uint8_t Cost = 1; 1206 switch (VT.SimpleTy) { 1207 default: 1208 return TargetLowering::findRepresentativeClass(TRI, VT); 1209 // Use DPR as representative register class for all floating point 1210 // and vector types. Since there are 32 SPR registers and 32 DPR registers so 1211 // the cost is 1 for both f32 and f64. 1212 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16: 1213 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32: 1214 RRC = &ARM::DPRRegClass; 1215 // When NEON is used for SP, only half of the register file is available 1216 // because operations that define both SP and DP results will be constrained 1217 // to the VFP2 class (D0-D15). We currently model this constraint prior to 1218 // coalescing by double-counting the SP regs. See the FIXME above. 1219 if (Subtarget->useNEONForSinglePrecisionFP()) 1220 Cost = 2; 1221 break; 1222 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: 1223 case MVT::v4f32: case MVT::v2f64: 1224 RRC = &ARM::DPRRegClass; 1225 Cost = 2; 1226 break; 1227 case MVT::v4i64: 1228 RRC = &ARM::DPRRegClass; 1229 Cost = 4; 1230 break; 1231 case MVT::v8i64: 1232 RRC = &ARM::DPRRegClass; 1233 Cost = 8; 1234 break; 1235 } 1236 return std::make_pair(RRC, Cost); 1237 } 1238 1239 const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const { 1240 switch ((ARMISD::NodeType)Opcode) { 1241 case ARMISD::FIRST_NUMBER: break; 1242 case ARMISD::Wrapper: return "ARMISD::Wrapper"; 1243 case ARMISD::WrapperPIC: return "ARMISD::WrapperPIC"; 1244 case ARMISD::WrapperJT: return "ARMISD::WrapperJT"; 1245 case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL"; 1246 case ARMISD::CALL: return "ARMISD::CALL"; 1247 case ARMISD::CALL_PRED: return "ARMISD::CALL_PRED"; 1248 case ARMISD::CALL_NOLINK: return "ARMISD::CALL_NOLINK"; 1249 case ARMISD::BRCOND: return "ARMISD::BRCOND"; 1250 case ARMISD::BR_JT: return "ARMISD::BR_JT"; 1251 case ARMISD::BR2_JT: return "ARMISD::BR2_JT"; 1252 case ARMISD::RET_FLAG: return "ARMISD::RET_FLAG"; 1253 case ARMISD::INTRET_FLAG: return "ARMISD::INTRET_FLAG"; 1254 case ARMISD::PIC_ADD: return "ARMISD::PIC_ADD"; 1255 case ARMISD::CMP: return "ARMISD::CMP"; 1256 case ARMISD::CMN: return "ARMISD::CMN"; 1257 case ARMISD::CMPZ: return "ARMISD::CMPZ"; 1258 case ARMISD::CMPFP: return "ARMISD::CMPFP"; 1259 case ARMISD::CMPFPw0: return "ARMISD::CMPFPw0"; 1260 case ARMISD::BCC_i64: return "ARMISD::BCC_i64"; 1261 case ARMISD::FMSTAT: return "ARMISD::FMSTAT"; 1262 1263 case ARMISD::CMOV: return "ARMISD::CMOV"; 1264 1265 case ARMISD::SSAT: return "ARMISD::SSAT"; 1266 case ARMISD::USAT: return "ARMISD::USAT"; 1267 1268 case ARMISD::SRL_FLAG: return "ARMISD::SRL_FLAG"; 1269 case ARMISD::SRA_FLAG: return "ARMISD::SRA_FLAG"; 1270 case ARMISD::RRX: return "ARMISD::RRX"; 1271 1272 case ARMISD::ADDC: return "ARMISD::ADDC"; 1273 case ARMISD::ADDE: return "ARMISD::ADDE"; 1274 case ARMISD::SUBC: return "ARMISD::SUBC"; 1275 case ARMISD::SUBE: return "ARMISD::SUBE"; 1276 1277 case ARMISD::VMOVRRD: return "ARMISD::VMOVRRD"; 1278 case ARMISD::VMOVDRR: return "ARMISD::VMOVDRR"; 1279 case ARMISD::VMOVhr: return "ARMISD::VMOVhr"; 1280 case ARMISD::VMOVrh: return "ARMISD::VMOVrh"; 1281 case ARMISD::VMOVSR: return "ARMISD::VMOVSR"; 1282 1283 case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP"; 1284 case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP"; 1285 case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH"; 1286 1287 case ARMISD::TC_RETURN: return "ARMISD::TC_RETURN"; 1288 1289 case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER"; 1290 1291 case ARMISD::DYN_ALLOC: return "ARMISD::DYN_ALLOC"; 1292 1293 case ARMISD::MEMBARRIER_MCR: return "ARMISD::MEMBARRIER_MCR"; 1294 1295 case ARMISD::PRELOAD: return "ARMISD::PRELOAD"; 1296 1297 case ARMISD::WIN__CHKSTK: return "ARMISD::WIN__CHKSTK"; 1298 case ARMISD::WIN__DBZCHK: return "ARMISD::WIN__DBZCHK"; 1299 1300 case ARMISD::VCEQ: return "ARMISD::VCEQ"; 1301 case ARMISD::VCEQZ: return "ARMISD::VCEQZ"; 1302 case ARMISD::VCGE: return "ARMISD::VCGE"; 1303 case ARMISD::VCGEZ: return "ARMISD::VCGEZ"; 1304 case ARMISD::VCLEZ: return "ARMISD::VCLEZ"; 1305 case ARMISD::VCGEU: return "ARMISD::VCGEU"; 1306 case ARMISD::VCGT: return "ARMISD::VCGT"; 1307 case ARMISD::VCGTZ: return "ARMISD::VCGTZ"; 1308 case ARMISD::VCLTZ: return "ARMISD::VCLTZ"; 1309 case ARMISD::VCGTU: return "ARMISD::VCGTU"; 1310 case ARMISD::VTST: return "ARMISD::VTST"; 1311 1312 case ARMISD::VSHL: return "ARMISD::VSHL"; 1313 case ARMISD::VSHRs: return "ARMISD::VSHRs"; 1314 case ARMISD::VSHRu: return "ARMISD::VSHRu"; 1315 case ARMISD::VRSHRs: return "ARMISD::VRSHRs"; 1316 case ARMISD::VRSHRu: return "ARMISD::VRSHRu"; 1317 case ARMISD::VRSHRN: return "ARMISD::VRSHRN"; 1318 case ARMISD::VQSHLs: return "ARMISD::VQSHLs"; 1319 case ARMISD::VQSHLu: return "ARMISD::VQSHLu"; 1320 case ARMISD::VQSHLsu: return "ARMISD::VQSHLsu"; 1321 case ARMISD::VQSHRNs: return "ARMISD::VQSHRNs"; 1322 case ARMISD::VQSHRNu: return "ARMISD::VQSHRNu"; 1323 case ARMISD::VQSHRNsu: return "ARMISD::VQSHRNsu"; 1324 case ARMISD::VQRSHRNs: return "ARMISD::VQRSHRNs"; 1325 case ARMISD::VQRSHRNu: return "ARMISD::VQRSHRNu"; 1326 case ARMISD::VQRSHRNsu: return "ARMISD::VQRSHRNsu"; 1327 case ARMISD::VSLI: return "ARMISD::VSLI"; 1328 case ARMISD::VSRI: return "ARMISD::VSRI"; 1329 case ARMISD::VGETLANEu: return "ARMISD::VGETLANEu"; 1330 case ARMISD::VGETLANEs: return "ARMISD::VGETLANEs"; 1331 case ARMISD::VMOVIMM: return "ARMISD::VMOVIMM"; 1332 case ARMISD::VMVNIMM: return "ARMISD::VMVNIMM"; 1333 case ARMISD::VMOVFPIMM: return "ARMISD::VMOVFPIMM"; 1334 case ARMISD::VDUP: return "ARMISD::VDUP"; 1335 case ARMISD::VDUPLANE: return "ARMISD::VDUPLANE"; 1336 case ARMISD::VEXT: return "ARMISD::VEXT"; 1337 case ARMISD::VREV64: return "ARMISD::VREV64"; 1338 case ARMISD::VREV32: return "ARMISD::VREV32"; 1339 case ARMISD::VREV16: return "ARMISD::VREV16"; 1340 case ARMISD::VZIP: return "ARMISD::VZIP"; 1341 case ARMISD::VUZP: return "ARMISD::VUZP"; 1342 case ARMISD::VTRN: return "ARMISD::VTRN"; 1343 case ARMISD::VTBL1: return "ARMISD::VTBL1"; 1344 case ARMISD::VTBL2: return "ARMISD::VTBL2"; 1345 case ARMISD::VMULLs: return "ARMISD::VMULLs"; 1346 case ARMISD::VMULLu: return "ARMISD::VMULLu"; 1347 case ARMISD::UMAAL: return "ARMISD::UMAAL"; 1348 case ARMISD::UMLAL: return "ARMISD::UMLAL"; 1349 case ARMISD::SMLAL: return "ARMISD::SMLAL"; 1350 case ARMISD::SMLALBB: return "ARMISD::SMLALBB"; 1351 case ARMISD::SMLALBT: return "ARMISD::SMLALBT"; 1352 case ARMISD::SMLALTB: return "ARMISD::SMLALTB"; 1353 case ARMISD::SMLALTT: return "ARMISD::SMLALTT"; 1354 case ARMISD::SMULWB: return "ARMISD::SMULWB"; 1355 case ARMISD::SMULWT: return "ARMISD::SMULWT"; 1356 case ARMISD::SMLALD: return "ARMISD::SMLALD"; 1357 case ARMISD::SMLALDX: return "ARMISD::SMLALDX"; 1358 case ARMISD::SMLSLD: return "ARMISD::SMLSLD"; 1359 case ARMISD::SMLSLDX: return "ARMISD::SMLSLDX"; 1360 case ARMISD::SMMLAR: return "ARMISD::SMMLAR"; 1361 case ARMISD::SMMLSR: return "ARMISD::SMMLSR"; 1362 case ARMISD::BUILD_VECTOR: return "ARMISD::BUILD_VECTOR"; 1363 case ARMISD::BFI: return "ARMISD::BFI"; 1364 case ARMISD::VORRIMM: return "ARMISD::VORRIMM"; 1365 case ARMISD::VBICIMM: return "ARMISD::VBICIMM"; 1366 case ARMISD::VBSL: return "ARMISD::VBSL"; 1367 case ARMISD::MEMCPY: return "ARMISD::MEMCPY"; 1368 case ARMISD::VLD1DUP: return "ARMISD::VLD1DUP"; 1369 case ARMISD::VLD2DUP: return "ARMISD::VLD2DUP"; 1370 case ARMISD::VLD3DUP: return "ARMISD::VLD3DUP"; 1371 case ARMISD::VLD4DUP: return "ARMISD::VLD4DUP"; 1372 case ARMISD::VLD1_UPD: return "ARMISD::VLD1_UPD"; 1373 case ARMISD::VLD2_UPD: return "ARMISD::VLD2_UPD"; 1374 case ARMISD::VLD3_UPD: return "ARMISD::VLD3_UPD"; 1375 case ARMISD::VLD4_UPD: return "ARMISD::VLD4_UPD"; 1376 case ARMISD::VLD2LN_UPD: return "ARMISD::VLD2LN_UPD"; 1377 case ARMISD::VLD3LN_UPD: return "ARMISD::VLD3LN_UPD"; 1378 case ARMISD::VLD4LN_UPD: return "ARMISD::VLD4LN_UPD"; 1379 case ARMISD::VLD1DUP_UPD: return "ARMISD::VLD1DUP_UPD"; 1380 case ARMISD::VLD2DUP_UPD: return "ARMISD::VLD2DUP_UPD"; 1381 case ARMISD::VLD3DUP_UPD: return "ARMISD::VLD3DUP_UPD"; 1382 case ARMISD::VLD4DUP_UPD: return "ARMISD::VLD4DUP_UPD"; 1383 case ARMISD::VST1_UPD: return "ARMISD::VST1_UPD"; 1384 case ARMISD::VST2_UPD: return "ARMISD::VST2_UPD"; 1385 case ARMISD::VST3_UPD: return "ARMISD::VST3_UPD"; 1386 case ARMISD::VST4_UPD: return "ARMISD::VST4_UPD"; 1387 case ARMISD::VST2LN_UPD: return "ARMISD::VST2LN_UPD"; 1388 case ARMISD::VST3LN_UPD: return "ARMISD::VST3LN_UPD"; 1389 case ARMISD::VST4LN_UPD: return "ARMISD::VST4LN_UPD"; 1390 } 1391 return nullptr; 1392 } 1393 1394 EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1395 EVT VT) const { 1396 if (!VT.isVector()) 1397 return getPointerTy(DL); 1398 return VT.changeVectorElementTypeToInteger(); 1399 } 1400 1401 /// getRegClassFor - Return the register class that should be used for the 1402 /// specified value type. 1403 const TargetRegisterClass *ARMTargetLowering::getRegClassFor(MVT VT) const { 1404 // Map v4i64 to QQ registers but do not make the type legal. Similarly map 1405 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to 1406 // load / store 4 to 8 consecutive D registers. 1407 if (Subtarget->hasNEON()) { 1408 if (VT == MVT::v4i64) 1409 return &ARM::QQPRRegClass; 1410 if (VT == MVT::v8i64) 1411 return &ARM::QQQQPRRegClass; 1412 } 1413 return TargetLowering::getRegClassFor(VT); 1414 } 1415 1416 // memcpy, and other memory intrinsics, typically tries to use LDM/STM if the 1417 // source/dest is aligned and the copy size is large enough. We therefore want 1418 // to align such objects passed to memory intrinsics. 1419 bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, 1420 unsigned &PrefAlign) const { 1421 if (!isa<MemIntrinsic>(CI)) 1422 return false; 1423 MinSize = 8; 1424 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1 1425 // cycle faster than 4-byte aligned LDM. 1426 PrefAlign = (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? 8 : 4); 1427 return true; 1428 } 1429 1430 // Create a fast isel object. 1431 FastISel * 1432 ARMTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo, 1433 const TargetLibraryInfo *libInfo) const { 1434 return ARM::createFastISel(funcInfo, libInfo); 1435 } 1436 1437 Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const { 1438 unsigned NumVals = N->getNumValues(); 1439 if (!NumVals) 1440 return Sched::RegPressure; 1441 1442 for (unsigned i = 0; i != NumVals; ++i) { 1443 EVT VT = N->getValueType(i); 1444 if (VT == MVT::Glue || VT == MVT::Other) 1445 continue; 1446 if (VT.isFloatingPoint() || VT.isVector()) 1447 return Sched::ILP; 1448 } 1449 1450 if (!N->isMachineOpcode()) 1451 return Sched::RegPressure; 1452 1453 // Load are scheduled for latency even if there instruction itinerary 1454 // is not available. 1455 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 1456 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); 1457 1458 if (MCID.getNumDefs() == 0) 1459 return Sched::RegPressure; 1460 if (!Itins->isEmpty() && 1461 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2) 1462 return Sched::ILP; 1463 1464 return Sched::RegPressure; 1465 } 1466 1467 //===----------------------------------------------------------------------===// 1468 // Lowering Code 1469 //===----------------------------------------------------------------------===// 1470 1471 static bool isSRL16(const SDValue &Op) { 1472 if (Op.getOpcode() != ISD::SRL) 1473 return false; 1474 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 1475 return Const->getZExtValue() == 16; 1476 return false; 1477 } 1478 1479 static bool isSRA16(const SDValue &Op) { 1480 if (Op.getOpcode() != ISD::SRA) 1481 return false; 1482 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 1483 return Const->getZExtValue() == 16; 1484 return false; 1485 } 1486 1487 static bool isSHL16(const SDValue &Op) { 1488 if (Op.getOpcode() != ISD::SHL) 1489 return false; 1490 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 1491 return Const->getZExtValue() == 16; 1492 return false; 1493 } 1494 1495 // Check for a signed 16-bit value. We special case SRA because it makes it 1496 // more simple when also looking for SRAs that aren't sign extending a 1497 // smaller value. Without the check, we'd need to take extra care with 1498 // checking order for some operations. 1499 static bool isS16(const SDValue &Op, SelectionDAG &DAG) { 1500 if (isSRA16(Op)) 1501 return isSHL16(Op.getOperand(0)); 1502 return DAG.ComputeNumSignBits(Op) == 17; 1503 } 1504 1505 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC 1506 static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) { 1507 switch (CC) { 1508 default: llvm_unreachable("Unknown condition code!"); 1509 case ISD::SETNE: return ARMCC::NE; 1510 case ISD::SETEQ: return ARMCC::EQ; 1511 case ISD::SETGT: return ARMCC::GT; 1512 case ISD::SETGE: return ARMCC::GE; 1513 case ISD::SETLT: return ARMCC::LT; 1514 case ISD::SETLE: return ARMCC::LE; 1515 case ISD::SETUGT: return ARMCC::HI; 1516 case ISD::SETUGE: return ARMCC::HS; 1517 case ISD::SETULT: return ARMCC::LO; 1518 case ISD::SETULE: return ARMCC::LS; 1519 } 1520 } 1521 1522 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC. 1523 static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, 1524 ARMCC::CondCodes &CondCode2, bool &InvalidOnQNaN) { 1525 CondCode2 = ARMCC::AL; 1526 InvalidOnQNaN = true; 1527 switch (CC) { 1528 default: llvm_unreachable("Unknown FP condition!"); 1529 case ISD::SETEQ: 1530 case ISD::SETOEQ: 1531 CondCode = ARMCC::EQ; 1532 InvalidOnQNaN = false; 1533 break; 1534 case ISD::SETGT: 1535 case ISD::SETOGT: CondCode = ARMCC::GT; break; 1536 case ISD::SETGE: 1537 case ISD::SETOGE: CondCode = ARMCC::GE; break; 1538 case ISD::SETOLT: CondCode = ARMCC::MI; break; 1539 case ISD::SETOLE: CondCode = ARMCC::LS; break; 1540 case ISD::SETONE: 1541 CondCode = ARMCC::MI; 1542 CondCode2 = ARMCC::GT; 1543 InvalidOnQNaN = false; 1544 break; 1545 case ISD::SETO: CondCode = ARMCC::VC; break; 1546 case ISD::SETUO: CondCode = ARMCC::VS; break; 1547 case ISD::SETUEQ: 1548 CondCode = ARMCC::EQ; 1549 CondCode2 = ARMCC::VS; 1550 InvalidOnQNaN = false; 1551 break; 1552 case ISD::SETUGT: CondCode = ARMCC::HI; break; 1553 case ISD::SETUGE: CondCode = ARMCC::PL; break; 1554 case ISD::SETLT: 1555 case ISD::SETULT: CondCode = ARMCC::LT; break; 1556 case ISD::SETLE: 1557 case ISD::SETULE: CondCode = ARMCC::LE; break; 1558 case ISD::SETNE: 1559 case ISD::SETUNE: 1560 CondCode = ARMCC::NE; 1561 InvalidOnQNaN = false; 1562 break; 1563 } 1564 } 1565 1566 //===----------------------------------------------------------------------===// 1567 // Calling Convention Implementation 1568 //===----------------------------------------------------------------------===// 1569 1570 #include "ARMGenCallingConv.inc" 1571 1572 /// getEffectiveCallingConv - Get the effective calling convention, taking into 1573 /// account presence of floating point hardware and calling convention 1574 /// limitations, such as support for variadic functions. 1575 CallingConv::ID 1576 ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC, 1577 bool isVarArg) const { 1578 switch (CC) { 1579 default: 1580 report_fatal_error("Unsupported calling convention"); 1581 case CallingConv::ARM_AAPCS: 1582 case CallingConv::ARM_APCS: 1583 case CallingConv::GHC: 1584 return CC; 1585 case CallingConv::PreserveMost: 1586 return CallingConv::PreserveMost; 1587 case CallingConv::ARM_AAPCS_VFP: 1588 case CallingConv::Swift: 1589 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP; 1590 case CallingConv::C: 1591 if (!Subtarget->isAAPCS_ABI()) 1592 return CallingConv::ARM_APCS; 1593 else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && 1594 getTargetMachine().Options.FloatABIType == FloatABI::Hard && 1595 !isVarArg) 1596 return CallingConv::ARM_AAPCS_VFP; 1597 else 1598 return CallingConv::ARM_AAPCS; 1599 case CallingConv::Fast: 1600 case CallingConv::CXX_FAST_TLS: 1601 if (!Subtarget->isAAPCS_ABI()) { 1602 if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg) 1603 return CallingConv::Fast; 1604 return CallingConv::ARM_APCS; 1605 } else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg) 1606 return CallingConv::ARM_AAPCS_VFP; 1607 else 1608 return CallingConv::ARM_AAPCS; 1609 } 1610 } 1611 1612 CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC, 1613 bool isVarArg) const { 1614 return CCAssignFnForNode(CC, false, isVarArg); 1615 } 1616 1617 CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC, 1618 bool isVarArg) const { 1619 return CCAssignFnForNode(CC, true, isVarArg); 1620 } 1621 1622 /// CCAssignFnForNode - Selects the correct CCAssignFn for the given 1623 /// CallingConvention. 1624 CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC, 1625 bool Return, 1626 bool isVarArg) const { 1627 switch (getEffectiveCallingConv(CC, isVarArg)) { 1628 default: 1629 report_fatal_error("Unsupported calling convention"); 1630 case CallingConv::ARM_APCS: 1631 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS); 1632 case CallingConv::ARM_AAPCS: 1633 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS); 1634 case CallingConv::ARM_AAPCS_VFP: 1635 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP); 1636 case CallingConv::Fast: 1637 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS); 1638 case CallingConv::GHC: 1639 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC); 1640 case CallingConv::PreserveMost: 1641 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS); 1642 } 1643 } 1644 1645 /// LowerCallResult - Lower the result values of a call into the 1646 /// appropriate copies out of appropriate physical registers. 1647 SDValue ARMTargetLowering::LowerCallResult( 1648 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 1649 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 1650 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn, 1651 SDValue ThisVal) const { 1652 // Assign locations to each value returned by this call. 1653 SmallVector<CCValAssign, 16> RVLocs; 1654 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 1655 *DAG.getContext()); 1656 CCInfo.AnalyzeCallResult(Ins, CCAssignFnForReturn(CallConv, isVarArg)); 1657 1658 // Copy all of the result registers out of their specified physreg. 1659 for (unsigned i = 0; i != RVLocs.size(); ++i) { 1660 CCValAssign VA = RVLocs[i]; 1661 1662 // Pass 'this' value directly from the argument to return value, to avoid 1663 // reg unit interference 1664 if (i == 0 && isThisReturn) { 1665 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 && 1666 "unexpected return calling convention register assignment"); 1667 InVals.push_back(ThisVal); 1668 continue; 1669 } 1670 1671 SDValue Val; 1672 if (VA.needsCustom()) { 1673 // Handle f64 or half of a v2f64. 1674 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 1675 InFlag); 1676 Chain = Lo.getValue(1); 1677 InFlag = Lo.getValue(2); 1678 VA = RVLocs[++i]; // skip ahead to next loc 1679 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 1680 InFlag); 1681 Chain = Hi.getValue(1); 1682 InFlag = Hi.getValue(2); 1683 if (!Subtarget->isLittle()) 1684 std::swap (Lo, Hi); 1685 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi); 1686 1687 if (VA.getLocVT() == MVT::v2f64) { 1688 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64); 1689 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val, 1690 DAG.getConstant(0, dl, MVT::i32)); 1691 1692 VA = RVLocs[++i]; // skip ahead to next loc 1693 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); 1694 Chain = Lo.getValue(1); 1695 InFlag = Lo.getValue(2); 1696 VA = RVLocs[++i]; // skip ahead to next loc 1697 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); 1698 Chain = Hi.getValue(1); 1699 InFlag = Hi.getValue(2); 1700 if (!Subtarget->isLittle()) 1701 std::swap (Lo, Hi); 1702 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi); 1703 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val, 1704 DAG.getConstant(1, dl, MVT::i32)); 1705 } 1706 } else { 1707 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(), 1708 InFlag); 1709 Chain = Val.getValue(1); 1710 InFlag = Val.getValue(2); 1711 } 1712 1713 switch (VA.getLocInfo()) { 1714 default: llvm_unreachable("Unknown loc info!"); 1715 case CCValAssign::Full: break; 1716 case CCValAssign::BCvt: 1717 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val); 1718 break; 1719 } 1720 1721 InVals.push_back(Val); 1722 } 1723 1724 return Chain; 1725 } 1726 1727 /// LowerMemOpCallTo - Store the argument to the stack. 1728 SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, 1729 SDValue Arg, const SDLoc &dl, 1730 SelectionDAG &DAG, 1731 const CCValAssign &VA, 1732 ISD::ArgFlagsTy Flags) const { 1733 unsigned LocMemOffset = VA.getLocMemOffset(); 1734 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 1735 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), 1736 StackPtr, PtrOff); 1737 return DAG.getStore( 1738 Chain, dl, Arg, PtrOff, 1739 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset)); 1740 } 1741 1742 void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, 1743 SDValue Chain, SDValue &Arg, 1744 RegsToPassVector &RegsToPass, 1745 CCValAssign &VA, CCValAssign &NextVA, 1746 SDValue &StackPtr, 1747 SmallVectorImpl<SDValue> &MemOpChains, 1748 ISD::ArgFlagsTy Flags) const { 1749 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl, 1750 DAG.getVTList(MVT::i32, MVT::i32), Arg); 1751 unsigned id = Subtarget->isLittle() ? 0 : 1; 1752 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id))); 1753 1754 if (NextVA.isRegLoc()) 1755 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id))); 1756 else { 1757 assert(NextVA.isMemLoc()); 1758 if (!StackPtr.getNode()) 1759 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, 1760 getPointerTy(DAG.getDataLayout())); 1761 1762 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id), 1763 dl, DAG, NextVA, 1764 Flags)); 1765 } 1766 } 1767 1768 /// LowerCall - Lowering a call into a callseq_start <- 1769 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter 1770 /// nodes. 1771 SDValue 1772 ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 1773 SmallVectorImpl<SDValue> &InVals) const { 1774 SelectionDAG &DAG = CLI.DAG; 1775 SDLoc &dl = CLI.DL; 1776 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 1777 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 1778 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 1779 SDValue Chain = CLI.Chain; 1780 SDValue Callee = CLI.Callee; 1781 bool &isTailCall = CLI.IsTailCall; 1782 CallingConv::ID CallConv = CLI.CallConv; 1783 bool doesNotRet = CLI.DoesNotReturn; 1784 bool isVarArg = CLI.IsVarArg; 1785 1786 MachineFunction &MF = DAG.getMachineFunction(); 1787 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet(); 1788 bool isThisReturn = false; 1789 bool isSibCall = false; 1790 auto Attr = MF.getFunction().getFnAttribute("disable-tail-calls"); 1791 1792 // Disable tail calls if they're not supported. 1793 if (!Subtarget->supportsTailCall() || Attr.getValueAsString() == "true") 1794 isTailCall = false; 1795 1796 if (isTailCall) { 1797 // Check if it's really possible to do a tail call. 1798 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, 1799 isVarArg, isStructRet, MF.getFunction().hasStructRetAttr(), 1800 Outs, OutVals, Ins, DAG); 1801 if (!isTailCall && CLI.CS && CLI.CS.isMustTailCall()) 1802 report_fatal_error("failed to perform tail call elimination on a call " 1803 "site marked musttail"); 1804 // We don't support GuaranteedTailCallOpt for ARM, only automatically 1805 // detected sibcalls. 1806 if (isTailCall) { 1807 ++NumTailCalls; 1808 isSibCall = true; 1809 } 1810 } 1811 1812 // Analyze operands of the call, assigning locations to each operand. 1813 SmallVector<CCValAssign, 16> ArgLocs; 1814 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 1815 *DAG.getContext()); 1816 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg)); 1817 1818 // Get a count of how many bytes are to be pushed on the stack. 1819 unsigned NumBytes = CCInfo.getNextStackOffset(); 1820 1821 // For tail calls, memory operands are available in our caller's stack. 1822 if (isSibCall) 1823 NumBytes = 0; 1824 1825 // Adjust the stack pointer for the new arguments... 1826 // These operations are automatically eliminated by the prolog/epilog pass 1827 if (!isSibCall) 1828 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 1829 1830 SDValue StackPtr = 1831 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout())); 1832 1833 RegsToPassVector RegsToPass; 1834 SmallVector<SDValue, 8> MemOpChains; 1835 1836 // Walk the register/memloc assignments, inserting copies/loads. In the case 1837 // of tail call optimization, arguments are handled later. 1838 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); 1839 i != e; 1840 ++i, ++realArgIdx) { 1841 CCValAssign &VA = ArgLocs[i]; 1842 SDValue Arg = OutVals[realArgIdx]; 1843 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; 1844 bool isByVal = Flags.isByVal(); 1845 1846 // Promote the value if needed. 1847 switch (VA.getLocInfo()) { 1848 default: llvm_unreachable("Unknown loc info!"); 1849 case CCValAssign::Full: break; 1850 case CCValAssign::SExt: 1851 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 1852 break; 1853 case CCValAssign::ZExt: 1854 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 1855 break; 1856 case CCValAssign::AExt: 1857 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 1858 break; 1859 case CCValAssign::BCvt: 1860 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); 1861 break; 1862 } 1863 1864 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces 1865 if (VA.needsCustom()) { 1866 if (VA.getLocVT() == MVT::v2f64) { 1867 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 1868 DAG.getConstant(0, dl, MVT::i32)); 1869 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 1870 DAG.getConstant(1, dl, MVT::i32)); 1871 1872 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, 1873 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags); 1874 1875 VA = ArgLocs[++i]; // skip ahead to next loc 1876 if (VA.isRegLoc()) { 1877 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, 1878 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags); 1879 } else { 1880 assert(VA.isMemLoc()); 1881 1882 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Op1, 1883 dl, DAG, VA, Flags)); 1884 } 1885 } else { 1886 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i], 1887 StackPtr, MemOpChains, Flags); 1888 } 1889 } else if (VA.isRegLoc()) { 1890 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() && 1891 Outs[0].VT == MVT::i32) { 1892 assert(VA.getLocVT() == MVT::i32 && 1893 "unexpected calling convention register assignment"); 1894 assert(!Ins.empty() && Ins[0].VT == MVT::i32 && 1895 "unexpected use of 'returned'"); 1896 isThisReturn = true; 1897 } 1898 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1899 } else if (isByVal) { 1900 assert(VA.isMemLoc()); 1901 unsigned offset = 0; 1902 1903 // True if this byval aggregate will be split between registers 1904 // and memory. 1905 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount(); 1906 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed(); 1907 1908 if (CurByValIdx < ByValArgsCount) { 1909 1910 unsigned RegBegin, RegEnd; 1911 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd); 1912 1913 EVT PtrVT = 1914 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 1915 unsigned int i, j; 1916 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) { 1917 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32); 1918 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 1919 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg, 1920 MachinePointerInfo(), 1921 DAG.InferPtrAlignment(AddArg)); 1922 MemOpChains.push_back(Load.getValue(1)); 1923 RegsToPass.push_back(std::make_pair(j, Load)); 1924 } 1925 1926 // If parameter size outsides register area, "offset" value 1927 // helps us to calculate stack slot for remained part properly. 1928 offset = RegEnd - RegBegin; 1929 1930 CCInfo.nextInRegsParam(); 1931 } 1932 1933 if (Flags.getByValSize() > 4*offset) { 1934 auto PtrVT = getPointerTy(DAG.getDataLayout()); 1935 unsigned LocMemOffset = VA.getLocMemOffset(); 1936 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 1937 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff); 1938 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl); 1939 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset); 1940 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl, 1941 MVT::i32); 1942 SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), dl, 1943 MVT::i32); 1944 1945 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 1946 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode}; 1947 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, 1948 Ops)); 1949 } 1950 } else if (!isSibCall) { 1951 assert(VA.isMemLoc()); 1952 1953 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, 1954 dl, DAG, VA, Flags)); 1955 } 1956 } 1957 1958 if (!MemOpChains.empty()) 1959 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 1960 1961 // Build a sequence of copy-to-reg nodes chained together with token chain 1962 // and flag operands which copy the outgoing args into the appropriate regs. 1963 SDValue InFlag; 1964 // Tail call byval lowering might overwrite argument registers so in case of 1965 // tail call optimization the copies to registers are lowered later. 1966 if (!isTailCall) 1967 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1968 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 1969 RegsToPass[i].second, InFlag); 1970 InFlag = Chain.getValue(1); 1971 } 1972 1973 // For tail calls lower the arguments to the 'real' stack slot. 1974 if (isTailCall) { 1975 // Force all the incoming stack arguments to be loaded from the stack 1976 // before any new outgoing arguments are stored to the stack, because the 1977 // outgoing stack slots may alias the incoming argument stack slots, and 1978 // the alias isn't otherwise explicit. This is slightly more conservative 1979 // than necessary, because it means that each store effectively depends 1980 // on every argument instead of just those arguments it would clobber. 1981 1982 // Do not flag preceding copytoreg stuff together with the following stuff. 1983 InFlag = SDValue(); 1984 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1985 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 1986 RegsToPass[i].second, InFlag); 1987 InFlag = Chain.getValue(1); 1988 } 1989 InFlag = SDValue(); 1990 } 1991 1992 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every 1993 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol 1994 // node so that legalize doesn't hack it. 1995 bool isDirect = false; 1996 1997 const TargetMachine &TM = getTargetMachine(); 1998 const Module *Mod = MF.getFunction().getParent(); 1999 const GlobalValue *GV = nullptr; 2000 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 2001 GV = G->getGlobal(); 2002 bool isStub = 2003 !TM.shouldAssumeDSOLocal(*Mod, GV) && Subtarget->isTargetMachO(); 2004 2005 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass()); 2006 bool isLocalARMFunc = false; 2007 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2008 auto PtrVt = getPointerTy(DAG.getDataLayout()); 2009 2010 if (Subtarget->genLongCalls()) { 2011 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) && 2012 "long-calls codegen is not position independent!"); 2013 // Handle a global address or an external symbol. If it's not one of 2014 // those, the target's already in a register, so we don't need to do 2015 // anything extra. 2016 if (isa<GlobalAddressSDNode>(Callee)) { 2017 // Create a constant pool entry for the callee address 2018 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2019 ARMConstantPoolValue *CPV = 2020 ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0); 2021 2022 // Get the address of the callee into a register 2023 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4); 2024 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 2025 Callee = DAG.getLoad( 2026 PtrVt, dl, DAG.getEntryNode(), CPAddr, 2027 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2028 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) { 2029 const char *Sym = S->getSymbol(); 2030 2031 // Create a constant pool entry for the callee address 2032 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2033 ARMConstantPoolValue *CPV = 2034 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym, 2035 ARMPCLabelIndex, 0); 2036 // Get the address of the callee into a register 2037 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4); 2038 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 2039 Callee = DAG.getLoad( 2040 PtrVt, dl, DAG.getEntryNode(), CPAddr, 2041 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2042 } 2043 } else if (isa<GlobalAddressSDNode>(Callee)) { 2044 // If we're optimizing for minimum size and the function is called three or 2045 // more times in this block, we can improve codesize by calling indirectly 2046 // as BLXr has a 16-bit encoding. 2047 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal(); 2048 auto *BB = CLI.CS.getParent(); 2049 bool PreferIndirect = 2050 Subtarget->isThumb() && MF.getFunction().optForMinSize() && 2051 count_if(GV->users(), [&BB](const User *U) { 2052 return isa<Instruction>(U) && cast<Instruction>(U)->getParent() == BB; 2053 }) > 2; 2054 2055 if (!PreferIndirect) { 2056 isDirect = true; 2057 bool isDef = GV->isStrongDefinitionForLinker(); 2058 2059 // ARM call to a local ARM function is predicable. 2060 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking); 2061 // tBX takes a register source operand. 2062 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { 2063 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?"); 2064 Callee = DAG.getNode( 2065 ARMISD::WrapperPIC, dl, PtrVt, 2066 DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY)); 2067 Callee = DAG.getLoad( 2068 PtrVt, dl, DAG.getEntryNode(), Callee, 2069 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 2070 /* Alignment = */ 0, MachineMemOperand::MODereferenceable | 2071 MachineMemOperand::MOInvariant); 2072 } else if (Subtarget->isTargetCOFF()) { 2073 assert(Subtarget->isTargetWindows() && 2074 "Windows is the only supported COFF target"); 2075 unsigned TargetFlags = GV->hasDLLImportStorageClass() 2076 ? ARMII::MO_DLLIMPORT 2077 : ARMII::MO_NO_FLAG; 2078 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, /*Offset=*/0, 2079 TargetFlags); 2080 if (GV->hasDLLImportStorageClass()) 2081 Callee = 2082 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), 2083 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee), 2084 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 2085 } else { 2086 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, 0); 2087 } 2088 } 2089 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 2090 isDirect = true; 2091 // tBX takes a register source operand. 2092 const char *Sym = S->getSymbol(); 2093 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { 2094 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2095 ARMConstantPoolValue *CPV = 2096 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym, 2097 ARMPCLabelIndex, 4); 2098 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4); 2099 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 2100 Callee = DAG.getLoad( 2101 PtrVt, dl, DAG.getEntryNode(), CPAddr, 2102 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2103 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 2104 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel); 2105 } else { 2106 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0); 2107 } 2108 } 2109 2110 // FIXME: handle tail calls differently. 2111 unsigned CallOpc; 2112 if (Subtarget->isThumb()) { 2113 if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps()) 2114 CallOpc = ARMISD::CALL_NOLINK; 2115 else 2116 CallOpc = ARMISD::CALL; 2117 } else { 2118 if (!isDirect && !Subtarget->hasV5TOps()) 2119 CallOpc = ARMISD::CALL_NOLINK; 2120 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() && 2121 // Emit regular call when code size is the priority 2122 !MF.getFunction().optForMinSize()) 2123 // "mov lr, pc; b _foo" to avoid confusing the RSP 2124 CallOpc = ARMISD::CALL_NOLINK; 2125 else 2126 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL; 2127 } 2128 2129 std::vector<SDValue> Ops; 2130 Ops.push_back(Chain); 2131 Ops.push_back(Callee); 2132 2133 // Add argument registers to the end of the list so that they are known live 2134 // into the call. 2135 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 2136 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 2137 RegsToPass[i].second.getValueType())); 2138 2139 // Add a register mask operand representing the call-preserved registers. 2140 if (!isTailCall) { 2141 const uint32_t *Mask; 2142 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo(); 2143 if (isThisReturn) { 2144 // For 'this' returns, use the R0-preserving mask if applicable 2145 Mask = ARI->getThisReturnPreservedMask(MF, CallConv); 2146 if (!Mask) { 2147 // Set isThisReturn to false if the calling convention is not one that 2148 // allows 'returned' to be modeled in this way, so LowerCallResult does 2149 // not try to pass 'this' straight through 2150 isThisReturn = false; 2151 Mask = ARI->getCallPreservedMask(MF, CallConv); 2152 } 2153 } else 2154 Mask = ARI->getCallPreservedMask(MF, CallConv); 2155 2156 assert(Mask && "Missing call preserved mask for calling convention"); 2157 Ops.push_back(DAG.getRegisterMask(Mask)); 2158 } 2159 2160 if (InFlag.getNode()) 2161 Ops.push_back(InFlag); 2162 2163 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2164 if (isTailCall) { 2165 MF.getFrameInfo().setHasTailCall(); 2166 return DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops); 2167 } 2168 2169 // Returns a chain and a flag for retval copy to use. 2170 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); 2171 InFlag = Chain.getValue(1); 2172 2173 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 2174 DAG.getIntPtrConstant(0, dl, true), InFlag, dl); 2175 if (!Ins.empty()) 2176 InFlag = Chain.getValue(1); 2177 2178 // Handle result values, copying them out of physregs into vregs that we 2179 // return. 2180 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG, 2181 InVals, isThisReturn, 2182 isThisReturn ? OutVals[0] : SDValue()); 2183 } 2184 2185 /// HandleByVal - Every parameter *after* a byval parameter is passed 2186 /// on the stack. Remember the next parameter register to allocate, 2187 /// and then confiscate the rest of the parameter registers to insure 2188 /// this. 2189 void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size, 2190 unsigned Align) const { 2191 // Byval (as with any stack) slots are always at least 4 byte aligned. 2192 Align = std::max(Align, 4U); 2193 2194 unsigned Reg = State->AllocateReg(GPRArgRegs); 2195 if (!Reg) 2196 return; 2197 2198 unsigned AlignInRegs = Align / 4; 2199 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs; 2200 for (unsigned i = 0; i < Waste; ++i) 2201 Reg = State->AllocateReg(GPRArgRegs); 2202 2203 if (!Reg) 2204 return; 2205 2206 unsigned Excess = 4 * (ARM::R4 - Reg); 2207 2208 // Special case when NSAA != SP and parameter size greater than size of 2209 // all remained GPR regs. In that case we can't split parameter, we must 2210 // send it to stack. We also must set NCRN to R4, so waste all 2211 // remained registers. 2212 const unsigned NSAAOffset = State->getNextStackOffset(); 2213 if (NSAAOffset != 0 && Size > Excess) { 2214 while (State->AllocateReg(GPRArgRegs)) 2215 ; 2216 return; 2217 } 2218 2219 // First register for byval parameter is the first register that wasn't 2220 // allocated before this method call, so it would be "reg". 2221 // If parameter is small enough to be saved in range [reg, r4), then 2222 // the end (first after last) register would be reg + param-size-in-regs, 2223 // else parameter would be splitted between registers and stack, 2224 // end register would be r4 in this case. 2225 unsigned ByValRegBegin = Reg; 2226 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4); 2227 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd); 2228 // Note, first register is allocated in the beginning of function already, 2229 // allocate remained amount of registers we need. 2230 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i) 2231 State->AllocateReg(GPRArgRegs); 2232 // A byval parameter that is split between registers and memory needs its 2233 // size truncated here. 2234 // In the case where the entire structure fits in registers, we set the 2235 // size in memory to zero. 2236 Size = std::max<int>(Size - Excess, 0); 2237 } 2238 2239 /// MatchingStackOffset - Return true if the given stack call argument is 2240 /// already available in the same position (relatively) of the caller's 2241 /// incoming argument stack. 2242 static 2243 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, 2244 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, 2245 const TargetInstrInfo *TII) { 2246 unsigned Bytes = Arg.getValueSizeInBits() / 8; 2247 int FI = std::numeric_limits<int>::max(); 2248 if (Arg.getOpcode() == ISD::CopyFromReg) { 2249 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); 2250 if (!TargetRegisterInfo::isVirtualRegister(VR)) 2251 return false; 2252 MachineInstr *Def = MRI->getVRegDef(VR); 2253 if (!Def) 2254 return false; 2255 if (!Flags.isByVal()) { 2256 if (!TII->isLoadFromStackSlot(*Def, FI)) 2257 return false; 2258 } else { 2259 return false; 2260 } 2261 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) { 2262 if (Flags.isByVal()) 2263 // ByVal argument is passed in as a pointer but it's now being 2264 // dereferenced. e.g. 2265 // define @foo(%struct.X* %A) { 2266 // tail call @bar(%struct.X* byval %A) 2267 // } 2268 return false; 2269 SDValue Ptr = Ld->getBasePtr(); 2270 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr); 2271 if (!FINode) 2272 return false; 2273 FI = FINode->getIndex(); 2274 } else 2275 return false; 2276 2277 assert(FI != std::numeric_limits<int>::max()); 2278 if (!MFI.isFixedObjectIndex(FI)) 2279 return false; 2280 return Offset == MFI.getObjectOffset(FI) && Bytes == MFI.getObjectSize(FI); 2281 } 2282 2283 /// IsEligibleForTailCallOptimization - Check whether the call is eligible 2284 /// for tail call optimization. Targets which want to do tail call 2285 /// optimization should implement this function. 2286 bool 2287 ARMTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 2288 CallingConv::ID CalleeCC, 2289 bool isVarArg, 2290 bool isCalleeStructRet, 2291 bool isCallerStructRet, 2292 const SmallVectorImpl<ISD::OutputArg> &Outs, 2293 const SmallVectorImpl<SDValue> &OutVals, 2294 const SmallVectorImpl<ISD::InputArg> &Ins, 2295 SelectionDAG& DAG) const { 2296 MachineFunction &MF = DAG.getMachineFunction(); 2297 const Function &CallerF = MF.getFunction(); 2298 CallingConv::ID CallerCC = CallerF.getCallingConv(); 2299 2300 assert(Subtarget->supportsTailCall()); 2301 2302 // Tail calls to function pointers cannot be optimized for Thumb1 if the args 2303 // to the call take up r0-r3. The reason is that there are no legal registers 2304 // left to hold the pointer to the function to be called. 2305 if (Subtarget->isThumb1Only() && Outs.size() >= 4 && 2306 !isa<GlobalAddressSDNode>(Callee.getNode())) 2307 return false; 2308 2309 // Look for obvious safe cases to perform tail call optimization that do not 2310 // require ABI changes. This is what gcc calls sibcall. 2311 2312 // Exception-handling functions need a special set of instructions to indicate 2313 // a return to the hardware. Tail-calling another function would probably 2314 // break this. 2315 if (CallerF.hasFnAttribute("interrupt")) 2316 return false; 2317 2318 // Also avoid sibcall optimization if either caller or callee uses struct 2319 // return semantics. 2320 if (isCalleeStructRet || isCallerStructRet) 2321 return false; 2322 2323 // Externally-defined functions with weak linkage should not be 2324 // tail-called on ARM when the OS does not support dynamic 2325 // pre-emption of symbols, as the AAELF spec requires normal calls 2326 // to undefined weak functions to be replaced with a NOP or jump to the 2327 // next instruction. The behaviour of branch instructions in this 2328 // situation (as used for tail calls) is implementation-defined, so we 2329 // cannot rely on the linker replacing the tail call with a return. 2330 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 2331 const GlobalValue *GV = G->getGlobal(); 2332 const Triple &TT = getTargetMachine().getTargetTriple(); 2333 if (GV->hasExternalWeakLinkage() && 2334 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO())) 2335 return false; 2336 } 2337 2338 // Check that the call results are passed in the same way. 2339 LLVMContext &C = *DAG.getContext(); 2340 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins, 2341 CCAssignFnForReturn(CalleeCC, isVarArg), 2342 CCAssignFnForReturn(CallerCC, isVarArg))) 2343 return false; 2344 // The callee has to preserve all registers the caller needs to preserve. 2345 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); 2346 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); 2347 if (CalleeCC != CallerCC) { 2348 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); 2349 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) 2350 return false; 2351 } 2352 2353 // If Caller's vararg or byval argument has been split between registers and 2354 // stack, do not perform tail call, since part of the argument is in caller's 2355 // local frame. 2356 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>(); 2357 if (AFI_Caller->getArgRegsSaveSize()) 2358 return false; 2359 2360 // If the callee takes no arguments then go on to check the results of the 2361 // call. 2362 if (!Outs.empty()) { 2363 // Check if stack adjustment is needed. For now, do not do this if any 2364 // argument is passed on the stack. 2365 SmallVector<CCValAssign, 16> ArgLocs; 2366 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C); 2367 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg)); 2368 if (CCInfo.getNextStackOffset()) { 2369 // Check if the arguments are already laid out in the right way as 2370 // the caller's fixed stack objects. 2371 MachineFrameInfo &MFI = MF.getFrameInfo(); 2372 const MachineRegisterInfo *MRI = &MF.getRegInfo(); 2373 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 2374 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); 2375 i != e; 2376 ++i, ++realArgIdx) { 2377 CCValAssign &VA = ArgLocs[i]; 2378 EVT RegVT = VA.getLocVT(); 2379 SDValue Arg = OutVals[realArgIdx]; 2380 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; 2381 if (VA.getLocInfo() == CCValAssign::Indirect) 2382 return false; 2383 if (VA.needsCustom()) { 2384 // f64 and vector types are split into multiple registers or 2385 // register/stack-slot combinations. The types will not match 2386 // the registers; give up on memory f64 refs until we figure 2387 // out what to do about this. 2388 if (!VA.isRegLoc()) 2389 return false; 2390 if (!ArgLocs[++i].isRegLoc()) 2391 return false; 2392 if (RegVT == MVT::v2f64) { 2393 if (!ArgLocs[++i].isRegLoc()) 2394 return false; 2395 if (!ArgLocs[++i].isRegLoc()) 2396 return false; 2397 } 2398 } else if (!VA.isRegLoc()) { 2399 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags, 2400 MFI, MRI, TII)) 2401 return false; 2402 } 2403 } 2404 } 2405 2406 const MachineRegisterInfo &MRI = MF.getRegInfo(); 2407 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) 2408 return false; 2409 } 2410 2411 return true; 2412 } 2413 2414 bool 2415 ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 2416 MachineFunction &MF, bool isVarArg, 2417 const SmallVectorImpl<ISD::OutputArg> &Outs, 2418 LLVMContext &Context) const { 2419 SmallVector<CCValAssign, 16> RVLocs; 2420 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 2421 return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg)); 2422 } 2423 2424 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps, 2425 const SDLoc &DL, SelectionDAG &DAG) { 2426 const MachineFunction &MF = DAG.getMachineFunction(); 2427 const Function &F = MF.getFunction(); 2428 2429 StringRef IntKind = F.getFnAttribute("interrupt").getValueAsString(); 2430 2431 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset 2432 // version of the "preferred return address". These offsets affect the return 2433 // instruction if this is a return from PL1 without hypervisor extensions. 2434 // IRQ/FIQ: +4 "subs pc, lr, #4" 2435 // SWI: 0 "subs pc, lr, #0" 2436 // ABORT: +4 "subs pc, lr, #4" 2437 // UNDEF: +4/+2 "subs pc, lr, #0" 2438 // UNDEF varies depending on where the exception came from ARM or Thumb 2439 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0. 2440 2441 int64_t LROffset; 2442 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" || 2443 IntKind == "ABORT") 2444 LROffset = 4; 2445 else if (IntKind == "SWI" || IntKind == "UNDEF") 2446 LROffset = 0; 2447 else 2448 report_fatal_error("Unsupported interrupt attribute. If present, value " 2449 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF"); 2450 2451 RetOps.insert(RetOps.begin() + 1, 2452 DAG.getConstant(LROffset, DL, MVT::i32, false)); 2453 2454 return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps); 2455 } 2456 2457 SDValue 2458 ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 2459 bool isVarArg, 2460 const SmallVectorImpl<ISD::OutputArg> &Outs, 2461 const SmallVectorImpl<SDValue> &OutVals, 2462 const SDLoc &dl, SelectionDAG &DAG) const { 2463 // CCValAssign - represent the assignment of the return value to a location. 2464 SmallVector<CCValAssign, 16> RVLocs; 2465 2466 // CCState - Info about the registers and stack slots. 2467 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 2468 *DAG.getContext()); 2469 2470 // Analyze outgoing return values. 2471 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg)); 2472 2473 SDValue Flag; 2474 SmallVector<SDValue, 4> RetOps; 2475 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 2476 bool isLittleEndian = Subtarget->isLittle(); 2477 2478 MachineFunction &MF = DAG.getMachineFunction(); 2479 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2480 AFI->setReturnRegsCount(RVLocs.size()); 2481 2482 // Copy the result values into the output registers. 2483 for (unsigned i = 0, realRVLocIdx = 0; 2484 i != RVLocs.size(); 2485 ++i, ++realRVLocIdx) { 2486 CCValAssign &VA = RVLocs[i]; 2487 assert(VA.isRegLoc() && "Can only return in registers!"); 2488 2489 SDValue Arg = OutVals[realRVLocIdx]; 2490 bool ReturnF16 = false; 2491 2492 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) { 2493 // Half-precision return values can be returned like this: 2494 // 2495 // t11 f16 = fadd ... 2496 // t12: i16 = bitcast t11 2497 // t13: i32 = zero_extend t12 2498 // t14: f32 = bitcast t13 <~~~~~~~ Arg 2499 // 2500 // to avoid code generation for bitcasts, we simply set Arg to the node 2501 // that produces the f16 value, t11 in this case. 2502 // 2503 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) { 2504 SDValue ZE = Arg.getOperand(0); 2505 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) { 2506 SDValue BC = ZE.getOperand(0); 2507 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) { 2508 Arg = BC.getOperand(0); 2509 ReturnF16 = true; 2510 } 2511 } 2512 } 2513 } 2514 2515 switch (VA.getLocInfo()) { 2516 default: llvm_unreachable("Unknown loc info!"); 2517 case CCValAssign::Full: break; 2518 case CCValAssign::BCvt: 2519 if (!ReturnF16) 2520 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); 2521 break; 2522 } 2523 2524 if (VA.needsCustom()) { 2525 if (VA.getLocVT() == MVT::v2f64) { 2526 // Extract the first half and return it in two registers. 2527 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 2528 DAG.getConstant(0, dl, MVT::i32)); 2529 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl, 2530 DAG.getVTList(MVT::i32, MVT::i32), Half); 2531 2532 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2533 HalfGPRs.getValue(isLittleEndian ? 0 : 1), 2534 Flag); 2535 Flag = Chain.getValue(1); 2536 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2537 VA = RVLocs[++i]; // skip ahead to next loc 2538 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2539 HalfGPRs.getValue(isLittleEndian ? 1 : 0), 2540 Flag); 2541 Flag = Chain.getValue(1); 2542 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2543 VA = RVLocs[++i]; // skip ahead to next loc 2544 2545 // Extract the 2nd half and fall through to handle it as an f64 value. 2546 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, 2547 DAG.getConstant(1, dl, MVT::i32)); 2548 } 2549 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is 2550 // available. 2551 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl, 2552 DAG.getVTList(MVT::i32, MVT::i32), Arg); 2553 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2554 fmrrd.getValue(isLittleEndian ? 0 : 1), 2555 Flag); 2556 Flag = Chain.getValue(1); 2557 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2558 VA = RVLocs[++i]; // skip ahead to next loc 2559 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), 2560 fmrrd.getValue(isLittleEndian ? 1 : 0), 2561 Flag); 2562 } else 2563 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 2564 2565 // Guarantee that all emitted copies are 2566 // stuck together, avoiding something bad. 2567 Flag = Chain.getValue(1); 2568 RetOps.push_back(DAG.getRegister(VA.getLocReg(), 2569 ReturnF16 ? MVT::f16 : VA.getLocVT())); 2570 } 2571 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); 2572 const MCPhysReg *I = 2573 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 2574 if (I) { 2575 for (; *I; ++I) { 2576 if (ARM::GPRRegClass.contains(*I)) 2577 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); 2578 else if (ARM::DPRRegClass.contains(*I)) 2579 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); 2580 else 2581 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 2582 } 2583 } 2584 2585 // Update chain and glue. 2586 RetOps[0] = Chain; 2587 if (Flag.getNode()) 2588 RetOps.push_back(Flag); 2589 2590 // CPUs which aren't M-class use a special sequence to return from 2591 // exceptions (roughly, any instruction setting pc and cpsr simultaneously, 2592 // though we use "subs pc, lr, #N"). 2593 // 2594 // M-class CPUs actually use a normal return sequence with a special 2595 // (hardware-provided) value in LR, so the normal code path works. 2596 if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") && 2597 !Subtarget->isMClass()) { 2598 if (Subtarget->isThumb1Only()) 2599 report_fatal_error("interrupt attribute is not supported in Thumb1"); 2600 return LowerInterruptReturn(RetOps, dl, DAG); 2601 } 2602 2603 return DAG.getNode(ARMISD::RET_FLAG, dl, MVT::Other, RetOps); 2604 } 2605 2606 bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { 2607 if (N->getNumValues() != 1) 2608 return false; 2609 if (!N->hasNUsesOfValue(1, 0)) 2610 return false; 2611 2612 SDValue TCChain = Chain; 2613 SDNode *Copy = *N->use_begin(); 2614 if (Copy->getOpcode() == ISD::CopyToReg) { 2615 // If the copy has a glue operand, we conservatively assume it isn't safe to 2616 // perform a tail call. 2617 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) 2618 return false; 2619 TCChain = Copy->getOperand(0); 2620 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) { 2621 SDNode *VMov = Copy; 2622 // f64 returned in a pair of GPRs. 2623 SmallPtrSet<SDNode*, 2> Copies; 2624 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end(); 2625 UI != UE; ++UI) { 2626 if (UI->getOpcode() != ISD::CopyToReg) 2627 return false; 2628 Copies.insert(*UI); 2629 } 2630 if (Copies.size() > 2) 2631 return false; 2632 2633 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end(); 2634 UI != UE; ++UI) { 2635 SDValue UseChain = UI->getOperand(0); 2636 if (Copies.count(UseChain.getNode())) 2637 // Second CopyToReg 2638 Copy = *UI; 2639 else { 2640 // We are at the top of this chain. 2641 // If the copy has a glue operand, we conservatively assume it 2642 // isn't safe to perform a tail call. 2643 if (UI->getOperand(UI->getNumOperands()-1).getValueType() == MVT::Glue) 2644 return false; 2645 // First CopyToReg 2646 TCChain = UseChain; 2647 } 2648 } 2649 } else if (Copy->getOpcode() == ISD::BITCAST) { 2650 // f32 returned in a single GPR. 2651 if (!Copy->hasOneUse()) 2652 return false; 2653 Copy = *Copy->use_begin(); 2654 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0)) 2655 return false; 2656 // If the copy has a glue operand, we conservatively assume it isn't safe to 2657 // perform a tail call. 2658 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) 2659 return false; 2660 TCChain = Copy->getOperand(0); 2661 } else { 2662 return false; 2663 } 2664 2665 bool HasRet = false; 2666 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end(); 2667 UI != UE; ++UI) { 2668 if (UI->getOpcode() != ARMISD::RET_FLAG && 2669 UI->getOpcode() != ARMISD::INTRET_FLAG) 2670 return false; 2671 HasRet = true; 2672 } 2673 2674 if (!HasRet) 2675 return false; 2676 2677 Chain = TCChain; 2678 return true; 2679 } 2680 2681 bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 2682 if (!Subtarget->supportsTailCall()) 2683 return false; 2684 2685 auto Attr = 2686 CI->getParent()->getParent()->getFnAttribute("disable-tail-calls"); 2687 if (!CI->isTailCall() || Attr.getValueAsString() == "true") 2688 return false; 2689 2690 return true; 2691 } 2692 2693 // Trying to write a 64 bit value so need to split into two 32 bit values first, 2694 // and pass the lower and high parts through. 2695 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) { 2696 SDLoc DL(Op); 2697 SDValue WriteValue = Op->getOperand(2); 2698 2699 // This function is only supposed to be called for i64 type argument. 2700 assert(WriteValue.getValueType() == MVT::i64 2701 && "LowerWRITE_REGISTER called for non-i64 type argument."); 2702 2703 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue, 2704 DAG.getConstant(0, DL, MVT::i32)); 2705 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue, 2706 DAG.getConstant(1, DL, MVT::i32)); 2707 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi }; 2708 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops); 2709 } 2710 2711 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 2712 // their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is 2713 // one of the above mentioned nodes. It has to be wrapped because otherwise 2714 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 2715 // be used to form addressing mode. These wrapped nodes will be selected 2716 // into MOVi. 2717 SDValue ARMTargetLowering::LowerConstantPool(SDValue Op, 2718 SelectionDAG &DAG) const { 2719 EVT PtrVT = Op.getValueType(); 2720 // FIXME there is no actual debug info here 2721 SDLoc dl(Op); 2722 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 2723 SDValue Res; 2724 2725 // When generating execute-only code Constant Pools must be promoted to the 2726 // global data section. It's a bit ugly that we can't share them across basic 2727 // blocks, but this way we guarantee that execute-only behaves correct with 2728 // position-independent addressing modes. 2729 if (Subtarget->genExecuteOnly()) { 2730 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>(); 2731 auto T = const_cast<Type*>(CP->getType()); 2732 auto C = const_cast<Constant*>(CP->getConstVal()); 2733 auto M = const_cast<Module*>(DAG.getMachineFunction(). 2734 getFunction().getParent()); 2735 auto GV = new GlobalVariable( 2736 *M, T, /*isConst=*/true, GlobalVariable::InternalLinkage, C, 2737 Twine(DAG.getDataLayout().getPrivateGlobalPrefix()) + "CP" + 2738 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" + 2739 Twine(AFI->createPICLabelUId()) 2740 ); 2741 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV), 2742 dl, PtrVT); 2743 return LowerGlobalAddress(GA, DAG); 2744 } 2745 2746 if (CP->isMachineConstantPoolEntry()) 2747 Res = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 2748 CP->getAlignment()); 2749 else 2750 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 2751 CP->getAlignment()); 2752 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res); 2753 } 2754 2755 unsigned ARMTargetLowering::getJumpTableEncoding() const { 2756 return MachineJumpTableInfo::EK_Inline; 2757 } 2758 2759 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op, 2760 SelectionDAG &DAG) const { 2761 MachineFunction &MF = DAG.getMachineFunction(); 2762 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2763 unsigned ARMPCLabelIndex = 0; 2764 SDLoc DL(Op); 2765 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2766 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); 2767 SDValue CPAddr; 2768 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI(); 2769 if (!IsPositionIndependent) { 2770 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, 4); 2771 } else { 2772 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8; 2773 ARMPCLabelIndex = AFI->createPICLabelUId(); 2774 ARMConstantPoolValue *CPV = 2775 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex, 2776 ARMCP::CPBlockAddress, PCAdj); 2777 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2778 } 2779 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr); 2780 SDValue Result = DAG.getLoad( 2781 PtrVT, DL, DAG.getEntryNode(), CPAddr, 2782 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2783 if (!IsPositionIndependent) 2784 return Result; 2785 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32); 2786 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel); 2787 } 2788 2789 /// Convert a TLS address reference into the correct sequence of loads 2790 /// and calls to compute the variable's address for Darwin, and return an 2791 /// SDValue containing the final node. 2792 2793 /// Darwin only has one TLS scheme which must be capable of dealing with the 2794 /// fully general situation, in the worst case. This means: 2795 /// + "extern __thread" declaration. 2796 /// + Defined in a possibly unknown dynamic library. 2797 /// 2798 /// The general system is that each __thread variable has a [3 x i32] descriptor 2799 /// which contains information used by the runtime to calculate the address. The 2800 /// only part of this the compiler needs to know about is the first word, which 2801 /// contains a function pointer that must be called with the address of the 2802 /// entire descriptor in "r0". 2803 /// 2804 /// Since this descriptor may be in a different unit, in general access must 2805 /// proceed along the usual ARM rules. A common sequence to produce is: 2806 /// 2807 /// movw rT1, :lower16:_var$non_lazy_ptr 2808 /// movt rT1, :upper16:_var$non_lazy_ptr 2809 /// ldr r0, [rT1] 2810 /// ldr rT2, [r0] 2811 /// blx rT2 2812 /// [...address now in r0...] 2813 SDValue 2814 ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op, 2815 SelectionDAG &DAG) const { 2816 assert(Subtarget->isTargetDarwin() && 2817 "This function expects a Darwin target"); 2818 SDLoc DL(Op); 2819 2820 // First step is to get the address of the actua global symbol. This is where 2821 // the TLS descriptor lives. 2822 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG); 2823 2824 // The first entry in the descriptor is a function pointer that we must call 2825 // to obtain the address of the variable. 2826 SDValue Chain = DAG.getEntryNode(); 2827 SDValue FuncTLVGet = DAG.getLoad( 2828 MVT::i32, DL, Chain, DescAddr, 2829 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 2830 /* Alignment = */ 4, 2831 MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable | 2832 MachineMemOperand::MOInvariant); 2833 Chain = FuncTLVGet.getValue(1); 2834 2835 MachineFunction &F = DAG.getMachineFunction(); 2836 MachineFrameInfo &MFI = F.getFrameInfo(); 2837 MFI.setAdjustsStack(true); 2838 2839 // TLS calls preserve all registers except those that absolutely must be 2840 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be 2841 // silly). 2842 auto TRI = 2843 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo(); 2844 auto ARI = static_cast<const ARMRegisterInfo *>(TRI); 2845 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction()); 2846 2847 // Finally, we can make the call. This is just a degenerate version of a 2848 // normal AArch64 call node: r0 takes the address of the descriptor, and 2849 // returns the address of the variable in this thread. 2850 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue()); 2851 Chain = 2852 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue), 2853 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32), 2854 DAG.getRegisterMask(Mask), Chain.getValue(1)); 2855 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1)); 2856 } 2857 2858 SDValue 2859 ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op, 2860 SelectionDAG &DAG) const { 2861 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering"); 2862 2863 SDValue Chain = DAG.getEntryNode(); 2864 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2865 SDLoc DL(Op); 2866 2867 // Load the current TEB (thread environment block) 2868 SDValue Ops[] = {Chain, 2869 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32), 2870 DAG.getConstant(15, DL, MVT::i32), 2871 DAG.getConstant(0, DL, MVT::i32), 2872 DAG.getConstant(13, DL, MVT::i32), 2873 DAG.getConstant(0, DL, MVT::i32), 2874 DAG.getConstant(2, DL, MVT::i32)}; 2875 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, 2876 DAG.getVTList(MVT::i32, MVT::Other), Ops); 2877 2878 SDValue TEB = CurrentTEB.getValue(0); 2879 Chain = CurrentTEB.getValue(1); 2880 2881 // Load the ThreadLocalStoragePointer from the TEB 2882 // A pointer to the TLS array is located at offset 0x2c from the TEB. 2883 SDValue TLSArray = 2884 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL)); 2885 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo()); 2886 2887 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4 2888 // offset into the TLSArray. 2889 2890 // Load the TLS index from the C runtime 2891 SDValue TLSIndex = 2892 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG); 2893 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex); 2894 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo()); 2895 2896 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex, 2897 DAG.getConstant(2, DL, MVT::i32)); 2898 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain, 2899 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot), 2900 MachinePointerInfo()); 2901 2902 // Get the offset of the start of the .tls section (section base) 2903 const auto *GA = cast<GlobalAddressSDNode>(Op); 2904 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL); 2905 SDValue Offset = DAG.getLoad( 2906 PtrVT, DL, Chain, DAG.getNode(ARMISD::Wrapper, DL, MVT::i32, 2907 DAG.getTargetConstantPool(CPV, PtrVT, 4)), 2908 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2909 2910 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset); 2911 } 2912 2913 // Lower ISD::GlobalTLSAddress using the "general dynamic" model 2914 SDValue 2915 ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, 2916 SelectionDAG &DAG) const { 2917 SDLoc dl(GA); 2918 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2919 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; 2920 MachineFunction &MF = DAG.getMachineFunction(); 2921 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2922 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2923 ARMConstantPoolValue *CPV = 2924 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, 2925 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true); 2926 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2927 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument); 2928 Argument = DAG.getLoad( 2929 PtrVT, dl, DAG.getEntryNode(), Argument, 2930 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2931 SDValue Chain = Argument.getValue(1); 2932 2933 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 2934 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel); 2935 2936 // call __tls_get_addr. 2937 ArgListTy Args; 2938 ArgListEntry Entry; 2939 Entry.Node = Argument; 2940 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext()); 2941 Args.push_back(Entry); 2942 2943 // FIXME: is there useful debug info available here? 2944 TargetLowering::CallLoweringInfo CLI(DAG); 2945 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( 2946 CallingConv::C, Type::getInt32Ty(*DAG.getContext()), 2947 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args)); 2948 2949 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 2950 return CallResult.first; 2951 } 2952 2953 // Lower ISD::GlobalTLSAddress using the "initial exec" or 2954 // "local exec" model. 2955 SDValue 2956 ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA, 2957 SelectionDAG &DAG, 2958 TLSModel::Model model) const { 2959 const GlobalValue *GV = GA->getGlobal(); 2960 SDLoc dl(GA); 2961 SDValue Offset; 2962 SDValue Chain = DAG.getEntryNode(); 2963 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2964 // Get the Thread Pointer 2965 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT); 2966 2967 if (model == TLSModel::InitialExec) { 2968 MachineFunction &MF = DAG.getMachineFunction(); 2969 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 2970 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 2971 // Initial exec model. 2972 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; 2973 ARMConstantPoolValue *CPV = 2974 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, 2975 ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF, 2976 true); 2977 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2978 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset); 2979 Offset = DAG.getLoad( 2980 PtrVT, dl, Chain, Offset, 2981 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2982 Chain = Offset.getValue(1); 2983 2984 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 2985 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel); 2986 2987 Offset = DAG.getLoad( 2988 PtrVT, dl, Chain, Offset, 2989 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2990 } else { 2991 // local exec model 2992 assert(model == TLSModel::LocalExec); 2993 ARMConstantPoolValue *CPV = 2994 ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF); 2995 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4); 2996 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset); 2997 Offset = DAG.getLoad( 2998 PtrVT, dl, Chain, Offset, 2999 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3000 } 3001 3002 // The address of the thread local variable is the add of the thread 3003 // pointer with the offset of the variable. 3004 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); 3005 } 3006 3007 SDValue 3008 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { 3009 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 3010 if (DAG.getTarget().useEmulatedTLS()) 3011 return LowerToTLSEmulatedModel(GA, DAG); 3012 3013 if (Subtarget->isTargetDarwin()) 3014 return LowerGlobalTLSAddressDarwin(Op, DAG); 3015 3016 if (Subtarget->isTargetWindows()) 3017 return LowerGlobalTLSAddressWindows(Op, DAG); 3018 3019 // TODO: implement the "local dynamic" model 3020 assert(Subtarget->isTargetELF() && "Only ELF implemented here"); 3021 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal()); 3022 3023 switch (model) { 3024 case TLSModel::GeneralDynamic: 3025 case TLSModel::LocalDynamic: 3026 return LowerToTLSGeneralDynamicModel(GA, DAG); 3027 case TLSModel::InitialExec: 3028 case TLSModel::LocalExec: 3029 return LowerToTLSExecModels(GA, DAG, model); 3030 } 3031 llvm_unreachable("bogus TLS model"); 3032 } 3033 3034 /// Return true if all users of V are within function F, looking through 3035 /// ConstantExprs. 3036 static bool allUsersAreInFunction(const Value *V, const Function *F) { 3037 SmallVector<const User*,4> Worklist; 3038 for (auto *U : V->users()) 3039 Worklist.push_back(U); 3040 while (!Worklist.empty()) { 3041 auto *U = Worklist.pop_back_val(); 3042 if (isa<ConstantExpr>(U)) { 3043 for (auto *UU : U->users()) 3044 Worklist.push_back(UU); 3045 continue; 3046 } 3047 3048 auto *I = dyn_cast<Instruction>(U); 3049 if (!I || I->getParent()->getParent() != F) 3050 return false; 3051 } 3052 return true; 3053 } 3054 3055 /// Return true if all users of V are within some (any) function, looking through 3056 /// ConstantExprs. In other words, are there any global constant users? 3057 static bool allUsersAreInFunctions(const Value *V) { 3058 SmallVector<const User*,4> Worklist; 3059 for (auto *U : V->users()) 3060 Worklist.push_back(U); 3061 while (!Worklist.empty()) { 3062 auto *U = Worklist.pop_back_val(); 3063 if (isa<ConstantExpr>(U)) { 3064 for (auto *UU : U->users()) 3065 Worklist.push_back(UU); 3066 continue; 3067 } 3068 3069 if (!isa<Instruction>(U)) 3070 return false; 3071 } 3072 return true; 3073 } 3074 3075 // Return true if T is an integer, float or an array/vector of either. 3076 static bool isSimpleType(Type *T) { 3077 if (T->isIntegerTy() || T->isFloatingPointTy()) 3078 return true; 3079 Type *SubT = nullptr; 3080 if (T->isArrayTy()) 3081 SubT = T->getArrayElementType(); 3082 else if (T->isVectorTy()) 3083 SubT = T->getVectorElementType(); 3084 else 3085 return false; 3086 return SubT->isIntegerTy() || SubT->isFloatingPointTy(); 3087 } 3088 3089 static SDValue promoteToConstantPool(const GlobalValue *GV, SelectionDAG &DAG, 3090 EVT PtrVT, const SDLoc &dl) { 3091 // If we're creating a pool entry for a constant global with unnamed address, 3092 // and the global is small enough, we can emit it inline into the constant pool 3093 // to save ourselves an indirection. 3094 // 3095 // This is a win if the constant is only used in one function (so it doesn't 3096 // need to be duplicated) or duplicating the constant wouldn't increase code 3097 // size (implying the constant is no larger than 4 bytes). 3098 const Function &F = DAG.getMachineFunction().getFunction(); 3099 3100 // We rely on this decision to inline being idemopotent and unrelated to the 3101 // use-site. We know that if we inline a variable at one use site, we'll 3102 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel 3103 // doesn't know about this optimization, so bail out if it's enabled else 3104 // we could decide to inline here (and thus never emit the GV) but require 3105 // the GV from fast-isel generated code. 3106 if (!EnableConstpoolPromotion || 3107 DAG.getMachineFunction().getTarget().Options.EnableFastISel) 3108 return SDValue(); 3109 3110 auto *GVar = dyn_cast<GlobalVariable>(GV); 3111 if (!GVar || !GVar->hasInitializer() || 3112 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() || 3113 !GVar->hasLocalLinkage()) 3114 return SDValue(); 3115 3116 // Ensure that we don't try and inline any type that contains pointers. If 3117 // we inline a value that contains relocations, we move the relocations from 3118 // .data to .text which is not ideal. 3119 auto *Init = GVar->getInitializer(); 3120 if (!isSimpleType(Init->getType())) 3121 return SDValue(); 3122 3123 // The constant islands pass can only really deal with alignment requests 3124 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote 3125 // any type wanting greater alignment requirements than 4 bytes. We also 3126 // can only promote constants that are multiples of 4 bytes in size or 3127 // are paddable to a multiple of 4. Currently we only try and pad constants 3128 // that are strings for simplicity. 3129 auto *CDAInit = dyn_cast<ConstantDataArray>(Init); 3130 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType()); 3131 unsigned Align = GVar->getAlignment(); 3132 unsigned RequiredPadding = 4 - (Size % 4); 3133 bool PaddingPossible = 3134 RequiredPadding == 4 || (CDAInit && CDAInit->isString()); 3135 if (!PaddingPossible || Align > 4 || Size > ConstpoolPromotionMaxSize || 3136 Size == 0) 3137 return SDValue(); 3138 3139 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding); 3140 MachineFunction &MF = DAG.getMachineFunction(); 3141 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3142 3143 // We can't bloat the constant pool too much, else the ConstantIslands pass 3144 // may fail to converge. If we haven't promoted this global yet (it may have 3145 // multiple uses), and promoting it would increase the constant pool size (Sz 3146 // > 4), ensure we have space to do so up to MaxTotal. 3147 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4) 3148 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >= 3149 ConstpoolPromotionMaxTotal) 3150 return SDValue(); 3151 3152 // This is only valid if all users are in a single function OR it has users 3153 // in multiple functions but it no larger than a pointer. We also check if 3154 // GVar has constant (non-ConstantExpr) users. If so, it essentially has its 3155 // address taken. 3156 if (!allUsersAreInFunction(GVar, &F) && 3157 !(Size <= 4 && allUsersAreInFunctions(GVar))) 3158 return SDValue(); 3159 3160 // We're going to inline this global. Pad it out if needed. 3161 if (RequiredPadding != 4) { 3162 StringRef S = CDAInit->getAsString(); 3163 3164 SmallVector<uint8_t,16> V(S.size()); 3165 std::copy(S.bytes_begin(), S.bytes_end(), V.begin()); 3166 while (RequiredPadding--) 3167 V.push_back(0); 3168 Init = ConstantDataArray::get(*DAG.getContext(), V); 3169 } 3170 3171 auto CPVal = ARMConstantPoolConstant::Create(GVar, Init); 3172 SDValue CPAddr = 3173 DAG.getTargetConstantPool(CPVal, PtrVT, /*Align=*/4); 3174 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) { 3175 AFI->markGlobalAsPromotedToConstantPool(GVar); 3176 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() + 3177 PaddedSize - 4); 3178 } 3179 ++NumConstpoolPromoted; 3180 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3181 } 3182 3183 bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const { 3184 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 3185 GV = GA->getBaseObject(); 3186 return (isa<GlobalVariable>(GV) && cast<GlobalVariable>(GV)->isConstant()) || 3187 isa<Function>(GV); 3188 } 3189 3190 SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op, 3191 SelectionDAG &DAG) const { 3192 switch (Subtarget->getTargetTriple().getObjectFormat()) { 3193 default: llvm_unreachable("unknown object format"); 3194 case Triple::COFF: 3195 return LowerGlobalAddressWindows(Op, DAG); 3196 case Triple::ELF: 3197 return LowerGlobalAddressELF(Op, DAG); 3198 case Triple::MachO: 3199 return LowerGlobalAddressDarwin(Op, DAG); 3200 } 3201 } 3202 3203 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op, 3204 SelectionDAG &DAG) const { 3205 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3206 SDLoc dl(Op); 3207 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3208 const TargetMachine &TM = getTargetMachine(); 3209 bool IsRO = isReadOnly(GV); 3210 3211 // promoteToConstantPool only if not generating XO text section 3212 if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV) && !Subtarget->genExecuteOnly()) 3213 if (SDValue V = promoteToConstantPool(GV, DAG, PtrVT, dl)) 3214 return V; 3215 3216 if (isPositionIndependent()) { 3217 bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV); 3218 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 3219 UseGOT_PREL ? ARMII::MO_GOT : 0); 3220 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G); 3221 if (UseGOT_PREL) 3222 Result = 3223 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result, 3224 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 3225 return Result; 3226 } else if (Subtarget->isROPI() && IsRO) { 3227 // PC-relative. 3228 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT); 3229 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G); 3230 return Result; 3231 } else if (Subtarget->isRWPI() && !IsRO) { 3232 // SB-relative. 3233 SDValue RelAddr; 3234 if (Subtarget->useMovt(DAG.getMachineFunction())) { 3235 ++NumMovwMovt; 3236 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL); 3237 RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G); 3238 } else { // use literal pool for address constant 3239 ARMConstantPoolValue *CPV = 3240 ARMConstantPoolConstant::Create(GV, ARMCP::SBREL); 3241 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4); 3242 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3243 RelAddr = DAG.getLoad( 3244 PtrVT, dl, DAG.getEntryNode(), CPAddr, 3245 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3246 } 3247 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT); 3248 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr); 3249 return Result; 3250 } 3251 3252 // If we have T2 ops, we can materialize the address directly via movt/movw 3253 // pair. This is always cheaper. 3254 if (Subtarget->useMovt(DAG.getMachineFunction())) { 3255 ++NumMovwMovt; 3256 // FIXME: Once remat is capable of dealing with instructions with register 3257 // operands, expand this into two nodes. 3258 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT, 3259 DAG.getTargetGlobalAddress(GV, dl, PtrVT)); 3260 } else { 3261 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4); 3262 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3263 return DAG.getLoad( 3264 PtrVT, dl, DAG.getEntryNode(), CPAddr, 3265 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3266 } 3267 } 3268 3269 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op, 3270 SelectionDAG &DAG) const { 3271 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && 3272 "ROPI/RWPI not currently supported for Darwin"); 3273 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3274 SDLoc dl(Op); 3275 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3276 3277 if (Subtarget->useMovt(DAG.getMachineFunction())) 3278 ++NumMovwMovt; 3279 3280 // FIXME: Once remat is capable of dealing with instructions with register 3281 // operands, expand this into multiple nodes 3282 unsigned Wrapper = 3283 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper; 3284 3285 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY); 3286 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G); 3287 3288 if (Subtarget->isGVIndirectSymbol(GV)) 3289 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result, 3290 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 3291 return Result; 3292 } 3293 3294 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op, 3295 SelectionDAG &DAG) const { 3296 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported"); 3297 assert(Subtarget->useMovt(DAG.getMachineFunction()) && 3298 "Windows on ARM expects to use movw/movt"); 3299 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && 3300 "ROPI/RWPI not currently supported for Windows"); 3301 3302 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3303 const ARMII::TOF TargetFlags = 3304 (GV->hasDLLImportStorageClass() ? ARMII::MO_DLLIMPORT : ARMII::MO_NO_FLAG); 3305 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3306 SDValue Result; 3307 SDLoc DL(Op); 3308 3309 ++NumMovwMovt; 3310 3311 // FIXME: Once remat is capable of dealing with instructions with register 3312 // operands, expand this into two nodes. 3313 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, 3314 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*Offset=*/0, 3315 TargetFlags)); 3316 if (GV->hasDLLImportStorageClass()) 3317 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 3318 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 3319 return Result; 3320 } 3321 3322 SDValue 3323 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const { 3324 SDLoc dl(Op); 3325 SDValue Val = DAG.getConstant(0, dl, MVT::i32); 3326 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl, 3327 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0), 3328 Op.getOperand(1), Val); 3329 } 3330 3331 SDValue 3332 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const { 3333 SDLoc dl(Op); 3334 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0), 3335 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32)); 3336 } 3337 3338 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, 3339 SelectionDAG &DAG) const { 3340 SDLoc dl(Op); 3341 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other, 3342 Op.getOperand(0)); 3343 } 3344 3345 SDValue 3346 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG, 3347 const ARMSubtarget *Subtarget) const { 3348 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 3349 SDLoc dl(Op); 3350 switch (IntNo) { 3351 default: return SDValue(); // Don't custom lower most intrinsics. 3352 case Intrinsic::thread_pointer: { 3353 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3354 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT); 3355 } 3356 case Intrinsic::eh_sjlj_lsda: { 3357 MachineFunction &MF = DAG.getMachineFunction(); 3358 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3359 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); 3360 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3361 SDValue CPAddr; 3362 bool IsPositionIndependent = isPositionIndependent(); 3363 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0; 3364 ARMConstantPoolValue *CPV = 3365 ARMConstantPoolConstant::Create(&MF.getFunction(), ARMPCLabelIndex, 3366 ARMCP::CPLSDA, PCAdj); 3367 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4); 3368 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr); 3369 SDValue Result = DAG.getLoad( 3370 PtrVT, dl, DAG.getEntryNode(), CPAddr, 3371 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 3372 3373 if (IsPositionIndependent) { 3374 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32); 3375 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel); 3376 } 3377 return Result; 3378 } 3379 case Intrinsic::arm_neon_vabs: 3380 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(), 3381 Op.getOperand(1)); 3382 case Intrinsic::arm_neon_vmulls: 3383 case Intrinsic::arm_neon_vmullu: { 3384 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls) 3385 ? ARMISD::VMULLs : ARMISD::VMULLu; 3386 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3387 Op.getOperand(1), Op.getOperand(2)); 3388 } 3389 case Intrinsic::arm_neon_vminnm: 3390 case Intrinsic::arm_neon_vmaxnm: { 3391 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm) 3392 ? ISD::FMINNUM : ISD::FMAXNUM; 3393 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3394 Op.getOperand(1), Op.getOperand(2)); 3395 } 3396 case Intrinsic::arm_neon_vminu: 3397 case Intrinsic::arm_neon_vmaxu: { 3398 if (Op.getValueType().isFloatingPoint()) 3399 return SDValue(); 3400 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu) 3401 ? ISD::UMIN : ISD::UMAX; 3402 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3403 Op.getOperand(1), Op.getOperand(2)); 3404 } 3405 case Intrinsic::arm_neon_vmins: 3406 case Intrinsic::arm_neon_vmaxs: { 3407 // v{min,max}s is overloaded between signed integers and floats. 3408 if (!Op.getValueType().isFloatingPoint()) { 3409 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins) 3410 ? ISD::SMIN : ISD::SMAX; 3411 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3412 Op.getOperand(1), Op.getOperand(2)); 3413 } 3414 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins) 3415 ? ISD::FMINNAN : ISD::FMAXNAN; 3416 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(), 3417 Op.getOperand(1), Op.getOperand(2)); 3418 } 3419 case Intrinsic::arm_neon_vtbl1: 3420 return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(), 3421 Op.getOperand(1), Op.getOperand(2)); 3422 case Intrinsic::arm_neon_vtbl2: 3423 return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(), 3424 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 3425 } 3426 } 3427 3428 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, 3429 const ARMSubtarget *Subtarget) { 3430 SDLoc dl(Op); 3431 ConstantSDNode *SSIDNode = cast<ConstantSDNode>(Op.getOperand(2)); 3432 auto SSID = static_cast<SyncScope::ID>(SSIDNode->getZExtValue()); 3433 if (SSID == SyncScope::SingleThread) 3434 return Op; 3435 3436 if (!Subtarget->hasDataBarrier()) { 3437 // Some ARMv6 cpus can support data barriers with an mcr instruction. 3438 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get 3439 // here. 3440 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() && 3441 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!"); 3442 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0), 3443 DAG.getConstant(0, dl, MVT::i32)); 3444 } 3445 3446 ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1)); 3447 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue()); 3448 ARM_MB::MemBOpt Domain = ARM_MB::ISH; 3449 if (Subtarget->isMClass()) { 3450 // Only a full system barrier exists in the M-class architectures. 3451 Domain = ARM_MB::SY; 3452 } else if (Subtarget->preferISHSTBarriers() && 3453 Ord == AtomicOrdering::Release) { 3454 // Swift happens to implement ISHST barriers in a way that's compatible with 3455 // Release semantics but weaker than ISH so we'd be fools not to use 3456 // it. Beware: other processors probably don't! 3457 Domain = ARM_MB::ISHST; 3458 } 3459 3460 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0), 3461 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32), 3462 DAG.getConstant(Domain, dl, MVT::i32)); 3463 } 3464 3465 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG, 3466 const ARMSubtarget *Subtarget) { 3467 // ARM pre v5TE and Thumb1 does not have preload instructions. 3468 if (!(Subtarget->isThumb2() || 3469 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps()))) 3470 // Just preserve the chain. 3471 return Op.getOperand(0); 3472 3473 SDLoc dl(Op); 3474 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1; 3475 if (!isRead && 3476 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension())) 3477 // ARMv7 with MP extension has PLDW. 3478 return Op.getOperand(0); 3479 3480 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 3481 if (Subtarget->isThumb()) { 3482 // Invert the bits. 3483 isRead = ~isRead & 1; 3484 isData = ~isData & 1; 3485 } 3486 3487 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0), 3488 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32), 3489 DAG.getConstant(isData, dl, MVT::i32)); 3490 } 3491 3492 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) { 3493 MachineFunction &MF = DAG.getMachineFunction(); 3494 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>(); 3495 3496 // vastart just stores the address of the VarArgsFrameIndex slot into the 3497 // memory location argument. 3498 SDLoc dl(Op); 3499 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 3500 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3501 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3502 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 3503 MachinePointerInfo(SV)); 3504 } 3505 3506 SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA, 3507 CCValAssign &NextVA, 3508 SDValue &Root, 3509 SelectionDAG &DAG, 3510 const SDLoc &dl) const { 3511 MachineFunction &MF = DAG.getMachineFunction(); 3512 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3513 3514 const TargetRegisterClass *RC; 3515 if (AFI->isThumb1OnlyFunction()) 3516 RC = &ARM::tGPRRegClass; 3517 else 3518 RC = &ARM::GPRRegClass; 3519 3520 // Transform the arguments stored in physical registers into virtual ones. 3521 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 3522 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32); 3523 3524 SDValue ArgValue2; 3525 if (NextVA.isMemLoc()) { 3526 MachineFrameInfo &MFI = MF.getFrameInfo(); 3527 int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true); 3528 3529 // Create load node to retrieve arguments from the stack. 3530 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 3531 ArgValue2 = DAG.getLoad( 3532 MVT::i32, dl, Root, FIN, 3533 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)); 3534 } else { 3535 Reg = MF.addLiveIn(NextVA.getLocReg(), RC); 3536 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32); 3537 } 3538 if (!Subtarget->isLittle()) 3539 std::swap (ArgValue, ArgValue2); 3540 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2); 3541 } 3542 3543 // The remaining GPRs hold either the beginning of variable-argument 3544 // data, or the beginning of an aggregate passed by value (usually 3545 // byval). Either way, we allocate stack slots adjacent to the data 3546 // provided by our caller, and store the unallocated registers there. 3547 // If this is a variadic function, the va_list pointer will begin with 3548 // these values; otherwise, this reassembles a (byval) structure that 3549 // was split between registers and memory. 3550 // Return: The frame index registers were stored into. 3551 int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG, 3552 const SDLoc &dl, SDValue &Chain, 3553 const Value *OrigArg, 3554 unsigned InRegsParamRecordIdx, 3555 int ArgOffset, unsigned ArgSize) const { 3556 // Currently, two use-cases possible: 3557 // Case #1. Non-var-args function, and we meet first byval parameter. 3558 // Setup first unallocated register as first byval register; 3559 // eat all remained registers 3560 // (these two actions are performed by HandleByVal method). 3561 // Then, here, we initialize stack frame with 3562 // "store-reg" instructions. 3563 // Case #2. Var-args function, that doesn't contain byval parameters. 3564 // The same: eat all remained unallocated registers, 3565 // initialize stack frame. 3566 3567 MachineFunction &MF = DAG.getMachineFunction(); 3568 MachineFrameInfo &MFI = MF.getFrameInfo(); 3569 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3570 unsigned RBegin, REnd; 3571 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) { 3572 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd); 3573 } else { 3574 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs); 3575 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx]; 3576 REnd = ARM::R4; 3577 } 3578 3579 if (REnd != RBegin) 3580 ArgOffset = -4 * (ARM::R4 - RBegin); 3581 3582 auto PtrVT = getPointerTy(DAG.getDataLayout()); 3583 int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false); 3584 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT); 3585 3586 SmallVector<SDValue, 4> MemOps; 3587 const TargetRegisterClass *RC = 3588 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass; 3589 3590 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) { 3591 unsigned VReg = MF.addLiveIn(Reg, RC); 3592 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); 3593 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3594 MachinePointerInfo(OrigArg, 4 * i)); 3595 MemOps.push_back(Store); 3596 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT)); 3597 } 3598 3599 if (!MemOps.empty()) 3600 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3601 return FrameIndex; 3602 } 3603 3604 // Setup stack frame, the va_list pointer will start from. 3605 void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG, 3606 const SDLoc &dl, SDValue &Chain, 3607 unsigned ArgOffset, 3608 unsigned TotalArgRegsSaveSize, 3609 bool ForceMutable) const { 3610 MachineFunction &MF = DAG.getMachineFunction(); 3611 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3612 3613 // Try to store any remaining integer argument regs 3614 // to their spots on the stack so that they may be loaded by dereferencing 3615 // the result of va_next. 3616 // If there is no regs to be stored, just point address after last 3617 // argument passed via stack. 3618 int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, nullptr, 3619 CCInfo.getInRegsParamsCount(), 3620 CCInfo.getNextStackOffset(), 4); 3621 AFI->setVarArgsFrameIndex(FrameIndex); 3622 } 3623 3624 SDValue ARMTargetLowering::LowerFormalArguments( 3625 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3626 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3627 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3628 MachineFunction &MF = DAG.getMachineFunction(); 3629 MachineFrameInfo &MFI = MF.getFrameInfo(); 3630 3631 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 3632 3633 // Assign locations to all of the incoming arguments. 3634 SmallVector<CCValAssign, 16> ArgLocs; 3635 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 3636 *DAG.getContext()); 3637 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg)); 3638 3639 SmallVector<SDValue, 16> ArgValues; 3640 SDValue ArgValue; 3641 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin(); 3642 unsigned CurArgIdx = 0; 3643 3644 // Initially ArgRegsSaveSize is zero. 3645 // Then we increase this value each time we meet byval parameter. 3646 // We also increase this value in case of varargs function. 3647 AFI->setArgRegsSaveSize(0); 3648 3649 // Calculate the amount of stack space that we need to allocate to store 3650 // byval and variadic arguments that are passed in registers. 3651 // We need to know this before we allocate the first byval or variadic 3652 // argument, as they will be allocated a stack slot below the CFA (Canonical 3653 // Frame Address, the stack pointer at entry to the function). 3654 unsigned ArgRegBegin = ARM::R4; 3655 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3656 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount()) 3657 break; 3658 3659 CCValAssign &VA = ArgLocs[i]; 3660 unsigned Index = VA.getValNo(); 3661 ISD::ArgFlagsTy Flags = Ins[Index].Flags; 3662 if (!Flags.isByVal()) 3663 continue; 3664 3665 assert(VA.isMemLoc() && "unexpected byval pointer in reg"); 3666 unsigned RBegin, REnd; 3667 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd); 3668 ArgRegBegin = std::min(ArgRegBegin, RBegin); 3669 3670 CCInfo.nextInRegsParam(); 3671 } 3672 CCInfo.rewindByValRegsInfo(); 3673 3674 int lastInsIndex = -1; 3675 if (isVarArg && MFI.hasVAStart()) { 3676 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs); 3677 if (RegIdx != array_lengthof(GPRArgRegs)) 3678 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]); 3679 } 3680 3681 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin); 3682 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize); 3683 auto PtrVT = getPointerTy(DAG.getDataLayout()); 3684 3685 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3686 CCValAssign &VA = ArgLocs[i]; 3687 if (Ins[VA.getValNo()].isOrigArg()) { 3688 std::advance(CurOrigArg, 3689 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx); 3690 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex(); 3691 } 3692 // Arguments stored in registers. 3693 if (VA.isRegLoc()) { 3694 EVT RegVT = VA.getLocVT(); 3695 3696 if (VA.needsCustom()) { 3697 // f64 and vector types are split up into multiple registers or 3698 // combinations of registers and stack slots. 3699 if (VA.getLocVT() == MVT::v2f64) { 3700 SDValue ArgValue1 = GetF64FormalArgument(VA, ArgLocs[++i], 3701 Chain, DAG, dl); 3702 VA = ArgLocs[++i]; // skip ahead to next loc 3703 SDValue ArgValue2; 3704 if (VA.isMemLoc()) { 3705 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true); 3706 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3707 ArgValue2 = DAG.getLoad(MVT::f64, dl, Chain, FIN, 3708 MachinePointerInfo::getFixedStack( 3709 DAG.getMachineFunction(), FI)); 3710 } else { 3711 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], 3712 Chain, DAG, dl); 3713 } 3714 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64); 3715 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, 3716 ArgValue, ArgValue1, 3717 DAG.getIntPtrConstant(0, dl)); 3718 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, 3719 ArgValue, ArgValue2, 3720 DAG.getIntPtrConstant(1, dl)); 3721 } else 3722 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl); 3723 } else { 3724 const TargetRegisterClass *RC; 3725 3726 3727 if (RegVT == MVT::f16) 3728 RC = &ARM::HPRRegClass; 3729 else if (RegVT == MVT::f32) 3730 RC = &ARM::SPRRegClass; 3731 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16) 3732 RC = &ARM::DPRRegClass; 3733 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16) 3734 RC = &ARM::QPRRegClass; 3735 else if (RegVT == MVT::i32) 3736 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass 3737 : &ARM::GPRRegClass; 3738 else 3739 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering"); 3740 3741 // Transform the arguments in physical registers into virtual ones. 3742 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 3743 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); 3744 } 3745 3746 // If this is an 8 or 16-bit value, it is really passed promoted 3747 // to 32 bits. Insert an assert[sz]ext to capture this, then 3748 // truncate to the right size. 3749 switch (VA.getLocInfo()) { 3750 default: llvm_unreachable("Unknown loc info!"); 3751 case CCValAssign::Full: break; 3752 case CCValAssign::BCvt: 3753 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); 3754 break; 3755 case CCValAssign::SExt: 3756 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, 3757 DAG.getValueType(VA.getValVT())); 3758 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 3759 break; 3760 case CCValAssign::ZExt: 3761 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, 3762 DAG.getValueType(VA.getValVT())); 3763 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 3764 break; 3765 } 3766 3767 InVals.push_back(ArgValue); 3768 } else { // VA.isRegLoc() 3769 // sanity check 3770 assert(VA.isMemLoc()); 3771 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered"); 3772 3773 int index = VA.getValNo(); 3774 3775 // Some Ins[] entries become multiple ArgLoc[] entries. 3776 // Process them only once. 3777 if (index != lastInsIndex) 3778 { 3779 ISD::ArgFlagsTy Flags = Ins[index].Flags; 3780 // FIXME: For now, all byval parameter objects are marked mutable. 3781 // This can be changed with more analysis. 3782 // In case of tail call optimization mark all arguments mutable. 3783 // Since they could be overwritten by lowering of arguments in case of 3784 // a tail call. 3785 if (Flags.isByVal()) { 3786 assert(Ins[index].isOrigArg() && 3787 "Byval arguments cannot be implicit"); 3788 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed(); 3789 3790 int FrameIndex = StoreByValRegs( 3791 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex, 3792 VA.getLocMemOffset(), Flags.getByValSize()); 3793 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT)); 3794 CCInfo.nextInRegsParam(); 3795 } else { 3796 unsigned FIOffset = VA.getLocMemOffset(); 3797 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8, 3798 FIOffset, true); 3799 3800 // Create load nodes to retrieve arguments from the stack. 3801 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3802 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN, 3803 MachinePointerInfo::getFixedStack( 3804 DAG.getMachineFunction(), FI))); 3805 } 3806 lastInsIndex = index; 3807 } 3808 } 3809 } 3810 3811 // varargs 3812 if (isVarArg && MFI.hasVAStart()) 3813 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, 3814 CCInfo.getNextStackOffset(), 3815 TotalArgRegsSaveSize); 3816 3817 AFI->setArgumentStackSize(CCInfo.getNextStackOffset()); 3818 3819 return Chain; 3820 } 3821 3822 /// isFloatingPointZero - Return true if this is +0.0. 3823 static bool isFloatingPointZero(SDValue Op) { 3824 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 3825 return CFP->getValueAPF().isPosZero(); 3826 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 3827 // Maybe this has already been legalized into the constant pool? 3828 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) { 3829 SDValue WrapperOp = Op.getOperand(1).getOperand(0); 3830 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp)) 3831 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 3832 return CFP->getValueAPF().isPosZero(); 3833 } 3834 } else if (Op->getOpcode() == ISD::BITCAST && 3835 Op->getValueType(0) == MVT::f64) { 3836 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64) 3837 // created by LowerConstantFP(). 3838 SDValue BitcastOp = Op->getOperand(0); 3839 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM && 3840 isNullConstant(BitcastOp->getOperand(0))) 3841 return true; 3842 } 3843 return false; 3844 } 3845 3846 /// Returns appropriate ARM CMP (cmp) and corresponding condition code for 3847 /// the given operands. 3848 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, 3849 SDValue &ARMcc, SelectionDAG &DAG, 3850 const SDLoc &dl) const { 3851 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) { 3852 unsigned C = RHSC->getZExtValue(); 3853 if (!isLegalICmpImmediate((int32_t)C)) { 3854 // Constant does not fit, try adjusting it by one. 3855 switch (CC) { 3856 default: break; 3857 case ISD::SETLT: 3858 case ISD::SETGE: 3859 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) { 3860 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; 3861 RHS = DAG.getConstant(C - 1, dl, MVT::i32); 3862 } 3863 break; 3864 case ISD::SETULT: 3865 case ISD::SETUGE: 3866 if (C != 0 && isLegalICmpImmediate(C-1)) { 3867 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; 3868 RHS = DAG.getConstant(C - 1, dl, MVT::i32); 3869 } 3870 break; 3871 case ISD::SETLE: 3872 case ISD::SETGT: 3873 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) { 3874 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; 3875 RHS = DAG.getConstant(C + 1, dl, MVT::i32); 3876 } 3877 break; 3878 case ISD::SETULE: 3879 case ISD::SETUGT: 3880 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) { 3881 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; 3882 RHS = DAG.getConstant(C + 1, dl, MVT::i32); 3883 } 3884 break; 3885 } 3886 } 3887 } else if ((ARM_AM::getShiftOpcForNode(LHS.getOpcode()) != ARM_AM::no_shift) && 3888 (ARM_AM::getShiftOpcForNode(RHS.getOpcode()) == ARM_AM::no_shift)) { 3889 // In ARM and Thumb-2, the compare instructions can shift their second 3890 // operand. 3891 CC = ISD::getSetCCSwappedOperands(CC); 3892 std::swap(LHS, RHS); 3893 } 3894 3895 ARMCC::CondCodes CondCode = IntCCToARMCC(CC); 3896 ARMISD::NodeType CompareType; 3897 switch (CondCode) { 3898 default: 3899 CompareType = ARMISD::CMP; 3900 break; 3901 case ARMCC::EQ: 3902 case ARMCC::NE: 3903 // Uses only Z Flag 3904 CompareType = ARMISD::CMPZ; 3905 break; 3906 } 3907 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 3908 return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS); 3909 } 3910 3911 /// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands. 3912 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS, 3913 SelectionDAG &DAG, const SDLoc &dl, 3914 bool InvalidOnQNaN) const { 3915 assert(!Subtarget->isFPOnlySP() || RHS.getValueType() != MVT::f64); 3916 SDValue Cmp; 3917 SDValue C = DAG.getConstant(InvalidOnQNaN, dl, MVT::i32); 3918 if (!isFloatingPointZero(RHS)) 3919 Cmp = DAG.getNode(ARMISD::CMPFP, dl, MVT::Glue, LHS, RHS, C); 3920 else 3921 Cmp = DAG.getNode(ARMISD::CMPFPw0, dl, MVT::Glue, LHS, C); 3922 return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Cmp); 3923 } 3924 3925 /// duplicateCmp - Glue values can have only one use, so this function 3926 /// duplicates a comparison node. 3927 SDValue 3928 ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const { 3929 unsigned Opc = Cmp.getOpcode(); 3930 SDLoc DL(Cmp); 3931 if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ) 3932 return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1)); 3933 3934 assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation"); 3935 Cmp = Cmp.getOperand(0); 3936 Opc = Cmp.getOpcode(); 3937 if (Opc == ARMISD::CMPFP) 3938 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0), 3939 Cmp.getOperand(1), Cmp.getOperand(2)); 3940 else { 3941 assert(Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT"); 3942 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0), 3943 Cmp.getOperand(1)); 3944 } 3945 return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Cmp); 3946 } 3947 3948 // This function returns three things: the arithmetic computation itself 3949 // (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The 3950 // comparison and the condition code define the case in which the arithmetic 3951 // computation *does not* overflow. 3952 std::pair<SDValue, SDValue> 3953 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG, 3954 SDValue &ARMcc) const { 3955 assert(Op.getValueType() == MVT::i32 && "Unsupported value type"); 3956 3957 SDValue Value, OverflowCmp; 3958 SDValue LHS = Op.getOperand(0); 3959 SDValue RHS = Op.getOperand(1); 3960 SDLoc dl(Op); 3961 3962 // FIXME: We are currently always generating CMPs because we don't support 3963 // generating CMN through the backend. This is not as good as the natural 3964 // CMP case because it causes a register dependency and cannot be folded 3965 // later. 3966 3967 switch (Op.getOpcode()) { 3968 default: 3969 llvm_unreachable("Unknown overflow instruction!"); 3970 case ISD::SADDO: 3971 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32); 3972 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS); 3973 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS); 3974 break; 3975 case ISD::UADDO: 3976 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32); 3977 // We use ADDC here to correspond to its use in LowerUnsignedALUO. 3978 // We do not use it in the USUBO case as Value may not be used. 3979 Value = DAG.getNode(ARMISD::ADDC, dl, 3980 DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS) 3981 .getValue(0); 3982 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS); 3983 break; 3984 case ISD::SSUBO: 3985 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32); 3986 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS); 3987 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS); 3988 break; 3989 case ISD::USUBO: 3990 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32); 3991 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS); 3992 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS); 3993 break; 3994 case ISD::UMULO: 3995 // We generate a UMUL_LOHI and then check if the high word is 0. 3996 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32); 3997 Value = DAG.getNode(ISD::UMUL_LOHI, dl, 3998 DAG.getVTList(Op.getValueType(), Op.getValueType()), 3999 LHS, RHS); 4000 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1), 4001 DAG.getConstant(0, dl, MVT::i32)); 4002 Value = Value.getValue(0); // We only want the low 32 bits for the result. 4003 break; 4004 case ISD::SMULO: 4005 // We generate a SMUL_LOHI and then check if all the bits of the high word 4006 // are the same as the sign bit of the low word. 4007 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32); 4008 Value = DAG.getNode(ISD::SMUL_LOHI, dl, 4009 DAG.getVTList(Op.getValueType(), Op.getValueType()), 4010 LHS, RHS); 4011 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1), 4012 DAG.getNode(ISD::SRA, dl, Op.getValueType(), 4013 Value.getValue(0), 4014 DAG.getConstant(31, dl, MVT::i32))); 4015 Value = Value.getValue(0); // We only want the low 32 bits for the result. 4016 break; 4017 } // switch (...) 4018 4019 return std::make_pair(Value, OverflowCmp); 4020 } 4021 4022 SDValue 4023 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const { 4024 // Let legalize expand this if it isn't a legal type yet. 4025 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType())) 4026 return SDValue(); 4027 4028 SDValue Value, OverflowCmp; 4029 SDValue ARMcc; 4030 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc); 4031 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4032 SDLoc dl(Op); 4033 // We use 0 and 1 as false and true values. 4034 SDValue TVal = DAG.getConstant(1, dl, MVT::i32); 4035 SDValue FVal = DAG.getConstant(0, dl, MVT::i32); 4036 EVT VT = Op.getValueType(); 4037 4038 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal, 4039 ARMcc, CCR, OverflowCmp); 4040 4041 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); 4042 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); 4043 } 4044 4045 static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, 4046 SelectionDAG &DAG) { 4047 SDLoc DL(BoolCarry); 4048 EVT CarryVT = BoolCarry.getValueType(); 4049 4050 // This converts the boolean value carry into the carry flag by doing 4051 // ARMISD::SUBC Carry, 1 4052 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL, 4053 DAG.getVTList(CarryVT, MVT::i32), 4054 BoolCarry, DAG.getConstant(1, DL, CarryVT)); 4055 return Carry.getValue(1); 4056 } 4057 4058 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, 4059 SelectionDAG &DAG) { 4060 SDLoc DL(Flags); 4061 4062 // Now convert the carry flag into a boolean carry. We do this 4063 // using ARMISD:ADDE 0, 0, Carry 4064 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32), 4065 DAG.getConstant(0, DL, MVT::i32), 4066 DAG.getConstant(0, DL, MVT::i32), Flags); 4067 } 4068 4069 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op, 4070 SelectionDAG &DAG) const { 4071 // Let legalize expand this if it isn't a legal type yet. 4072 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType())) 4073 return SDValue(); 4074 4075 SDValue LHS = Op.getOperand(0); 4076 SDValue RHS = Op.getOperand(1); 4077 SDLoc dl(Op); 4078 4079 EVT VT = Op.getValueType(); 4080 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 4081 SDValue Value; 4082 SDValue Overflow; 4083 switch (Op.getOpcode()) { 4084 default: 4085 llvm_unreachable("Unknown overflow instruction!"); 4086 case ISD::UADDO: 4087 Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS); 4088 // Convert the carry flag into a boolean value. 4089 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); 4090 break; 4091 case ISD::USUBO: { 4092 Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS); 4093 // Convert the carry flag into a boolean value. 4094 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); 4095 // ARMISD::SUBC returns 0 when we have to borrow, so make it an overflow 4096 // value. So compute 1 - C. 4097 Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32, 4098 DAG.getConstant(1, dl, MVT::i32), Overflow); 4099 break; 4100 } 4101 } 4102 4103 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); 4104 } 4105 4106 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { 4107 SDValue Cond = Op.getOperand(0); 4108 SDValue SelectTrue = Op.getOperand(1); 4109 SDValue SelectFalse = Op.getOperand(2); 4110 SDLoc dl(Op); 4111 unsigned Opc = Cond.getOpcode(); 4112 4113 if (Cond.getResNo() == 1 && 4114 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 4115 Opc == ISD::USUBO)) { 4116 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) 4117 return SDValue(); 4118 4119 SDValue Value, OverflowCmp; 4120 SDValue ARMcc; 4121 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc); 4122 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4123 EVT VT = Op.getValueType(); 4124 4125 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR, 4126 OverflowCmp, DAG); 4127 } 4128 4129 // Convert: 4130 // 4131 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond) 4132 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond) 4133 // 4134 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) { 4135 const ConstantSDNode *CMOVTrue = 4136 dyn_cast<ConstantSDNode>(Cond.getOperand(0)); 4137 const ConstantSDNode *CMOVFalse = 4138 dyn_cast<ConstantSDNode>(Cond.getOperand(1)); 4139 4140 if (CMOVTrue && CMOVFalse) { 4141 unsigned CMOVTrueVal = CMOVTrue->getZExtValue(); 4142 unsigned CMOVFalseVal = CMOVFalse->getZExtValue(); 4143 4144 SDValue True; 4145 SDValue False; 4146 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) { 4147 True = SelectTrue; 4148 False = SelectFalse; 4149 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) { 4150 True = SelectFalse; 4151 False = SelectTrue; 4152 } 4153 4154 if (True.getNode() && False.getNode()) { 4155 EVT VT = Op.getValueType(); 4156 SDValue ARMcc = Cond.getOperand(2); 4157 SDValue CCR = Cond.getOperand(3); 4158 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG); 4159 assert(True.getValueType() == VT); 4160 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG); 4161 } 4162 } 4163 } 4164 4165 // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the 4166 // undefined bits before doing a full-word comparison with zero. 4167 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond, 4168 DAG.getConstant(1, dl, Cond.getValueType())); 4169 4170 return DAG.getSelectCC(dl, Cond, 4171 DAG.getConstant(0, dl, Cond.getValueType()), 4172 SelectTrue, SelectFalse, ISD::SETNE); 4173 } 4174 4175 static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, 4176 bool &swpCmpOps, bool &swpVselOps) { 4177 // Start by selecting the GE condition code for opcodes that return true for 4178 // 'equality' 4179 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE || 4180 CC == ISD::SETULE) 4181 CondCode = ARMCC::GE; 4182 4183 // and GT for opcodes that return false for 'equality'. 4184 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT || 4185 CC == ISD::SETULT) 4186 CondCode = ARMCC::GT; 4187 4188 // Since we are constrained to GE/GT, if the opcode contains 'less', we need 4189 // to swap the compare operands. 4190 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT || 4191 CC == ISD::SETULT) 4192 swpCmpOps = true; 4193 4194 // Both GT and GE are ordered comparisons, and return false for 'unordered'. 4195 // If we have an unordered opcode, we need to swap the operands to the VSEL 4196 // instruction (effectively negating the condition). 4197 // 4198 // This also has the effect of swapping which one of 'less' or 'greater' 4199 // returns true, so we also swap the compare operands. It also switches 4200 // whether we return true for 'equality', so we compensate by picking the 4201 // opposite condition code to our original choice. 4202 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE || 4203 CC == ISD::SETUGT) { 4204 swpCmpOps = !swpCmpOps; 4205 swpVselOps = !swpVselOps; 4206 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT; 4207 } 4208 4209 // 'ordered' is 'anything but unordered', so use the VS condition code and 4210 // swap the VSEL operands. 4211 if (CC == ISD::SETO) { 4212 CondCode = ARMCC::VS; 4213 swpVselOps = true; 4214 } 4215 4216 // 'unordered or not equal' is 'anything but equal', so use the EQ condition 4217 // code and swap the VSEL operands. 4218 if (CC == ISD::SETUNE) { 4219 CondCode = ARMCC::EQ; 4220 swpVselOps = true; 4221 } 4222 } 4223 4224 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, 4225 SDValue TrueVal, SDValue ARMcc, SDValue CCR, 4226 SDValue Cmp, SelectionDAG &DAG) const { 4227 if (Subtarget->isFPOnlySP() && VT == MVT::f64) { 4228 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl, 4229 DAG.getVTList(MVT::i32, MVT::i32), FalseVal); 4230 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl, 4231 DAG.getVTList(MVT::i32, MVT::i32), TrueVal); 4232 4233 SDValue TrueLow = TrueVal.getValue(0); 4234 SDValue TrueHigh = TrueVal.getValue(1); 4235 SDValue FalseLow = FalseVal.getValue(0); 4236 SDValue FalseHigh = FalseVal.getValue(1); 4237 4238 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow, 4239 ARMcc, CCR, Cmp); 4240 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh, 4241 ARMcc, CCR, duplicateCmp(Cmp, DAG)); 4242 4243 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High); 4244 } else { 4245 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR, 4246 Cmp); 4247 } 4248 } 4249 4250 static bool isGTorGE(ISD::CondCode CC) { 4251 return CC == ISD::SETGT || CC == ISD::SETGE; 4252 } 4253 4254 static bool isLTorLE(ISD::CondCode CC) { 4255 return CC == ISD::SETLT || CC == ISD::SETLE; 4256 } 4257 4258 // See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating. 4259 // All of these conditions (and their <= and >= counterparts) will do: 4260 // x < k ? k : x 4261 // x > k ? x : k 4262 // k < x ? x : k 4263 // k > x ? k : x 4264 static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, 4265 const SDValue TrueVal, const SDValue FalseVal, 4266 const ISD::CondCode CC, const SDValue K) { 4267 return (isGTorGE(CC) && 4268 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) || 4269 (isLTorLE(CC) && 4270 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))); 4271 } 4272 4273 // Similar to isLowerSaturate(), but checks for upper-saturating conditions. 4274 static bool isUpperSaturate(const SDValue LHS, const SDValue RHS, 4275 const SDValue TrueVal, const SDValue FalseVal, 4276 const ISD::CondCode CC, const SDValue K) { 4277 return (isGTorGE(CC) && 4278 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))) || 4279 (isLTorLE(CC) && 4280 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))); 4281 } 4282 4283 // Check if two chained conditionals could be converted into SSAT or USAT. 4284 // 4285 // SSAT can replace a set of two conditional selectors that bound a number to an 4286 // interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples: 4287 // 4288 // x < -k ? -k : (x > k ? k : x) 4289 // x < -k ? -k : (x < k ? x : k) 4290 // x > -k ? (x > k ? k : x) : -k 4291 // x < k ? (x < -k ? -k : x) : k 4292 // etc. 4293 // 4294 // USAT works similarily to SSAT but bounds on the interval [0, k] where k + 1 is 4295 // a power of 2. 4296 // 4297 // It returns true if the conversion can be done, false otherwise. 4298 // Additionally, the variable is returned in parameter V, the constant in K and 4299 // usat is set to true if the conditional represents an unsigned saturation 4300 static bool isSaturatingConditional(const SDValue &Op, SDValue &V, 4301 uint64_t &K, bool &usat) { 4302 SDValue LHS1 = Op.getOperand(0); 4303 SDValue RHS1 = Op.getOperand(1); 4304 SDValue TrueVal1 = Op.getOperand(2); 4305 SDValue FalseVal1 = Op.getOperand(3); 4306 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4307 4308 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1; 4309 if (Op2.getOpcode() != ISD::SELECT_CC) 4310 return false; 4311 4312 SDValue LHS2 = Op2.getOperand(0); 4313 SDValue RHS2 = Op2.getOperand(1); 4314 SDValue TrueVal2 = Op2.getOperand(2); 4315 SDValue FalseVal2 = Op2.getOperand(3); 4316 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get(); 4317 4318 // Find out which are the constants and which are the variables 4319 // in each conditional 4320 SDValue *K1 = isa<ConstantSDNode>(LHS1) ? &LHS1 : isa<ConstantSDNode>(RHS1) 4321 ? &RHS1 4322 : nullptr; 4323 SDValue *K2 = isa<ConstantSDNode>(LHS2) ? &LHS2 : isa<ConstantSDNode>(RHS2) 4324 ? &RHS2 4325 : nullptr; 4326 SDValue K2Tmp = isa<ConstantSDNode>(TrueVal2) ? TrueVal2 : FalseVal2; 4327 SDValue V1Tmp = (K1 && *K1 == LHS1) ? RHS1 : LHS1; 4328 SDValue V2Tmp = (K2 && *K2 == LHS2) ? RHS2 : LHS2; 4329 SDValue V2 = (K2Tmp == TrueVal2) ? FalseVal2 : TrueVal2; 4330 4331 // We must detect cases where the original operations worked with 16- or 4332 // 8-bit values. In such case, V2Tmp != V2 because the comparison operations 4333 // must work with sign-extended values but the select operations return 4334 // the original non-extended value. 4335 SDValue V2TmpReg = V2Tmp; 4336 if (V2Tmp->getOpcode() == ISD::SIGN_EXTEND_INREG) 4337 V2TmpReg = V2Tmp->getOperand(0); 4338 4339 // Check that the registers and the constants have the correct values 4340 // in both conditionals 4341 if (!K1 || !K2 || *K1 == Op2 || *K2 != K2Tmp || V1Tmp != V2Tmp || 4342 V2TmpReg != V2) 4343 return false; 4344 4345 // Figure out which conditional is saturating the lower/upper bound. 4346 const SDValue *LowerCheckOp = 4347 isLowerSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1) 4348 ? &Op 4349 : isLowerSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2) 4350 ? &Op2 4351 : nullptr; 4352 const SDValue *UpperCheckOp = 4353 isUpperSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1) 4354 ? &Op 4355 : isUpperSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2) 4356 ? &Op2 4357 : nullptr; 4358 4359 if (!UpperCheckOp || !LowerCheckOp || LowerCheckOp == UpperCheckOp) 4360 return false; 4361 4362 // Check that the constant in the lower-bound check is 4363 // the opposite of the constant in the upper-bound check 4364 // in 1's complement. 4365 int64_t Val1 = cast<ConstantSDNode>(*K1)->getSExtValue(); 4366 int64_t Val2 = cast<ConstantSDNode>(*K2)->getSExtValue(); 4367 int64_t PosVal = std::max(Val1, Val2); 4368 int64_t NegVal = std::min(Val1, Val2); 4369 4370 if (((Val1 > Val2 && UpperCheckOp == &Op) || 4371 (Val1 < Val2 && UpperCheckOp == &Op2)) && 4372 isPowerOf2_64(PosVal + 1)) { 4373 4374 // Handle the difference between USAT (unsigned) and SSAT (signed) saturation 4375 if (Val1 == ~Val2) 4376 usat = false; 4377 else if (NegVal == 0) 4378 usat = true; 4379 else 4380 return false; 4381 4382 V = V2; 4383 K = (uint64_t)PosVal; // At this point, PosVal is guaranteed to be positive 4384 4385 return true; 4386 } 4387 4388 return false; 4389 } 4390 4391 // Check if a condition of the type x < k ? k : x can be converted into a 4392 // bit operation instead of conditional moves. 4393 // Currently this is allowed given: 4394 // - The conditions and values match up 4395 // - k is 0 or -1 (all ones) 4396 // This function will not check the last condition, thats up to the caller 4397 // It returns true if the transformation can be made, and in such case 4398 // returns x in V, and k in SatK. 4399 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, 4400 SDValue &SatK) 4401 { 4402 SDValue LHS = Op.getOperand(0); 4403 SDValue RHS = Op.getOperand(1); 4404 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4405 SDValue TrueVal = Op.getOperand(2); 4406 SDValue FalseVal = Op.getOperand(3); 4407 4408 SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS) 4409 ? &RHS 4410 : nullptr; 4411 4412 // No constant operation in comparison, early out 4413 if (!K) 4414 return false; 4415 4416 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal; 4417 V = (KTmp == TrueVal) ? FalseVal : TrueVal; 4418 SDValue VTmp = (K && *K == LHS) ? RHS : LHS; 4419 4420 // If the constant on left and right side, or variable on left and right, 4421 // does not match, early out 4422 if (*K != KTmp || V != VTmp) 4423 return false; 4424 4425 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, *K)) { 4426 SatK = *K; 4427 return true; 4428 } 4429 4430 return false; 4431 } 4432 4433 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 4434 EVT VT = Op.getValueType(); 4435 SDLoc dl(Op); 4436 4437 // Try to convert two saturating conditional selects into a single SSAT 4438 SDValue SatValue; 4439 uint64_t SatConstant; 4440 bool SatUSat; 4441 if (((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2()) && 4442 isSaturatingConditional(Op, SatValue, SatConstant, SatUSat)) { 4443 if (SatUSat) 4444 return DAG.getNode(ARMISD::USAT, dl, VT, SatValue, 4445 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT)); 4446 else 4447 return DAG.getNode(ARMISD::SSAT, dl, VT, SatValue, 4448 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT)); 4449 } 4450 4451 // Try to convert expressions of the form x < k ? k : x (and similar forms) 4452 // into more efficient bit operations, which is possible when k is 0 or -1 4453 // On ARM and Thumb-2 which have flexible operand 2 this will result in 4454 // single instructions. On Thumb the shift and the bit operation will be two 4455 // instructions. 4456 // Only allow this transformation on full-width (32-bit) operations 4457 SDValue LowerSatConstant; 4458 if (VT == MVT::i32 && 4459 isLowerSaturatingConditional(Op, SatValue, LowerSatConstant)) { 4460 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue, 4461 DAG.getConstant(31, dl, VT)); 4462 if (isNullConstant(LowerSatConstant)) { 4463 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV, 4464 DAG.getAllOnesConstant(dl, VT)); 4465 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV); 4466 } else if (isAllOnesConstant(LowerSatConstant)) 4467 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV); 4468 } 4469 4470 SDValue LHS = Op.getOperand(0); 4471 SDValue RHS = Op.getOperand(1); 4472 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4473 SDValue TrueVal = Op.getOperand(2); 4474 SDValue FalseVal = Op.getOperand(3); 4475 4476 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) { 4477 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC, 4478 dl); 4479 4480 // If softenSetCCOperands only returned one value, we should compare it to 4481 // zero. 4482 if (!RHS.getNode()) { 4483 RHS = DAG.getConstant(0, dl, LHS.getValueType()); 4484 CC = ISD::SETNE; 4485 } 4486 } 4487 4488 if (LHS.getValueType() == MVT::i32) { 4489 // Try to generate VSEL on ARMv8. 4490 // The VSEL instruction can't use all the usual ARM condition 4491 // codes: it only has two bits to select the condition code, so it's 4492 // constrained to use only GE, GT, VS and EQ. 4493 // 4494 // To implement all the various ISD::SETXXX opcodes, we sometimes need to 4495 // swap the operands of the previous compare instruction (effectively 4496 // inverting the compare condition, swapping 'less' and 'greater') and 4497 // sometimes need to swap the operands to the VSEL (which inverts the 4498 // condition in the sense of firing whenever the previous condition didn't) 4499 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 || 4500 TrueVal.getValueType() == MVT::f64)) { 4501 ARMCC::CondCodes CondCode = IntCCToARMCC(CC); 4502 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE || 4503 CondCode == ARMCC::VC || CondCode == ARMCC::NE) { 4504 CC = ISD::getSetCCInverse(CC, true); 4505 std::swap(TrueVal, FalseVal); 4506 } 4507 } 4508 4509 SDValue ARMcc; 4510 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4511 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl); 4512 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); 4513 } 4514 4515 ARMCC::CondCodes CondCode, CondCode2; 4516 bool InvalidOnQNaN; 4517 FPCCToARMCC(CC, CondCode, CondCode2, InvalidOnQNaN); 4518 4519 // Normalize the fp compare. If RHS is zero we keep it there so we match 4520 // CMPFPw0 instead of CMPFP. 4521 if (Subtarget->hasFPARMv8() && !isFloatingPointZero(RHS) && 4522 (TrueVal.getValueType() == MVT::f16 || 4523 TrueVal.getValueType() == MVT::f32 || 4524 TrueVal.getValueType() == MVT::f64)) { 4525 bool swpCmpOps = false; 4526 bool swpVselOps = false; 4527 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps); 4528 4529 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE || 4530 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) { 4531 if (swpCmpOps) 4532 std::swap(LHS, RHS); 4533 if (swpVselOps) 4534 std::swap(TrueVal, FalseVal); 4535 } 4536 } 4537 4538 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 4539 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, InvalidOnQNaN); 4540 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4541 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); 4542 if (CondCode2 != ARMCC::AL) { 4543 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32); 4544 // FIXME: Needs another CMP because flag can have but one use. 4545 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl, InvalidOnQNaN); 4546 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG); 4547 } 4548 return Result; 4549 } 4550 4551 /// canChangeToInt - Given the fp compare operand, return true if it is suitable 4552 /// to morph to an integer compare sequence. 4553 static bool canChangeToInt(SDValue Op, bool &SeenZero, 4554 const ARMSubtarget *Subtarget) { 4555 SDNode *N = Op.getNode(); 4556 if (!N->hasOneUse()) 4557 // Otherwise it requires moving the value from fp to integer registers. 4558 return false; 4559 if (!N->getNumValues()) 4560 return false; 4561 EVT VT = Op.getValueType(); 4562 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow()) 4563 // f32 case is generally profitable. f64 case only makes sense when vcmpe + 4564 // vmrs are very slow, e.g. cortex-a8. 4565 return false; 4566 4567 if (isFloatingPointZero(Op)) { 4568 SeenZero = true; 4569 return true; 4570 } 4571 return ISD::isNormalLoad(N); 4572 } 4573 4574 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) { 4575 if (isFloatingPointZero(Op)) 4576 return DAG.getConstant(0, SDLoc(Op), MVT::i32); 4577 4578 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) 4579 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(), 4580 Ld->getPointerInfo(), Ld->getAlignment(), 4581 Ld->getMemOperand()->getFlags()); 4582 4583 llvm_unreachable("Unknown VFP cmp argument!"); 4584 } 4585 4586 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, 4587 SDValue &RetVal1, SDValue &RetVal2) { 4588 SDLoc dl(Op); 4589 4590 if (isFloatingPointZero(Op)) { 4591 RetVal1 = DAG.getConstant(0, dl, MVT::i32); 4592 RetVal2 = DAG.getConstant(0, dl, MVT::i32); 4593 return; 4594 } 4595 4596 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) { 4597 SDValue Ptr = Ld->getBasePtr(); 4598 RetVal1 = 4599 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(), 4600 Ld->getAlignment(), Ld->getMemOperand()->getFlags()); 4601 4602 EVT PtrType = Ptr.getValueType(); 4603 unsigned NewAlign = MinAlign(Ld->getAlignment(), 4); 4604 SDValue NewPtr = DAG.getNode(ISD::ADD, dl, 4605 PtrType, Ptr, DAG.getConstant(4, dl, PtrType)); 4606 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr, 4607 Ld->getPointerInfo().getWithOffset(4), NewAlign, 4608 Ld->getMemOperand()->getFlags()); 4609 return; 4610 } 4611 4612 llvm_unreachable("Unknown VFP cmp argument!"); 4613 } 4614 4615 /// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some 4616 /// f32 and even f64 comparisons to integer ones. 4617 SDValue 4618 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const { 4619 SDValue Chain = Op.getOperand(0); 4620 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 4621 SDValue LHS = Op.getOperand(2); 4622 SDValue RHS = Op.getOperand(3); 4623 SDValue Dest = Op.getOperand(4); 4624 SDLoc dl(Op); 4625 4626 bool LHSSeenZero = false; 4627 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget); 4628 bool RHSSeenZero = false; 4629 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget); 4630 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) { 4631 // If unsafe fp math optimization is enabled and there are no other uses of 4632 // the CMP operands, and the condition code is EQ or NE, we can optimize it 4633 // to an integer comparison. 4634 if (CC == ISD::SETOEQ) 4635 CC = ISD::SETEQ; 4636 else if (CC == ISD::SETUNE) 4637 CC = ISD::SETNE; 4638 4639 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32); 4640 SDValue ARMcc; 4641 if (LHS.getValueType() == MVT::f32) { 4642 LHS = DAG.getNode(ISD::AND, dl, MVT::i32, 4643 bitcastf32Toi32(LHS, DAG), Mask); 4644 RHS = DAG.getNode(ISD::AND, dl, MVT::i32, 4645 bitcastf32Toi32(RHS, DAG), Mask); 4646 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl); 4647 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4648 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, 4649 Chain, Dest, ARMcc, CCR, Cmp); 4650 } 4651 4652 SDValue LHS1, LHS2; 4653 SDValue RHS1, RHS2; 4654 expandf64Toi32(LHS, DAG, LHS1, LHS2); 4655 expandf64Toi32(RHS, DAG, RHS1, RHS2); 4656 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask); 4657 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask); 4658 ARMCC::CondCodes CondCode = IntCCToARMCC(CC); 4659 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 4660 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue); 4661 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest }; 4662 return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops); 4663 } 4664 4665 return SDValue(); 4666 } 4667 4668 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 4669 SDValue Chain = Op.getOperand(0); 4670 SDValue Cond = Op.getOperand(1); 4671 SDValue Dest = Op.getOperand(2); 4672 SDLoc dl(Op); 4673 4674 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch 4675 // instruction. 4676 unsigned Opc = Cond.getOpcode(); 4677 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) && 4678 !Subtarget->isThumb1Only(); 4679 if (Cond.getResNo() == 1 && 4680 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 4681 Opc == ISD::USUBO || OptimizeMul)) { 4682 // Only lower legal XALUO ops. 4683 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) 4684 return SDValue(); 4685 4686 // The actual operation with overflow check. 4687 SDValue Value, OverflowCmp; 4688 SDValue ARMcc; 4689 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc); 4690 4691 // Reverse the condition code. 4692 ARMCC::CondCodes CondCode = 4693 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue(); 4694 CondCode = ARMCC::getOppositeCondition(CondCode); 4695 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32); 4696 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4697 4698 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR, 4699 OverflowCmp); 4700 } 4701 4702 return SDValue(); 4703 } 4704 4705 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 4706 SDValue Chain = Op.getOperand(0); 4707 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 4708 SDValue LHS = Op.getOperand(2); 4709 SDValue RHS = Op.getOperand(3); 4710 SDValue Dest = Op.getOperand(4); 4711 SDLoc dl(Op); 4712 4713 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) { 4714 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC, 4715 dl); 4716 4717 // If softenSetCCOperands only returned one value, we should compare it to 4718 // zero. 4719 if (!RHS.getNode()) { 4720 RHS = DAG.getConstant(0, dl, LHS.getValueType()); 4721 CC = ISD::SETNE; 4722 } 4723 } 4724 4725 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch 4726 // instruction. 4727 unsigned Opc = LHS.getOpcode(); 4728 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) && 4729 !Subtarget->isThumb1Only(); 4730 if (LHS.getResNo() == 1 && (isOneConstant(RHS) || isNullConstant(RHS)) && 4731 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 4732 Opc == ISD::USUBO || OptimizeMul) && 4733 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 4734 // Only lower legal XALUO ops. 4735 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0))) 4736 return SDValue(); 4737 4738 // The actual operation with overflow check. 4739 SDValue Value, OverflowCmp; 4740 SDValue ARMcc; 4741 std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc); 4742 4743 if ((CC == ISD::SETNE) != isOneConstant(RHS)) { 4744 // Reverse the condition code. 4745 ARMCC::CondCodes CondCode = 4746 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue(); 4747 CondCode = ARMCC::getOppositeCondition(CondCode); 4748 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32); 4749 } 4750 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4751 4752 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR, 4753 OverflowCmp); 4754 } 4755 4756 if (LHS.getValueType() == MVT::i32) { 4757 SDValue ARMcc; 4758 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl); 4759 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4760 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, 4761 Chain, Dest, ARMcc, CCR, Cmp); 4762 } 4763 4764 if (getTargetMachine().Options.UnsafeFPMath && 4765 (CC == ISD::SETEQ || CC == ISD::SETOEQ || 4766 CC == ISD::SETNE || CC == ISD::SETUNE)) { 4767 if (SDValue Result = OptimizeVFPBrcond(Op, DAG)) 4768 return Result; 4769 } 4770 4771 ARMCC::CondCodes CondCode, CondCode2; 4772 bool InvalidOnQNaN; 4773 FPCCToARMCC(CC, CondCode, CondCode2, InvalidOnQNaN); 4774 4775 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32); 4776 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, InvalidOnQNaN); 4777 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 4778 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue); 4779 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp }; 4780 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops); 4781 if (CondCode2 != ARMCC::AL) { 4782 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32); 4783 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) }; 4784 Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops); 4785 } 4786 return Res; 4787 } 4788 4789 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const { 4790 SDValue Chain = Op.getOperand(0); 4791 SDValue Table = Op.getOperand(1); 4792 SDValue Index = Op.getOperand(2); 4793 SDLoc dl(Op); 4794 4795 EVT PTy = getPointerTy(DAG.getDataLayout()); 4796 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table); 4797 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy); 4798 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI); 4799 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy)); 4800 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index); 4801 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) { 4802 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table 4803 // which does another jump to the destination. This also makes it easier 4804 // to translate it to TBB / TBH later (Thumb2 only). 4805 // FIXME: This might not work if the function is extremely large. 4806 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain, 4807 Addr, Op.getOperand(2), JTI); 4808 } 4809 if (isPositionIndependent() || Subtarget->isROPI()) { 4810 Addr = 4811 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr, 4812 MachinePointerInfo::getJumpTable(DAG.getMachineFunction())); 4813 Chain = Addr.getValue(1); 4814 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr); 4815 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI); 4816 } else { 4817 Addr = 4818 DAG.getLoad(PTy, dl, Chain, Addr, 4819 MachinePointerInfo::getJumpTable(DAG.getMachineFunction())); 4820 Chain = Addr.getValue(1); 4821 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI); 4822 } 4823 } 4824 4825 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) { 4826 EVT VT = Op.getValueType(); 4827 SDLoc dl(Op); 4828 4829 if (Op.getValueType().getVectorElementType() == MVT::i32) { 4830 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32) 4831 return Op; 4832 return DAG.UnrollVectorOp(Op.getNode()); 4833 } 4834 4835 assert(Op.getOperand(0).getValueType() == MVT::v4f32 && 4836 "Invalid type for custom lowering!"); 4837 if (VT != MVT::v4i16) 4838 return DAG.UnrollVectorOp(Op.getNode()); 4839 4840 Op = DAG.getNode(Op.getOpcode(), dl, MVT::v4i32, Op.getOperand(0)); 4841 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op); 4842 } 4843 4844 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const { 4845 EVT VT = Op.getValueType(); 4846 if (VT.isVector()) 4847 return LowerVectorFP_TO_INT(Op, DAG); 4848 if (Subtarget->isFPOnlySP() && Op.getOperand(0).getValueType() == MVT::f64) { 4849 RTLIB::Libcall LC; 4850 if (Op.getOpcode() == ISD::FP_TO_SINT) 4851 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), 4852 Op.getValueType()); 4853 else 4854 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), 4855 Op.getValueType()); 4856 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0), 4857 /*isSigned*/ false, SDLoc(Op)).first; 4858 } 4859 4860 return Op; 4861 } 4862 4863 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 4864 EVT VT = Op.getValueType(); 4865 SDLoc dl(Op); 4866 4867 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) { 4868 if (VT.getVectorElementType() == MVT::f32) 4869 return Op; 4870 return DAG.UnrollVectorOp(Op.getNode()); 4871 } 4872 4873 assert(Op.getOperand(0).getValueType() == MVT::v4i16 && 4874 "Invalid type for custom lowering!"); 4875 if (VT != MVT::v4f32) 4876 return DAG.UnrollVectorOp(Op.getNode()); 4877 4878 unsigned CastOpc; 4879 unsigned Opc; 4880 switch (Op.getOpcode()) { 4881 default: llvm_unreachable("Invalid opcode!"); 4882 case ISD::SINT_TO_FP: 4883 CastOpc = ISD::SIGN_EXTEND; 4884 Opc = ISD::SINT_TO_FP; 4885 break; 4886 case ISD::UINT_TO_FP: 4887 CastOpc = ISD::ZERO_EXTEND; 4888 Opc = ISD::UINT_TO_FP; 4889 break; 4890 } 4891 4892 Op = DAG.getNode(CastOpc, dl, MVT::v4i32, Op.getOperand(0)); 4893 return DAG.getNode(Opc, dl, VT, Op); 4894 } 4895 4896 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { 4897 EVT VT = Op.getValueType(); 4898 if (VT.isVector()) 4899 return LowerVectorINT_TO_FP(Op, DAG); 4900 if (Subtarget->isFPOnlySP() && Op.getValueType() == MVT::f64) { 4901 RTLIB::Libcall LC; 4902 if (Op.getOpcode() == ISD::SINT_TO_FP) 4903 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), 4904 Op.getValueType()); 4905 else 4906 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), 4907 Op.getValueType()); 4908 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0), 4909 /*isSigned*/ false, SDLoc(Op)).first; 4910 } 4911 4912 return Op; 4913 } 4914 4915 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { 4916 // Implement fcopysign with a fabs and a conditional fneg. 4917 SDValue Tmp0 = Op.getOperand(0); 4918 SDValue Tmp1 = Op.getOperand(1); 4919 SDLoc dl(Op); 4920 EVT VT = Op.getValueType(); 4921 EVT SrcVT = Tmp1.getValueType(); 4922 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST || 4923 Tmp0.getOpcode() == ARMISD::VMOVDRR; 4924 bool UseNEON = !InGPR && Subtarget->hasNEON(); 4925 4926 if (UseNEON) { 4927 // Use VBSL to copy the sign bit. 4928 unsigned EncodedVal = ARM_AM::createNEONModImm(0x6, 0x80); 4929 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32, 4930 DAG.getTargetConstant(EncodedVal, dl, MVT::i32)); 4931 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64; 4932 if (VT == MVT::f64) 4933 Mask = DAG.getNode(ARMISD::VSHL, dl, OpVT, 4934 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask), 4935 DAG.getConstant(32, dl, MVT::i32)); 4936 else /*if (VT == MVT::f32)*/ 4937 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0); 4938 if (SrcVT == MVT::f32) { 4939 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1); 4940 if (VT == MVT::f64) 4941 Tmp1 = DAG.getNode(ARMISD::VSHL, dl, OpVT, 4942 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1), 4943 DAG.getConstant(32, dl, MVT::i32)); 4944 } else if (VT == MVT::f32) 4945 Tmp1 = DAG.getNode(ARMISD::VSHRu, dl, MVT::v1i64, 4946 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1), 4947 DAG.getConstant(32, dl, MVT::i32)); 4948 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0); 4949 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1); 4950 4951 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createNEONModImm(0xe, 0xff), 4952 dl, MVT::i32); 4953 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes); 4954 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask, 4955 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes)); 4956 4957 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT, 4958 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask), 4959 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot)); 4960 if (VT == MVT::f32) { 4961 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res); 4962 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, 4963 DAG.getConstant(0, dl, MVT::i32)); 4964 } else { 4965 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res); 4966 } 4967 4968 return Res; 4969 } 4970 4971 // Bitcast operand 1 to i32. 4972 if (SrcVT == MVT::f64) 4973 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32), 4974 Tmp1).getValue(1); 4975 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1); 4976 4977 // Or in the signbit with integer operations. 4978 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32); 4979 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32); 4980 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1); 4981 if (VT == MVT::f32) { 4982 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32, 4983 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2); 4984 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4985 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1)); 4986 } 4987 4988 // f64: Or the high part with signbit and then combine two parts. 4989 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32), 4990 Tmp0); 4991 SDValue Lo = Tmp0.getValue(0); 4992 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2); 4993 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1); 4994 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi); 4995 } 4996 4997 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{ 4998 MachineFunction &MF = DAG.getMachineFunction(); 4999 MachineFrameInfo &MFI = MF.getFrameInfo(); 5000 MFI.setReturnAddressIsTaken(true); 5001 5002 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 5003 return SDValue(); 5004 5005 EVT VT = Op.getValueType(); 5006 SDLoc dl(Op); 5007 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 5008 if (Depth) { 5009 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 5010 SDValue Offset = DAG.getConstant(4, dl, MVT::i32); 5011 return DAG.getLoad(VT, dl, DAG.getEntryNode(), 5012 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), 5013 MachinePointerInfo()); 5014 } 5015 5016 // Return LR, which contains the return address. Mark it an implicit live-in. 5017 unsigned Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32)); 5018 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); 5019 } 5020 5021 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { 5022 const ARMBaseRegisterInfo &ARI = 5023 *static_cast<const ARMBaseRegisterInfo*>(RegInfo); 5024 MachineFunction &MF = DAG.getMachineFunction(); 5025 MachineFrameInfo &MFI = MF.getFrameInfo(); 5026 MFI.setFrameAddressIsTaken(true); 5027 5028 EVT VT = Op.getValueType(); 5029 SDLoc dl(Op); // FIXME probably not meaningful 5030 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 5031 unsigned FrameReg = ARI.getFrameRegister(MF); 5032 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); 5033 while (Depth--) 5034 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, 5035 MachinePointerInfo()); 5036 return FrameAddr; 5037 } 5038 5039 // FIXME? Maybe this could be a TableGen attribute on some registers and 5040 // this table could be generated automatically from RegInfo. 5041 unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT, 5042 SelectionDAG &DAG) const { 5043 unsigned Reg = StringSwitch<unsigned>(RegName) 5044 .Case("sp", ARM::SP) 5045 .Default(0); 5046 if (Reg) 5047 return Reg; 5048 report_fatal_error(Twine("Invalid register name \"" 5049 + StringRef(RegName) + "\".")); 5050 } 5051 5052 // Result is 64 bit value so split into two 32 bit values and return as a 5053 // pair of values. 5054 static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results, 5055 SelectionDAG &DAG) { 5056 SDLoc DL(N); 5057 5058 // This function is only supposed to be called for i64 type destination. 5059 assert(N->getValueType(0) == MVT::i64 5060 && "ExpandREAD_REGISTER called for non-i64 type result."); 5061 5062 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL, 5063 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other), 5064 N->getOperand(0), 5065 N->getOperand(1)); 5066 5067 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0), 5068 Read.getValue(1))); 5069 Results.push_back(Read.getOperand(0)); 5070 } 5071 5072 /// \p BC is a bitcast that is about to be turned into a VMOVDRR. 5073 /// When \p DstVT, the destination type of \p BC, is on the vector 5074 /// register bank and the source of bitcast, \p Op, operates on the same bank, 5075 /// it might be possible to combine them, such that everything stays on the 5076 /// vector register bank. 5077 /// \p return The node that would replace \p BT, if the combine 5078 /// is possible. 5079 static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, 5080 SelectionDAG &DAG) { 5081 SDValue Op = BC->getOperand(0); 5082 EVT DstVT = BC->getValueType(0); 5083 5084 // The only vector instruction that can produce a scalar (remember, 5085 // since the bitcast was about to be turned into VMOVDRR, the source 5086 // type is i64) from a vector is EXTRACT_VECTOR_ELT. 5087 // Moreover, we can do this combine only if there is one use. 5088 // Finally, if the destination type is not a vector, there is not 5089 // much point on forcing everything on the vector bank. 5090 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 5091 !Op.hasOneUse()) 5092 return SDValue(); 5093 5094 // If the index is not constant, we will introduce an additional 5095 // multiply that will stick. 5096 // Give up in that case. 5097 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 5098 if (!Index) 5099 return SDValue(); 5100 unsigned DstNumElt = DstVT.getVectorNumElements(); 5101 5102 // Compute the new index. 5103 const APInt &APIntIndex = Index->getAPIntValue(); 5104 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt); 5105 NewIndex *= APIntIndex; 5106 // Check if the new constant index fits into i32. 5107 if (NewIndex.getBitWidth() > 32) 5108 return SDValue(); 5109 5110 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) -> 5111 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M) 5112 SDLoc dl(Op); 5113 SDValue ExtractSrc = Op.getOperand(0); 5114 EVT VecVT = EVT::getVectorVT( 5115 *DAG.getContext(), DstVT.getScalarType(), 5116 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt); 5117 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc); 5118 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast, 5119 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32)); 5120 } 5121 5122 /// ExpandBITCAST - If the target supports VFP, this function is called to 5123 /// expand a bit convert where either the source or destination type is i64 to 5124 /// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64 5125 /// operand type is illegal (e.g., v2f32 for a target that doesn't support 5126 /// vectors), since the legalizer won't know what to do with that. 5127 static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG, 5128 const ARMSubtarget *Subtarget) { 5129 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5130 SDLoc dl(N); 5131 SDValue Op = N->getOperand(0); 5132 5133 // This function is only supposed to be called for i64 types, either as the 5134 // source or destination of the bit convert. 5135 EVT SrcVT = Op.getValueType(); 5136 EVT DstVT = N->getValueType(0); 5137 const bool HasFullFP16 = Subtarget->hasFullFP16(); 5138 5139 if (SrcVT == MVT::f32 && DstVT == MVT::i32) { 5140 // FullFP16: half values are passed in S-registers, and we don't 5141 // need any of the bitcast and moves: 5142 // 5143 // t2: f32,ch = CopyFromReg t0, Register:f32 %0 5144 // t5: i32 = bitcast t2 5145 // t18: f16 = ARMISD::VMOVhr t5 5146 if (Op.getOpcode() != ISD::CopyFromReg || 5147 Op.getValueType() != MVT::f32) 5148 return SDValue(); 5149 5150 auto Move = N->use_begin(); 5151 if (Move->getOpcode() != ARMISD::VMOVhr) 5152 return SDValue(); 5153 5154 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 5155 SDValue Copy = DAG.getNode(ISD::CopyFromReg, SDLoc(Op), MVT::f16, Ops); 5156 DAG.ReplaceAllUsesWith(*Move, &Copy); 5157 return Copy; 5158 } 5159 5160 if (SrcVT == MVT::i16 && DstVT == MVT::f16) { 5161 if (!HasFullFP16) 5162 return SDValue(); 5163 // SoftFP: read half-precision arguments: 5164 // 5165 // t2: i32,ch = ... 5166 // t7: i16 = truncate t2 <~~~~ Op 5167 // t8: f16 = bitcast t7 <~~~~ N 5168 // 5169 if (Op.getOperand(0).getValueType() == MVT::i32) 5170 return DAG.getNode(ARMISD::VMOVhr, SDLoc(Op), 5171 MVT::f16, Op.getOperand(0)); 5172 5173 return SDValue(); 5174 } 5175 5176 // Half-precision return values 5177 if (SrcVT == MVT::f16 && DstVT == MVT::i16) { 5178 if (!HasFullFP16) 5179 return SDValue(); 5180 // 5181 // t11: f16 = fadd t8, t10 5182 // t12: i16 = bitcast t11 <~~~ SDNode N 5183 // t13: i32 = zero_extend t12 5184 // t16: ch,glue = CopyToReg t0, Register:i32 %r0, t13 5185 // t17: ch = ARMISD::RET_FLAG t16, Register:i32 %r0, t16:1 5186 // 5187 // transform this into: 5188 // 5189 // t20: i32 = ARMISD::VMOVrh t11 5190 // t16: ch,glue = CopyToReg t0, Register:i32 %r0, t20 5191 // 5192 auto ZeroExtend = N->use_begin(); 5193 if (N->use_size() != 1 || ZeroExtend->getOpcode() != ISD::ZERO_EXTEND || 5194 ZeroExtend->getValueType(0) != MVT::i32) 5195 return SDValue(); 5196 5197 auto Copy = ZeroExtend->use_begin(); 5198 if (Copy->getOpcode() == ISD::CopyToReg && 5199 Copy->use_begin()->getOpcode() == ARMISD::RET_FLAG) { 5200 SDValue Cvt = DAG.getNode(ARMISD::VMOVrh, SDLoc(Op), MVT::i32, Op); 5201 DAG.ReplaceAllUsesWith(*ZeroExtend, &Cvt); 5202 return Cvt; 5203 } 5204 return SDValue(); 5205 } 5206 5207 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64)) 5208 return SDValue(); 5209 5210 // Turn i64->f64 into VMOVDRR. 5211 if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) { 5212 // Do not force values to GPRs (this is what VMOVDRR does for the inputs) 5213 // if we can combine the bitcast with its source. 5214 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG)) 5215 return Val; 5216 5217 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op, 5218 DAG.getConstant(0, dl, MVT::i32)); 5219 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op, 5220 DAG.getConstant(1, dl, MVT::i32)); 5221 return DAG.getNode(ISD::BITCAST, dl, DstVT, 5222 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi)); 5223 } 5224 5225 // Turn f64->i64 into VMOVRRD. 5226 if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) { 5227 SDValue Cvt; 5228 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() && 5229 SrcVT.getVectorNumElements() > 1) 5230 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl, 5231 DAG.getVTList(MVT::i32, MVT::i32), 5232 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op)); 5233 else 5234 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl, 5235 DAG.getVTList(MVT::i32, MVT::i32), Op); 5236 // Merge the pieces into a single i64 value. 5237 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1)); 5238 } 5239 5240 return SDValue(); 5241 } 5242 5243 /// getZeroVector - Returns a vector of specified type with all zero elements. 5244 /// Zero vectors are used to represent vector negation and in those cases 5245 /// will be implemented with the NEON VNEG instruction. However, VNEG does 5246 /// not support i64 elements, so sometimes the zero vectors will need to be 5247 /// explicitly constructed. Regardless, use a canonical VMOV to create the 5248 /// zero vector. 5249 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { 5250 assert(VT.isVector() && "Expected a vector type"); 5251 // The canonical modified immediate encoding of a zero vector is....0! 5252 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32); 5253 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; 5254 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal); 5255 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); 5256 } 5257 5258 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two 5259 /// i32 values and take a 2 x i32 value to shift plus a shift amount. 5260 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op, 5261 SelectionDAG &DAG) const { 5262 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 5263 EVT VT = Op.getValueType(); 5264 unsigned VTBits = VT.getSizeInBits(); 5265 SDLoc dl(Op); 5266 SDValue ShOpLo = Op.getOperand(0); 5267 SDValue ShOpHi = Op.getOperand(1); 5268 SDValue ShAmt = Op.getOperand(2); 5269 SDValue ARMcc; 5270 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 5271 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; 5272 5273 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); 5274 5275 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, 5276 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt); 5277 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); 5278 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, 5279 DAG.getConstant(VTBits, dl, MVT::i32)); 5280 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); 5281 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 5282 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); 5283 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5284 ISD::SETGE, ARMcc, DAG, dl); 5285 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, 5286 ARMcc, CCR, CmpLo); 5287 5288 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); 5289 SDValue HiBigShift = Opc == ISD::SRA 5290 ? DAG.getNode(Opc, dl, VT, ShOpHi, 5291 DAG.getConstant(VTBits - 1, dl, VT)) 5292 : DAG.getConstant(0, dl, VT); 5293 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5294 ISD::SETGE, ARMcc, DAG, dl); 5295 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, 5296 ARMcc, CCR, CmpHi); 5297 5298 SDValue Ops[2] = { Lo, Hi }; 5299 return DAG.getMergeValues(Ops, dl); 5300 } 5301 5302 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two 5303 /// i32 values and take a 2 x i32 value to shift plus a shift amount. 5304 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op, 5305 SelectionDAG &DAG) const { 5306 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 5307 EVT VT = Op.getValueType(); 5308 unsigned VTBits = VT.getSizeInBits(); 5309 SDLoc dl(Op); 5310 SDValue ShOpLo = Op.getOperand(0); 5311 SDValue ShOpHi = Op.getOperand(1); 5312 SDValue ShAmt = Op.getOperand(2); 5313 SDValue ARMcc; 5314 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 5315 5316 assert(Op.getOpcode() == ISD::SHL_PARTS); 5317 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, 5318 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt); 5319 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); 5320 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); 5321 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 5322 5323 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, 5324 DAG.getConstant(VTBits, dl, MVT::i32)); 5325 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); 5326 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5327 ISD::SETGE, ARMcc, DAG, dl); 5328 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, 5329 ARMcc, CCR, CmpHi); 5330 5331 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32), 5332 ISD::SETGE, ARMcc, DAG, dl); 5333 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 5334 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, 5335 DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo); 5336 5337 SDValue Ops[2] = { Lo, Hi }; 5338 return DAG.getMergeValues(Ops, dl); 5339 } 5340 5341 SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 5342 SelectionDAG &DAG) const { 5343 // The rounding mode is in bits 23:22 of the FPSCR. 5344 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0 5345 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3) 5346 // so that the shift + and get folded into a bitfield extract. 5347 SDLoc dl(Op); 5348 SDValue Ops[] = { DAG.getEntryNode(), 5349 DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32) }; 5350 5351 SDValue FPSCR = DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, MVT::i32, Ops); 5352 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR, 5353 DAG.getConstant(1U << 22, dl, MVT::i32)); 5354 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds, 5355 DAG.getConstant(22, dl, MVT::i32)); 5356 return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE, 5357 DAG.getConstant(3, dl, MVT::i32)); 5358 } 5359 5360 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, 5361 const ARMSubtarget *ST) { 5362 SDLoc dl(N); 5363 EVT VT = N->getValueType(0); 5364 if (VT.isVector()) { 5365 assert(ST->hasNEON()); 5366 5367 // Compute the least significant set bit: LSB = X & -X 5368 SDValue X = N->getOperand(0); 5369 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X); 5370 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX); 5371 5372 EVT ElemTy = VT.getVectorElementType(); 5373 5374 if (ElemTy == MVT::i8) { 5375 // Compute with: cttz(x) = ctpop(lsb - 1) 5376 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5377 DAG.getTargetConstant(1, dl, ElemTy)); 5378 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); 5379 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); 5380 } 5381 5382 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) && 5383 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) { 5384 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0 5385 unsigned NumBits = ElemTy.getSizeInBits(); 5386 SDValue WidthMinus1 = 5387 DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5388 DAG.getTargetConstant(NumBits - 1, dl, ElemTy)); 5389 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB); 5390 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ); 5391 } 5392 5393 // Compute with: cttz(x) = ctpop(lsb - 1) 5394 5395 // Since we can only compute the number of bits in a byte with vcnt.8, we 5396 // have to gather the result with pairwise addition (vpaddl) for i16, i32, 5397 // and i64. 5398 5399 // Compute LSB - 1. 5400 SDValue Bits; 5401 if (ElemTy == MVT::i64) { 5402 // Load constant 0xffff'ffff'ffff'ffff to register. 5403 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5404 DAG.getTargetConstant(0x1eff, dl, MVT::i32)); 5405 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF); 5406 } else { 5407 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, 5408 DAG.getTargetConstant(1, dl, ElemTy)); 5409 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); 5410 } 5411 5412 // Count #bits with vcnt.8. 5413 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; 5414 SDValue BitsVT8 = DAG.getNode(ISD::BITCAST, dl, VT8Bit, Bits); 5415 SDValue Cnt8 = DAG.getNode(ISD::CTPOP, dl, VT8Bit, BitsVT8); 5416 5417 // Gather the #bits with vpaddl (pairwise add.) 5418 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16; 5419 SDValue Cnt16 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT16Bit, 5420 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32), 5421 Cnt8); 5422 if (ElemTy == MVT::i16) 5423 return Cnt16; 5424 5425 EVT VT32Bit = VT.is64BitVector() ? MVT::v2i32 : MVT::v4i32; 5426 SDValue Cnt32 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT32Bit, 5427 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32), 5428 Cnt16); 5429 if (ElemTy == MVT::i32) 5430 return Cnt32; 5431 5432 assert(ElemTy == MVT::i64); 5433 SDValue Cnt64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 5434 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32), 5435 Cnt32); 5436 return Cnt64; 5437 } 5438 5439 if (!ST->hasV6T2Ops()) 5440 return SDValue(); 5441 5442 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); 5443 return DAG.getNode(ISD::CTLZ, dl, VT, rbit); 5444 } 5445 5446 /// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count 5447 /// for each 16-bit element from operand, repeated. The basic idea is to 5448 /// leverage vcnt to get the 8-bit counts, gather and add the results. 5449 /// 5450 /// Trace for v4i16: 5451 /// input = [v0 v1 v2 v3 ] (vi 16-bit element) 5452 /// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element) 5453 /// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi) 5454 /// vrev: N2 = [b1 b0 b3 b2 b5 b4 b7 b6] 5455 /// [b0 b1 b2 b3 b4 b5 b6 b7] 5456 /// +[b1 b0 b3 b2 b5 b4 b7 b6] 5457 /// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0, 5458 /// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits) 5459 static SDValue getCTPOP16BitCounts(SDNode *N, SelectionDAG &DAG) { 5460 EVT VT = N->getValueType(0); 5461 SDLoc DL(N); 5462 5463 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; 5464 SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0)); 5465 SDValue N1 = DAG.getNode(ISD::CTPOP, DL, VT8Bit, N0); 5466 SDValue N2 = DAG.getNode(ARMISD::VREV16, DL, VT8Bit, N1); 5467 SDValue N3 = DAG.getNode(ISD::ADD, DL, VT8Bit, N1, N2); 5468 return DAG.getNode(ARMISD::VUZP, DL, VT8Bit, N3, N3); 5469 } 5470 5471 /// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the 5472 /// bit-count for each 16-bit element from the operand. We need slightly 5473 /// different sequencing for v4i16 and v8i16 to stay within NEON's available 5474 /// 64/128-bit registers. 5475 /// 5476 /// Trace for v4i16: 5477 /// input = [v0 v1 v2 v3 ] (vi 16-bit element) 5478 /// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi) 5479 /// v8i16:Extended = [k0 k1 k2 k3 k0 k1 k2 k3 ] 5480 /// v4i16:Extracted = [k0 k1 k2 k3 ] 5481 static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) { 5482 EVT VT = N->getValueType(0); 5483 SDLoc DL(N); 5484 5485 SDValue BitCounts = getCTPOP16BitCounts(N, DAG); 5486 if (VT.is64BitVector()) { 5487 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts); 5488 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended, 5489 DAG.getIntPtrConstant(0, DL)); 5490 } else { 5491 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, 5492 BitCounts, DAG.getIntPtrConstant(0, DL)); 5493 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted); 5494 } 5495 } 5496 5497 /// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the 5498 /// bit-count for each 32-bit element from the operand. The idea here is 5499 /// to split the vector into 16-bit elements, leverage the 16-bit count 5500 /// routine, and then combine the results. 5501 /// 5502 /// Trace for v2i32 (v4i32 similar with Extracted/Extended exchanged): 5503 /// input = [v0 v1 ] (vi: 32-bit elements) 5504 /// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1]) 5505 /// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi) 5506 /// vrev: N0 = [k1 k0 k3 k2 ] 5507 /// [k0 k1 k2 k3 ] 5508 /// N1 =+[k1 k0 k3 k2 ] 5509 /// [k0 k2 k1 k3 ] 5510 /// N2 =+[k1 k3 k0 k2 ] 5511 /// [k0 k2 k1 k3 ] 5512 /// Extended =+[k1 k3 k0 k2 ] 5513 /// [k0 k2 ] 5514 /// Extracted=+[k1 k3 ] 5515 /// 5516 static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) { 5517 EVT VT = N->getValueType(0); 5518 SDLoc DL(N); 5519 5520 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16; 5521 5522 SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0)); 5523 SDValue Counts16 = lowerCTPOP16BitElements(Bitcast.getNode(), DAG); 5524 SDValue N0 = DAG.getNode(ARMISD::VREV32, DL, VT16Bit, Counts16); 5525 SDValue N1 = DAG.getNode(ISD::ADD, DL, VT16Bit, Counts16, N0); 5526 SDValue N2 = DAG.getNode(ARMISD::VUZP, DL, VT16Bit, N1, N1); 5527 5528 if (VT.is64BitVector()) { 5529 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2); 5530 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended, 5531 DAG.getIntPtrConstant(0, DL)); 5532 } else { 5533 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2, 5534 DAG.getIntPtrConstant(0, DL)); 5535 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted); 5536 } 5537 } 5538 5539 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG, 5540 const ARMSubtarget *ST) { 5541 EVT VT = N->getValueType(0); 5542 5543 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON."); 5544 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || 5545 VT == MVT::v4i16 || VT == MVT::v8i16) && 5546 "Unexpected type for custom ctpop lowering"); 5547 5548 if (VT.getVectorElementType() == MVT::i32) 5549 return lowerCTPOP32BitElements(N, DAG); 5550 else 5551 return lowerCTPOP16BitElements(N, DAG); 5552 } 5553 5554 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, 5555 const ARMSubtarget *ST) { 5556 EVT VT = N->getValueType(0); 5557 SDLoc dl(N); 5558 5559 if (!VT.isVector()) 5560 return SDValue(); 5561 5562 // Lower vector shifts on NEON to use VSHL. 5563 assert(ST->hasNEON() && "unexpected vector shift"); 5564 5565 // Left shifts translate directly to the vshiftu intrinsic. 5566 if (N->getOpcode() == ISD::SHL) 5567 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 5568 DAG.getConstant(Intrinsic::arm_neon_vshiftu, dl, 5569 MVT::i32), 5570 N->getOperand(0), N->getOperand(1)); 5571 5572 assert((N->getOpcode() == ISD::SRA || 5573 N->getOpcode() == ISD::SRL) && "unexpected vector shift opcode"); 5574 5575 // NEON uses the same intrinsics for both left and right shifts. For 5576 // right shifts, the shift amounts are negative, so negate the vector of 5577 // shift amounts. 5578 EVT ShiftVT = N->getOperand(1).getValueType(); 5579 SDValue NegatedCount = DAG.getNode(ISD::SUB, dl, ShiftVT, 5580 getZeroVector(ShiftVT, DAG, dl), 5581 N->getOperand(1)); 5582 Intrinsic::ID vshiftInt = (N->getOpcode() == ISD::SRA ? 5583 Intrinsic::arm_neon_vshifts : 5584 Intrinsic::arm_neon_vshiftu); 5585 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 5586 DAG.getConstant(vshiftInt, dl, MVT::i32), 5587 N->getOperand(0), NegatedCount); 5588 } 5589 5590 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, 5591 const ARMSubtarget *ST) { 5592 EVT VT = N->getValueType(0); 5593 SDLoc dl(N); 5594 5595 // We can get here for a node like i32 = ISD::SHL i32, i64 5596 if (VT != MVT::i64) 5597 return SDValue(); 5598 5599 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && 5600 "Unknown shift to lower!"); 5601 5602 // We only lower SRA, SRL of 1 here, all others use generic lowering. 5603 if (!isOneConstant(N->getOperand(1))) 5604 return SDValue(); 5605 5606 // If we are in thumb mode, we don't have RRX. 5607 if (ST->isThumb1Only()) return SDValue(); 5608 5609 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr. 5610 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0), 5611 DAG.getConstant(0, dl, MVT::i32)); 5612 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0), 5613 DAG.getConstant(1, dl, MVT::i32)); 5614 5615 // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and 5616 // captures the result into a carry flag. 5617 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG; 5618 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Glue), Hi); 5619 5620 // The low part is an ARMISD::RRX operand, which shifts the carry in. 5621 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1)); 5622 5623 // Merge the pieces into a single i64 value. 5624 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 5625 } 5626 5627 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) { 5628 SDValue TmpOp0, TmpOp1; 5629 bool Invert = false; 5630 bool Swap = false; 5631 unsigned Opc = 0; 5632 5633 SDValue Op0 = Op.getOperand(0); 5634 SDValue Op1 = Op.getOperand(1); 5635 SDValue CC = Op.getOperand(2); 5636 EVT CmpVT = Op0.getValueType().changeVectorElementTypeToInteger(); 5637 EVT VT = Op.getValueType(); 5638 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 5639 SDLoc dl(Op); 5640 5641 if (Op0.getValueType().getVectorElementType() == MVT::i64 && 5642 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) { 5643 // Special-case integer 64-bit equality comparisons. They aren't legal, 5644 // but they can be lowered with a few vector instructions. 5645 unsigned CmpElements = CmpVT.getVectorNumElements() * 2; 5646 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements); 5647 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0); 5648 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1); 5649 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1, 5650 DAG.getCondCode(ISD::SETEQ)); 5651 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp); 5652 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed); 5653 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged); 5654 if (SetCCOpcode == ISD::SETNE) 5655 Merged = DAG.getNOT(dl, Merged, CmpVT); 5656 Merged = DAG.getSExtOrTrunc(Merged, dl, VT); 5657 return Merged; 5658 } 5659 5660 if (CmpVT.getVectorElementType() == MVT::i64) 5661 // 64-bit comparisons are not legal in general. 5662 return SDValue(); 5663 5664 if (Op1.getValueType().isFloatingPoint()) { 5665 switch (SetCCOpcode) { 5666 default: llvm_unreachable("Illegal FP comparison"); 5667 case ISD::SETUNE: 5668 case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH; 5669 case ISD::SETOEQ: 5670 case ISD::SETEQ: Opc = ARMISD::VCEQ; break; 5671 case ISD::SETOLT: 5672 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH; 5673 case ISD::SETOGT: 5674 case ISD::SETGT: Opc = ARMISD::VCGT; break; 5675 case ISD::SETOLE: 5676 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH; 5677 case ISD::SETOGE: 5678 case ISD::SETGE: Opc = ARMISD::VCGE; break; 5679 case ISD::SETUGE: Swap = true; LLVM_FALLTHROUGH; 5680 case ISD::SETULE: Invert = true; Opc = ARMISD::VCGT; break; 5681 case ISD::SETUGT: Swap = true; LLVM_FALLTHROUGH; 5682 case ISD::SETULT: Invert = true; Opc = ARMISD::VCGE; break; 5683 case ISD::SETUEQ: Invert = true; LLVM_FALLTHROUGH; 5684 case ISD::SETONE: 5685 // Expand this to (OLT | OGT). 5686 TmpOp0 = Op0; 5687 TmpOp1 = Op1; 5688 Opc = ISD::OR; 5689 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0); 5690 Op1 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp0, TmpOp1); 5691 break; 5692 case ISD::SETUO: 5693 Invert = true; 5694 LLVM_FALLTHROUGH; 5695 case ISD::SETO: 5696 // Expand this to (OLT | OGE). 5697 TmpOp0 = Op0; 5698 TmpOp1 = Op1; 5699 Opc = ISD::OR; 5700 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0); 5701 Op1 = DAG.getNode(ARMISD::VCGE, dl, CmpVT, TmpOp0, TmpOp1); 5702 break; 5703 } 5704 } else { 5705 // Integer comparisons. 5706 switch (SetCCOpcode) { 5707 default: llvm_unreachable("Illegal integer comparison"); 5708 case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH; 5709 case ISD::SETEQ: Opc = ARMISD::VCEQ; break; 5710 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH; 5711 case ISD::SETGT: Opc = ARMISD::VCGT; break; 5712 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH; 5713 case ISD::SETGE: Opc = ARMISD::VCGE; break; 5714 case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH; 5715 case ISD::SETUGT: Opc = ARMISD::VCGTU; break; 5716 case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH; 5717 case ISD::SETUGE: Opc = ARMISD::VCGEU; break; 5718 } 5719 5720 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero). 5721 if (Opc == ARMISD::VCEQ) { 5722 SDValue AndOp; 5723 if (ISD::isBuildVectorAllZeros(Op1.getNode())) 5724 AndOp = Op0; 5725 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) 5726 AndOp = Op1; 5727 5728 // Ignore bitconvert. 5729 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST) 5730 AndOp = AndOp.getOperand(0); 5731 5732 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) { 5733 Opc = ARMISD::VTST; 5734 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0)); 5735 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1)); 5736 Invert = !Invert; 5737 } 5738 } 5739 } 5740 5741 if (Swap) 5742 std::swap(Op0, Op1); 5743 5744 // If one of the operands is a constant vector zero, attempt to fold the 5745 // comparison to a specialized compare-against-zero form. 5746 SDValue SingleOp; 5747 if (ISD::isBuildVectorAllZeros(Op1.getNode())) 5748 SingleOp = Op0; 5749 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) { 5750 if (Opc == ARMISD::VCGE) 5751 Opc = ARMISD::VCLEZ; 5752 else if (Opc == ARMISD::VCGT) 5753 Opc = ARMISD::VCLTZ; 5754 SingleOp = Op1; 5755 } 5756 5757 SDValue Result; 5758 if (SingleOp.getNode()) { 5759 switch (Opc) { 5760 case ARMISD::VCEQ: 5761 Result = DAG.getNode(ARMISD::VCEQZ, dl, CmpVT, SingleOp); break; 5762 case ARMISD::VCGE: 5763 Result = DAG.getNode(ARMISD::VCGEZ, dl, CmpVT, SingleOp); break; 5764 case ARMISD::VCLEZ: 5765 Result = DAG.getNode(ARMISD::VCLEZ, dl, CmpVT, SingleOp); break; 5766 case ARMISD::VCGT: 5767 Result = DAG.getNode(ARMISD::VCGTZ, dl, CmpVT, SingleOp); break; 5768 case ARMISD::VCLTZ: 5769 Result = DAG.getNode(ARMISD::VCLTZ, dl, CmpVT, SingleOp); break; 5770 default: 5771 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1); 5772 } 5773 } else { 5774 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1); 5775 } 5776 5777 Result = DAG.getSExtOrTrunc(Result, dl, VT); 5778 5779 if (Invert) 5780 Result = DAG.getNOT(dl, Result, VT); 5781 5782 return Result; 5783 } 5784 5785 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) { 5786 SDValue LHS = Op.getOperand(0); 5787 SDValue RHS = Op.getOperand(1); 5788 SDValue Carry = Op.getOperand(2); 5789 SDValue Cond = Op.getOperand(3); 5790 SDLoc DL(Op); 5791 5792 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only."); 5793 5794 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we 5795 // have to invert the carry first. 5796 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, 5797 DAG.getConstant(1, DL, MVT::i32), Carry); 5798 // This converts the boolean value carry into the carry flag. 5799 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG); 5800 5801 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); 5802 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry); 5803 5804 SDValue FVal = DAG.getConstant(0, DL, MVT::i32); 5805 SDValue TVal = DAG.getConstant(1, DL, MVT::i32); 5806 SDValue ARMcc = DAG.getConstant( 5807 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32); 5808 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32); 5809 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR, 5810 Cmp.getValue(1), SDValue()); 5811 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc, 5812 CCR, Chain.getValue(1)); 5813 } 5814 5815 /// isNEONModifiedImm - Check if the specified splat value corresponds to a 5816 /// valid vector constant for a NEON instruction with a "modified immediate" 5817 /// operand (e.g., VMOV). If so, return the encoded value. 5818 static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, 5819 unsigned SplatBitSize, SelectionDAG &DAG, 5820 const SDLoc &dl, EVT &VT, bool is128Bits, 5821 NEONModImmType type) { 5822 unsigned OpCmode, Imm; 5823 5824 // SplatBitSize is set to the smallest size that splats the vector, so a 5825 // zero vector will always have SplatBitSize == 8. However, NEON modified 5826 // immediate instructions others than VMOV do not support the 8-bit encoding 5827 // of a zero vector, and the default encoding of zero is supposed to be the 5828 // 32-bit version. 5829 if (SplatBits == 0) 5830 SplatBitSize = 32; 5831 5832 switch (SplatBitSize) { 5833 case 8: 5834 if (type != VMOVModImm) 5835 return SDValue(); 5836 // Any 1-byte value is OK. Op=0, Cmode=1110. 5837 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big"); 5838 OpCmode = 0xe; 5839 Imm = SplatBits; 5840 VT = is128Bits ? MVT::v16i8 : MVT::v8i8; 5841 break; 5842 5843 case 16: 5844 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero. 5845 VT = is128Bits ? MVT::v8i16 : MVT::v4i16; 5846 if ((SplatBits & ~0xff) == 0) { 5847 // Value = 0x00nn: Op=x, Cmode=100x. 5848 OpCmode = 0x8; 5849 Imm = SplatBits; 5850 break; 5851 } 5852 if ((SplatBits & ~0xff00) == 0) { 5853 // Value = 0xnn00: Op=x, Cmode=101x. 5854 OpCmode = 0xa; 5855 Imm = SplatBits >> 8; 5856 break; 5857 } 5858 return SDValue(); 5859 5860 case 32: 5861 // NEON's 32-bit VMOV supports splat values where: 5862 // * only one byte is nonzero, or 5863 // * the least significant byte is 0xff and the second byte is nonzero, or 5864 // * the least significant 2 bytes are 0xff and the third is nonzero. 5865 VT = is128Bits ? MVT::v4i32 : MVT::v2i32; 5866 if ((SplatBits & ~0xff) == 0) { 5867 // Value = 0x000000nn: Op=x, Cmode=000x. 5868 OpCmode = 0; 5869 Imm = SplatBits; 5870 break; 5871 } 5872 if ((SplatBits & ~0xff00) == 0) { 5873 // Value = 0x0000nn00: Op=x, Cmode=001x. 5874 OpCmode = 0x2; 5875 Imm = SplatBits >> 8; 5876 break; 5877 } 5878 if ((SplatBits & ~0xff0000) == 0) { 5879 // Value = 0x00nn0000: Op=x, Cmode=010x. 5880 OpCmode = 0x4; 5881 Imm = SplatBits >> 16; 5882 break; 5883 } 5884 if ((SplatBits & ~0xff000000) == 0) { 5885 // Value = 0xnn000000: Op=x, Cmode=011x. 5886 OpCmode = 0x6; 5887 Imm = SplatBits >> 24; 5888 break; 5889 } 5890 5891 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC 5892 if (type == OtherModImm) return SDValue(); 5893 5894 if ((SplatBits & ~0xffff) == 0 && 5895 ((SplatBits | SplatUndef) & 0xff) == 0xff) { 5896 // Value = 0x0000nnff: Op=x, Cmode=1100. 5897 OpCmode = 0xc; 5898 Imm = SplatBits >> 8; 5899 break; 5900 } 5901 5902 if ((SplatBits & ~0xffffff) == 0 && 5903 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) { 5904 // Value = 0x00nnffff: Op=x, Cmode=1101. 5905 OpCmode = 0xd; 5906 Imm = SplatBits >> 16; 5907 break; 5908 } 5909 5910 // Note: there are a few 32-bit splat values (specifically: 00ffff00, 5911 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not 5912 // VMOV.I32. A (very) minor optimization would be to replicate the value 5913 // and fall through here to test for a valid 64-bit splat. But, then the 5914 // caller would also need to check and handle the change in size. 5915 return SDValue(); 5916 5917 case 64: { 5918 if (type != VMOVModImm) 5919 return SDValue(); 5920 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff. 5921 uint64_t BitMask = 0xff; 5922 uint64_t Val = 0; 5923 unsigned ImmMask = 1; 5924 Imm = 0; 5925 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) { 5926 if (((SplatBits | SplatUndef) & BitMask) == BitMask) { 5927 Val |= BitMask; 5928 Imm |= ImmMask; 5929 } else if ((SplatBits & BitMask) != 0) { 5930 return SDValue(); 5931 } 5932 BitMask <<= 8; 5933 ImmMask <<= 1; 5934 } 5935 5936 if (DAG.getDataLayout().isBigEndian()) 5937 // swap higher and lower 32 bit word 5938 Imm = ((Imm & 0xf) << 4) | ((Imm & 0xf0) >> 4); 5939 5940 // Op=1, Cmode=1110. 5941 OpCmode = 0x1e; 5942 VT = is128Bits ? MVT::v2i64 : MVT::v1i64; 5943 break; 5944 } 5945 5946 default: 5947 llvm_unreachable("unexpected size for isNEONModifiedImm"); 5948 } 5949 5950 unsigned EncodedVal = ARM_AM::createNEONModImm(OpCmode, Imm); 5951 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32); 5952 } 5953 5954 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG, 5955 const ARMSubtarget *ST) const { 5956 EVT VT = Op.getValueType(); 5957 bool IsDouble = (VT == MVT::f64); 5958 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op); 5959 const APFloat &FPVal = CFP->getValueAPF(); 5960 5961 // Prevent floating-point constants from using literal loads 5962 // when execute-only is enabled. 5963 if (ST->genExecuteOnly()) { 5964 // If we can represent the constant as an immediate, don't lower it 5965 if (isFPImmLegal(FPVal, VT)) 5966 return Op; 5967 // Otherwise, construct as integer, and move to float register 5968 APInt INTVal = FPVal.bitcastToAPInt(); 5969 SDLoc DL(CFP); 5970 switch (VT.getSimpleVT().SimpleTy) { 5971 default: 5972 llvm_unreachable("Unknown floating point type!"); 5973 break; 5974 case MVT::f64: { 5975 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32); 5976 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32); 5977 if (!ST->isLittle()) 5978 std::swap(Lo, Hi); 5979 return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi); 5980 } 5981 case MVT::f32: 5982 return DAG.getNode(ARMISD::VMOVSR, DL, VT, 5983 DAG.getConstant(INTVal, DL, MVT::i32)); 5984 } 5985 } 5986 5987 if (!ST->hasVFP3()) 5988 return SDValue(); 5989 5990 // Use the default (constant pool) lowering for double constants when we have 5991 // an SP-only FPU 5992 if (IsDouble && Subtarget->isFPOnlySP()) 5993 return SDValue(); 5994 5995 // Try splatting with a VMOV.f32... 5996 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal); 5997 5998 if (ImmVal != -1) { 5999 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) { 6000 // We have code in place to select a valid ConstantFP already, no need to 6001 // do any mangling. 6002 return Op; 6003 } 6004 6005 // It's a float and we are trying to use NEON operations where 6006 // possible. Lower it to a splat followed by an extract. 6007 SDLoc DL(Op); 6008 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32); 6009 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32, 6010 NewVal); 6011 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant, 6012 DAG.getConstant(0, DL, MVT::i32)); 6013 } 6014 6015 // The rest of our options are NEON only, make sure that's allowed before 6016 // proceeding.. 6017 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP())) 6018 return SDValue(); 6019 6020 EVT VMovVT; 6021 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue(); 6022 6023 // It wouldn't really be worth bothering for doubles except for one very 6024 // important value, which does happen to match: 0.0. So make sure we don't do 6025 // anything stupid. 6026 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32)) 6027 return SDValue(); 6028 6029 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too). 6030 SDValue NewVal = isNEONModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), 6031 VMovVT, false, VMOVModImm); 6032 if (NewVal != SDValue()) { 6033 SDLoc DL(Op); 6034 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT, 6035 NewVal); 6036 if (IsDouble) 6037 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant); 6038 6039 // It's a float: cast and extract a vector element. 6040 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, 6041 VecConstant); 6042 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant, 6043 DAG.getConstant(0, DL, MVT::i32)); 6044 } 6045 6046 // Finally, try a VMVN.i32 6047 NewVal = isNEONModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT, 6048 false, VMVNModImm); 6049 if (NewVal != SDValue()) { 6050 SDLoc DL(Op); 6051 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal); 6052 6053 if (IsDouble) 6054 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant); 6055 6056 // It's a float: cast and extract a vector element. 6057 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, 6058 VecConstant); 6059 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant, 6060 DAG.getConstant(0, DL, MVT::i32)); 6061 } 6062 6063 return SDValue(); 6064 } 6065 6066 // check if an VEXT instruction can handle the shuffle mask when the 6067 // vector sources of the shuffle are the same. 6068 static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { 6069 unsigned NumElts = VT.getVectorNumElements(); 6070 6071 // Assume that the first shuffle index is not UNDEF. Fail if it is. 6072 if (M[0] < 0) 6073 return false; 6074 6075 Imm = M[0]; 6076 6077 // If this is a VEXT shuffle, the immediate value is the index of the first 6078 // element. The other shuffle indices must be the successive elements after 6079 // the first one. 6080 unsigned ExpectedElt = Imm; 6081 for (unsigned i = 1; i < NumElts; ++i) { 6082 // Increment the expected index. If it wraps around, just follow it 6083 // back to index zero and keep going. 6084 ++ExpectedElt; 6085 if (ExpectedElt == NumElts) 6086 ExpectedElt = 0; 6087 6088 if (M[i] < 0) continue; // ignore UNDEF indices 6089 if (ExpectedElt != static_cast<unsigned>(M[i])) 6090 return false; 6091 } 6092 6093 return true; 6094 } 6095 6096 static bool isVEXTMask(ArrayRef<int> M, EVT VT, 6097 bool &ReverseVEXT, unsigned &Imm) { 6098 unsigned NumElts = VT.getVectorNumElements(); 6099 ReverseVEXT = false; 6100 6101 // Assume that the first shuffle index is not UNDEF. Fail if it is. 6102 if (M[0] < 0) 6103 return false; 6104 6105 Imm = M[0]; 6106 6107 // If this is a VEXT shuffle, the immediate value is the index of the first 6108 // element. The other shuffle indices must be the successive elements after 6109 // the first one. 6110 unsigned ExpectedElt = Imm; 6111 for (unsigned i = 1; i < NumElts; ++i) { 6112 // Increment the expected index. If it wraps around, it may still be 6113 // a VEXT but the source vectors must be swapped. 6114 ExpectedElt += 1; 6115 if (ExpectedElt == NumElts * 2) { 6116 ExpectedElt = 0; 6117 ReverseVEXT = true; 6118 } 6119 6120 if (M[i] < 0) continue; // ignore UNDEF indices 6121 if (ExpectedElt != static_cast<unsigned>(M[i])) 6122 return false; 6123 } 6124 6125 // Adjust the index value if the source operands will be swapped. 6126 if (ReverseVEXT) 6127 Imm -= NumElts; 6128 6129 return true; 6130 } 6131 6132 /// isVREVMask - Check if a vector shuffle corresponds to a VREV 6133 /// instruction with the specified blocksize. (The order of the elements 6134 /// within each block of the vector is reversed.) 6135 static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { 6136 assert((BlockSize==16 || BlockSize==32 || BlockSize==64) && 6137 "Only possible block sizes for VREV are: 16, 32, 64"); 6138 6139 unsigned EltSz = VT.getScalarSizeInBits(); 6140 if (EltSz == 64) 6141 return false; 6142 6143 unsigned NumElts = VT.getVectorNumElements(); 6144 unsigned BlockElts = M[0] + 1; 6145 // If the first shuffle index is UNDEF, be optimistic. 6146 if (M[0] < 0) 6147 BlockElts = BlockSize / EltSz; 6148 6149 if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz) 6150 return false; 6151 6152 for (unsigned i = 0; i < NumElts; ++i) { 6153 if (M[i] < 0) continue; // ignore UNDEF indices 6154 if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts)) 6155 return false; 6156 } 6157 6158 return true; 6159 } 6160 6161 static bool isVTBLMask(ArrayRef<int> M, EVT VT) { 6162 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of 6163 // range, then 0 is placed into the resulting vector. So pretty much any mask 6164 // of 8 elements can work here. 6165 return VT == MVT::v8i8 && M.size() == 8; 6166 } 6167 6168 static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask, 6169 unsigned Index) { 6170 if (Mask.size() == Elements * 2) 6171 return Index / Elements; 6172 return Mask[Index] == 0 ? 0 : 1; 6173 } 6174 6175 // Checks whether the shuffle mask represents a vector transpose (VTRN) by 6176 // checking that pairs of elements in the shuffle mask represent the same index 6177 // in each vector, incrementing the expected index by 2 at each step. 6178 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6] 6179 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g} 6180 // v2={e,f,g,h} 6181 // WhichResult gives the offset for each element in the mask based on which 6182 // of the two results it belongs to. 6183 // 6184 // The transpose can be represented either as: 6185 // result1 = shufflevector v1, v2, result1_shuffle_mask 6186 // result2 = shufflevector v1, v2, result2_shuffle_mask 6187 // where v1/v2 and the shuffle masks have the same number of elements 6188 // (here WhichResult (see below) indicates which result is being checked) 6189 // 6190 // or as: 6191 // results = shufflevector v1, v2, shuffle_mask 6192 // where both results are returned in one vector and the shuffle mask has twice 6193 // as many elements as v1/v2 (here WhichResult will always be 0 if true) here we 6194 // want to check the low half and high half of the shuffle mask as if it were 6195 // the other case 6196 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 6197 unsigned EltSz = VT.getScalarSizeInBits(); 6198 if (EltSz == 64) 6199 return false; 6200 6201 unsigned NumElts = VT.getVectorNumElements(); 6202 if (M.size() != NumElts && M.size() != NumElts*2) 6203 return false; 6204 6205 // If the mask is twice as long as the input vector then we need to check the 6206 // upper and lower parts of the mask with a matching value for WhichResult 6207 // FIXME: A mask with only even values will be rejected in case the first 6208 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only 6209 // M[0] is used to determine WhichResult 6210 for (unsigned i = 0; i < M.size(); i += NumElts) { 6211 WhichResult = SelectPairHalf(NumElts, M, i); 6212 for (unsigned j = 0; j < NumElts; j += 2) { 6213 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) || 6214 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult)) 6215 return false; 6216 } 6217 } 6218 6219 if (M.size() == NumElts*2) 6220 WhichResult = 0; 6221 6222 return true; 6223 } 6224 6225 /// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of 6226 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 6227 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>. 6228 static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ 6229 unsigned EltSz = VT.getScalarSizeInBits(); 6230 if (EltSz == 64) 6231 return false; 6232 6233 unsigned NumElts = VT.getVectorNumElements(); 6234 if (M.size() != NumElts && M.size() != NumElts*2) 6235 return false; 6236 6237 for (unsigned i = 0; i < M.size(); i += NumElts) { 6238 WhichResult = SelectPairHalf(NumElts, M, i); 6239 for (unsigned j = 0; j < NumElts; j += 2) { 6240 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) || 6241 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult)) 6242 return false; 6243 } 6244 } 6245 6246 if (M.size() == NumElts*2) 6247 WhichResult = 0; 6248 6249 return true; 6250 } 6251 6252 // Checks whether the shuffle mask represents a vector unzip (VUZP) by checking 6253 // that the mask elements are either all even and in steps of size 2 or all odd 6254 // and in steps of size 2. 6255 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6] 6256 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g} 6257 // v2={e,f,g,h} 6258 // Requires similar checks to that of isVTRNMask with 6259 // respect the how results are returned. 6260 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 6261 unsigned EltSz = VT.getScalarSizeInBits(); 6262 if (EltSz == 64) 6263 return false; 6264 6265 unsigned NumElts = VT.getVectorNumElements(); 6266 if (M.size() != NumElts && M.size() != NumElts*2) 6267 return false; 6268 6269 for (unsigned i = 0; i < M.size(); i += NumElts) { 6270 WhichResult = SelectPairHalf(NumElts, M, i); 6271 for (unsigned j = 0; j < NumElts; ++j) { 6272 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult) 6273 return false; 6274 } 6275 } 6276 6277 if (M.size() == NumElts*2) 6278 WhichResult = 0; 6279 6280 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6281 if (VT.is64BitVector() && EltSz == 32) 6282 return false; 6283 6284 return true; 6285 } 6286 6287 /// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of 6288 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 6289 /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>, 6290 static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ 6291 unsigned EltSz = VT.getScalarSizeInBits(); 6292 if (EltSz == 64) 6293 return false; 6294 6295 unsigned NumElts = VT.getVectorNumElements(); 6296 if (M.size() != NumElts && M.size() != NumElts*2) 6297 return false; 6298 6299 unsigned Half = NumElts / 2; 6300 for (unsigned i = 0; i < M.size(); i += NumElts) { 6301 WhichResult = SelectPairHalf(NumElts, M, i); 6302 for (unsigned j = 0; j < NumElts; j += Half) { 6303 unsigned Idx = WhichResult; 6304 for (unsigned k = 0; k < Half; ++k) { 6305 int MIdx = M[i + j + k]; 6306 if (MIdx >= 0 && (unsigned) MIdx != Idx) 6307 return false; 6308 Idx += 2; 6309 } 6310 } 6311 } 6312 6313 if (M.size() == NumElts*2) 6314 WhichResult = 0; 6315 6316 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6317 if (VT.is64BitVector() && EltSz == 32) 6318 return false; 6319 6320 return true; 6321 } 6322 6323 // Checks whether the shuffle mask represents a vector zip (VZIP) by checking 6324 // that pairs of elements of the shufflemask represent the same index in each 6325 // vector incrementing sequentially through the vectors. 6326 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5] 6327 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f} 6328 // v2={e,f,g,h} 6329 // Requires similar checks to that of isVTRNMask with respect the how results 6330 // are returned. 6331 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 6332 unsigned EltSz = VT.getScalarSizeInBits(); 6333 if (EltSz == 64) 6334 return false; 6335 6336 unsigned NumElts = VT.getVectorNumElements(); 6337 if (M.size() != NumElts && M.size() != NumElts*2) 6338 return false; 6339 6340 for (unsigned i = 0; i < M.size(); i += NumElts) { 6341 WhichResult = SelectPairHalf(NumElts, M, i); 6342 unsigned Idx = WhichResult * NumElts / 2; 6343 for (unsigned j = 0; j < NumElts; j += 2) { 6344 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) || 6345 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts)) 6346 return false; 6347 Idx += 1; 6348 } 6349 } 6350 6351 if (M.size() == NumElts*2) 6352 WhichResult = 0; 6353 6354 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6355 if (VT.is64BitVector() && EltSz == 32) 6356 return false; 6357 6358 return true; 6359 } 6360 6361 /// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of 6362 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 6363 /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>. 6364 static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ 6365 unsigned EltSz = VT.getScalarSizeInBits(); 6366 if (EltSz == 64) 6367 return false; 6368 6369 unsigned NumElts = VT.getVectorNumElements(); 6370 if (M.size() != NumElts && M.size() != NumElts*2) 6371 return false; 6372 6373 for (unsigned i = 0; i < M.size(); i += NumElts) { 6374 WhichResult = SelectPairHalf(NumElts, M, i); 6375 unsigned Idx = WhichResult * NumElts / 2; 6376 for (unsigned j = 0; j < NumElts; j += 2) { 6377 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) || 6378 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx)) 6379 return false; 6380 Idx += 1; 6381 } 6382 } 6383 6384 if (M.size() == NumElts*2) 6385 WhichResult = 0; 6386 6387 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. 6388 if (VT.is64BitVector() && EltSz == 32) 6389 return false; 6390 6391 return true; 6392 } 6393 6394 /// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), 6395 /// and return the corresponding ARMISD opcode if it is, or 0 if it isn't. 6396 static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT, 6397 unsigned &WhichResult, 6398 bool &isV_UNDEF) { 6399 isV_UNDEF = false; 6400 if (isVTRNMask(ShuffleMask, VT, WhichResult)) 6401 return ARMISD::VTRN; 6402 if (isVUZPMask(ShuffleMask, VT, WhichResult)) 6403 return ARMISD::VUZP; 6404 if (isVZIPMask(ShuffleMask, VT, WhichResult)) 6405 return ARMISD::VZIP; 6406 6407 isV_UNDEF = true; 6408 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) 6409 return ARMISD::VTRN; 6410 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) 6411 return ARMISD::VUZP; 6412 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) 6413 return ARMISD::VZIP; 6414 6415 return 0; 6416 } 6417 6418 /// \return true if this is a reverse operation on an vector. 6419 static bool isReverseMask(ArrayRef<int> M, EVT VT) { 6420 unsigned NumElts = VT.getVectorNumElements(); 6421 // Make sure the mask has the right size. 6422 if (NumElts != M.size()) 6423 return false; 6424 6425 // Look for <15, ..., 3, -1, 1, 0>. 6426 for (unsigned i = 0; i != NumElts; ++i) 6427 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i)) 6428 return false; 6429 6430 return true; 6431 } 6432 6433 // If N is an integer constant that can be moved into a register in one 6434 // instruction, return an SDValue of such a constant (will become a MOV 6435 // instruction). Otherwise return null. 6436 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, 6437 const ARMSubtarget *ST, const SDLoc &dl) { 6438 uint64_t Val; 6439 if (!isa<ConstantSDNode>(N)) 6440 return SDValue(); 6441 Val = cast<ConstantSDNode>(N)->getZExtValue(); 6442 6443 if (ST->isThumb1Only()) { 6444 if (Val <= 255 || ~Val <= 255) 6445 return DAG.getConstant(Val, dl, MVT::i32); 6446 } else { 6447 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1) 6448 return DAG.getConstant(Val, dl, MVT::i32); 6449 } 6450 return SDValue(); 6451 } 6452 6453 // If this is a case we can't handle, return null and let the default 6454 // expansion code take care of it. 6455 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, 6456 const ARMSubtarget *ST) const { 6457 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); 6458 SDLoc dl(Op); 6459 EVT VT = Op.getValueType(); 6460 6461 APInt SplatBits, SplatUndef; 6462 unsigned SplatBitSize; 6463 bool HasAnyUndefs; 6464 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { 6465 if (SplatUndef.isAllOnesValue()) 6466 return DAG.getUNDEF(VT); 6467 6468 if (SplatBitSize <= 64) { 6469 // Check if an immediate VMOV works. 6470 EVT VmovVT; 6471 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(), 6472 SplatUndef.getZExtValue(), SplatBitSize, 6473 DAG, dl, VmovVT, VT.is128BitVector(), 6474 VMOVModImm); 6475 if (Val.getNode()) { 6476 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val); 6477 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); 6478 } 6479 6480 // Try an immediate VMVN. 6481 uint64_t NegatedImm = (~SplatBits).getZExtValue(); 6482 Val = isNEONModifiedImm(NegatedImm, 6483 SplatUndef.getZExtValue(), SplatBitSize, 6484 DAG, dl, VmovVT, VT.is128BitVector(), 6485 VMVNModImm); 6486 if (Val.getNode()) { 6487 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val); 6488 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); 6489 } 6490 6491 // Use vmov.f32 to materialize other v2f32 and v4f32 splats. 6492 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) { 6493 int ImmVal = ARM_AM::getFP32Imm(SplatBits); 6494 if (ImmVal != -1) { 6495 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32); 6496 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val); 6497 } 6498 } 6499 } 6500 } 6501 6502 // Scan through the operands to see if only one value is used. 6503 // 6504 // As an optimisation, even if more than one value is used it may be more 6505 // profitable to splat with one value then change some lanes. 6506 // 6507 // Heuristically we decide to do this if the vector has a "dominant" value, 6508 // defined as splatted to more than half of the lanes. 6509 unsigned NumElts = VT.getVectorNumElements(); 6510 bool isOnlyLowElement = true; 6511 bool usesOnlyOneValue = true; 6512 bool hasDominantValue = false; 6513 bool isConstant = true; 6514 6515 // Map of the number of times a particular SDValue appears in the 6516 // element list. 6517 DenseMap<SDValue, unsigned> ValueCounts; 6518 SDValue Value; 6519 for (unsigned i = 0; i < NumElts; ++i) { 6520 SDValue V = Op.getOperand(i); 6521 if (V.isUndef()) 6522 continue; 6523 if (i > 0) 6524 isOnlyLowElement = false; 6525 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 6526 isConstant = false; 6527 6528 ValueCounts.insert(std::make_pair(V, 0)); 6529 unsigned &Count = ValueCounts[V]; 6530 6531 // Is this value dominant? (takes up more than half of the lanes) 6532 if (++Count > (NumElts / 2)) { 6533 hasDominantValue = true; 6534 Value = V; 6535 } 6536 } 6537 if (ValueCounts.size() != 1) 6538 usesOnlyOneValue = false; 6539 if (!Value.getNode() && !ValueCounts.empty()) 6540 Value = ValueCounts.begin()->first; 6541 6542 if (ValueCounts.empty()) 6543 return DAG.getUNDEF(VT); 6544 6545 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR. 6546 // Keep going if we are hitting this case. 6547 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode())) 6548 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); 6549 6550 unsigned EltSize = VT.getScalarSizeInBits(); 6551 6552 // Use VDUP for non-constant splats. For f32 constant splats, reduce to 6553 // i32 and try again. 6554 if (hasDominantValue && EltSize <= 32) { 6555 if (!isConstant) { 6556 SDValue N; 6557 6558 // If we are VDUPing a value that comes directly from a vector, that will 6559 // cause an unnecessary move to and from a GPR, where instead we could 6560 // just use VDUPLANE. We can only do this if the lane being extracted 6561 // is at a constant index, as the VDUP from lane instructions only have 6562 // constant-index forms. 6563 ConstantSDNode *constIndex; 6564 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT && 6565 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) { 6566 // We need to create a new undef vector to use for the VDUPLANE if the 6567 // size of the vector from which we get the value is different than the 6568 // size of the vector that we need to create. We will insert the element 6569 // such that the register coalescer will remove unnecessary copies. 6570 if (VT != Value->getOperand(0).getValueType()) { 6571 unsigned index = constIndex->getAPIntValue().getLimitedValue() % 6572 VT.getVectorNumElements(); 6573 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, 6574 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT), 6575 Value, DAG.getConstant(index, dl, MVT::i32)), 6576 DAG.getConstant(index, dl, MVT::i32)); 6577 } else 6578 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, 6579 Value->getOperand(0), Value->getOperand(1)); 6580 } else 6581 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value); 6582 6583 if (!usesOnlyOneValue) { 6584 // The dominant value was splatted as 'N', but we now have to insert 6585 // all differing elements. 6586 for (unsigned I = 0; I < NumElts; ++I) { 6587 if (Op.getOperand(I) == Value) 6588 continue; 6589 SmallVector<SDValue, 3> Ops; 6590 Ops.push_back(N); 6591 Ops.push_back(Op.getOperand(I)); 6592 Ops.push_back(DAG.getConstant(I, dl, MVT::i32)); 6593 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops); 6594 } 6595 } 6596 return N; 6597 } 6598 if (VT.getVectorElementType().isFloatingPoint()) { 6599 SmallVector<SDValue, 8> Ops; 6600 for (unsigned i = 0; i < NumElts; ++i) 6601 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, MVT::i32, 6602 Op.getOperand(i))); 6603 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts); 6604 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops); 6605 Val = LowerBUILD_VECTOR(Val, DAG, ST); 6606 if (Val.getNode()) 6607 return DAG.getNode(ISD::BITCAST, dl, VT, Val); 6608 } 6609 if (usesOnlyOneValue) { 6610 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl); 6611 if (isConstant && Val.getNode()) 6612 return DAG.getNode(ARMISD::VDUP, dl, VT, Val); 6613 } 6614 } 6615 6616 // If all elements are constants and the case above didn't get hit, fall back 6617 // to the default expansion, which will generate a load from the constant 6618 // pool. 6619 if (isConstant) 6620 return SDValue(); 6621 6622 // Empirical tests suggest this is rarely worth it for vectors of length <= 2. 6623 if (NumElts >= 4) { 6624 SDValue shuffle = ReconstructShuffle(Op, DAG); 6625 if (shuffle != SDValue()) 6626 return shuffle; 6627 } 6628 6629 if (VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) { 6630 // If we haven't found an efficient lowering, try splitting a 128-bit vector 6631 // into two 64-bit vectors; we might discover a better way to lower it. 6632 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts); 6633 EVT ExtVT = VT.getVectorElementType(); 6634 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2); 6635 SDValue Lower = 6636 DAG.getBuildVector(HVT, dl, makeArrayRef(&Ops[0], NumElts / 2)); 6637 if (Lower.getOpcode() == ISD::BUILD_VECTOR) 6638 Lower = LowerBUILD_VECTOR(Lower, DAG, ST); 6639 SDValue Upper = DAG.getBuildVector( 6640 HVT, dl, makeArrayRef(&Ops[NumElts / 2], NumElts / 2)); 6641 if (Upper.getOpcode() == ISD::BUILD_VECTOR) 6642 Upper = LowerBUILD_VECTOR(Upper, DAG, ST); 6643 if (Lower && Upper) 6644 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper); 6645 } 6646 6647 // Vectors with 32- or 64-bit elements can be built by directly assigning 6648 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands 6649 // will be legalized. 6650 if (EltSize >= 32) { 6651 // Do the expansion with floating-point types, since that is what the VFP 6652 // registers are defined to use, and since i64 is not legal. 6653 EVT EltVT = EVT::getFloatingPointVT(EltSize); 6654 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts); 6655 SmallVector<SDValue, 8> Ops; 6656 for (unsigned i = 0; i < NumElts; ++i) 6657 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i))); 6658 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops); 6659 return DAG.getNode(ISD::BITCAST, dl, VT, Val); 6660 } 6661 6662 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we 6663 // know the default expansion would otherwise fall back on something even 6664 // worse. For a vector with one or two non-undef values, that's 6665 // scalar_to_vector for the elements followed by a shuffle (provided the 6666 // shuffle is valid for the target) and materialization element by element 6667 // on the stack followed by a load for everything else. 6668 if (!isConstant && !usesOnlyOneValue) { 6669 SDValue Vec = DAG.getUNDEF(VT); 6670 for (unsigned i = 0 ; i < NumElts; ++i) { 6671 SDValue V = Op.getOperand(i); 6672 if (V.isUndef()) 6673 continue; 6674 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32); 6675 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); 6676 } 6677 return Vec; 6678 } 6679 6680 return SDValue(); 6681 } 6682 6683 // Gather data to see if the operation can be modelled as a 6684 // shuffle in combination with VEXTs. 6685 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op, 6686 SelectionDAG &DAG) const { 6687 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); 6688 SDLoc dl(Op); 6689 EVT VT = Op.getValueType(); 6690 unsigned NumElts = VT.getVectorNumElements(); 6691 6692 struct ShuffleSourceInfo { 6693 SDValue Vec; 6694 unsigned MinElt = std::numeric_limits<unsigned>::max(); 6695 unsigned MaxElt = 0; 6696 6697 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to 6698 // be compatible with the shuffle we intend to construct. As a result 6699 // ShuffleVec will be some sliding window into the original Vec. 6700 SDValue ShuffleVec; 6701 6702 // Code should guarantee that element i in Vec starts at element "WindowBase 6703 // + i * WindowScale in ShuffleVec". 6704 int WindowBase = 0; 6705 int WindowScale = 1; 6706 6707 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {} 6708 6709 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; } 6710 }; 6711 6712 // First gather all vectors used as an immediate source for this BUILD_VECTOR 6713 // node. 6714 SmallVector<ShuffleSourceInfo, 2> Sources; 6715 for (unsigned i = 0; i < NumElts; ++i) { 6716 SDValue V = Op.getOperand(i); 6717 if (V.isUndef()) 6718 continue; 6719 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) { 6720 // A shuffle can only come from building a vector from various 6721 // elements of other vectors. 6722 return SDValue(); 6723 } else if (!isa<ConstantSDNode>(V.getOperand(1))) { 6724 // Furthermore, shuffles require a constant mask, whereas extractelts 6725 // accept variable indices. 6726 return SDValue(); 6727 } 6728 6729 // Add this element source to the list if it's not already there. 6730 SDValue SourceVec = V.getOperand(0); 6731 auto Source = llvm::find(Sources, SourceVec); 6732 if (Source == Sources.end()) 6733 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec)); 6734 6735 // Update the minimum and maximum lane number seen. 6736 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue(); 6737 Source->MinElt = std::min(Source->MinElt, EltNo); 6738 Source->MaxElt = std::max(Source->MaxElt, EltNo); 6739 } 6740 6741 // Currently only do something sane when at most two source vectors 6742 // are involved. 6743 if (Sources.size() > 2) 6744 return SDValue(); 6745 6746 // Find out the smallest element size among result and two sources, and use 6747 // it as element size to build the shuffle_vector. 6748 EVT SmallestEltTy = VT.getVectorElementType(); 6749 for (auto &Source : Sources) { 6750 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType(); 6751 if (SrcEltTy.bitsLT(SmallestEltTy)) 6752 SmallestEltTy = SrcEltTy; 6753 } 6754 unsigned ResMultiplier = 6755 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits(); 6756 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); 6757 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts); 6758 6759 // If the source vector is too wide or too narrow, we may nevertheless be able 6760 // to construct a compatible shuffle either by concatenating it with UNDEF or 6761 // extracting a suitable range of elements. 6762 for (auto &Src : Sources) { 6763 EVT SrcVT = Src.ShuffleVec.getValueType(); 6764 6765 if (SrcVT.getSizeInBits() == VT.getSizeInBits()) 6766 continue; 6767 6768 // This stage of the search produces a source with the same element type as 6769 // the original, but with a total width matching the BUILD_VECTOR output. 6770 EVT EltVT = SrcVT.getVectorElementType(); 6771 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits(); 6772 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts); 6773 6774 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) { 6775 if (2 * SrcVT.getSizeInBits() != VT.getSizeInBits()) 6776 return SDValue(); 6777 // We can pad out the smaller vector for free, so if it's part of a 6778 // shuffle... 6779 Src.ShuffleVec = 6780 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, 6781 DAG.getUNDEF(Src.ShuffleVec.getValueType())); 6782 continue; 6783 } 6784 6785 if (SrcVT.getSizeInBits() != 2 * VT.getSizeInBits()) 6786 return SDValue(); 6787 6788 if (Src.MaxElt - Src.MinElt >= NumSrcElts) { 6789 // Span too large for a VEXT to cope 6790 return SDValue(); 6791 } 6792 6793 if (Src.MinElt >= NumSrcElts) { 6794 // The extraction can just take the second half 6795 Src.ShuffleVec = 6796 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6797 DAG.getConstant(NumSrcElts, dl, MVT::i32)); 6798 Src.WindowBase = -NumSrcElts; 6799 } else if (Src.MaxElt < NumSrcElts) { 6800 // The extraction can just take the first half 6801 Src.ShuffleVec = 6802 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6803 DAG.getConstant(0, dl, MVT::i32)); 6804 } else { 6805 // An actual VEXT is needed 6806 SDValue VEXTSrc1 = 6807 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6808 DAG.getConstant(0, dl, MVT::i32)); 6809 SDValue VEXTSrc2 = 6810 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 6811 DAG.getConstant(NumSrcElts, dl, MVT::i32)); 6812 6813 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1, 6814 VEXTSrc2, 6815 DAG.getConstant(Src.MinElt, dl, MVT::i32)); 6816 Src.WindowBase = -Src.MinElt; 6817 } 6818 } 6819 6820 // Another possible incompatibility occurs from the vector element types. We 6821 // can fix this by bitcasting the source vectors to the same type we intend 6822 // for the shuffle. 6823 for (auto &Src : Sources) { 6824 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType(); 6825 if (SrcEltTy == SmallestEltTy) 6826 continue; 6827 assert(ShuffleVT.getVectorElementType() == SmallestEltTy); 6828 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); 6829 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits(); 6830 Src.WindowBase *= Src.WindowScale; 6831 } 6832 6833 // Final sanity check before we try to actually produce a shuffle. 6834 LLVM_DEBUG(for (auto Src 6835 : Sources) 6836 assert(Src.ShuffleVec.getValueType() == ShuffleVT);); 6837 6838 // The stars all align, our next step is to produce the mask for the shuffle. 6839 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1); 6840 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits(); 6841 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 6842 SDValue Entry = Op.getOperand(i); 6843 if (Entry.isUndef()) 6844 continue; 6845 6846 auto Src = llvm::find(Sources, Entry.getOperand(0)); 6847 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue(); 6848 6849 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit 6850 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this 6851 // segment. 6852 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType(); 6853 int BitsDefined = std::min(OrigEltTy.getSizeInBits(), 6854 VT.getScalarSizeInBits()); 6855 int LanesDefined = BitsDefined / BitsPerShuffleLane; 6856 6857 // This source is expected to fill ResMultiplier lanes of the final shuffle, 6858 // starting at the appropriate offset. 6859 int *LaneMask = &Mask[i * ResMultiplier]; 6860 6861 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase; 6862 ExtractBase += NumElts * (Src - Sources.begin()); 6863 for (int j = 0; j < LanesDefined; ++j) 6864 LaneMask[j] = ExtractBase + j; 6865 } 6866 6867 // Final check before we try to produce nonsense... 6868 if (!isShuffleMaskLegal(Mask, ShuffleVT)) 6869 return SDValue(); 6870 6871 // We can't handle more than two sources. This should have already 6872 // been checked before this point. 6873 assert(Sources.size() <= 2 && "Too many sources!"); 6874 6875 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) }; 6876 for (unsigned i = 0; i < Sources.size(); ++i) 6877 ShuffleOps[i] = Sources[i].ShuffleVec; 6878 6879 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0], 6880 ShuffleOps[1], Mask); 6881 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); 6882 } 6883 6884 /// isShuffleMaskLegal - Targets can use this to indicate that they only 6885 /// support *some* VECTOR_SHUFFLE operations, those with specific masks. 6886 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 6887 /// are assumed to be legal. 6888 bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { 6889 if (VT.getVectorNumElements() == 4 && 6890 (VT.is128BitVector() || VT.is64BitVector())) { 6891 unsigned PFIndexes[4]; 6892 for (unsigned i = 0; i != 4; ++i) { 6893 if (M[i] < 0) 6894 PFIndexes[i] = 8; 6895 else 6896 PFIndexes[i] = M[i]; 6897 } 6898 6899 // Compute the index in the perfect shuffle table. 6900 unsigned PFTableIndex = 6901 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 6902 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 6903 unsigned Cost = (PFEntry >> 30); 6904 6905 if (Cost <= 4) 6906 return true; 6907 } 6908 6909 bool ReverseVEXT, isV_UNDEF; 6910 unsigned Imm, WhichResult; 6911 6912 unsigned EltSize = VT.getScalarSizeInBits(); 6913 return (EltSize >= 32 || 6914 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || 6915 isVREVMask(M, VT, 64) || 6916 isVREVMask(M, VT, 32) || 6917 isVREVMask(M, VT, 16) || 6918 isVEXTMask(M, VT, ReverseVEXT, Imm) || 6919 isVTBLMask(M, VT) || 6920 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF) || 6921 ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(M, VT))); 6922 } 6923 6924 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 6925 /// the specified operations to build the shuffle. 6926 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 6927 SDValue RHS, SelectionDAG &DAG, 6928 const SDLoc &dl) { 6929 unsigned OpNum = (PFEntry >> 26) & 0x0F; 6930 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 6931 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 6932 6933 enum { 6934 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 6935 OP_VREV, 6936 OP_VDUP0, 6937 OP_VDUP1, 6938 OP_VDUP2, 6939 OP_VDUP3, 6940 OP_VEXT1, 6941 OP_VEXT2, 6942 OP_VEXT3, 6943 OP_VUZPL, // VUZP, left result 6944 OP_VUZPR, // VUZP, right result 6945 OP_VZIPL, // VZIP, left result 6946 OP_VZIPR, // VZIP, right result 6947 OP_VTRNL, // VTRN, left result 6948 OP_VTRNR // VTRN, right result 6949 }; 6950 6951 if (OpNum == OP_COPY) { 6952 if (LHSID == (1*9+2)*9+3) return LHS; 6953 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 6954 return RHS; 6955 } 6956 6957 SDValue OpLHS, OpRHS; 6958 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 6959 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 6960 EVT VT = OpLHS.getValueType(); 6961 6962 switch (OpNum) { 6963 default: llvm_unreachable("Unknown shuffle opcode!"); 6964 case OP_VREV: 6965 // VREV divides the vector in half and swaps within the half. 6966 if (VT.getVectorElementType() == MVT::i32 || 6967 VT.getVectorElementType() == MVT::f32) 6968 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS); 6969 // vrev <4 x i16> -> VREV32 6970 if (VT.getVectorElementType() == MVT::i16) 6971 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS); 6972 // vrev <4 x i8> -> VREV16 6973 assert(VT.getVectorElementType() == MVT::i8); 6974 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS); 6975 case OP_VDUP0: 6976 case OP_VDUP1: 6977 case OP_VDUP2: 6978 case OP_VDUP3: 6979 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, 6980 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32)); 6981 case OP_VEXT1: 6982 case OP_VEXT2: 6983 case OP_VEXT3: 6984 return DAG.getNode(ARMISD::VEXT, dl, VT, 6985 OpLHS, OpRHS, 6986 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32)); 6987 case OP_VUZPL: 6988 case OP_VUZPR: 6989 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT), 6990 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL); 6991 case OP_VZIPL: 6992 case OP_VZIPR: 6993 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT), 6994 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL); 6995 case OP_VTRNL: 6996 case OP_VTRNR: 6997 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT), 6998 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL); 6999 } 7000 } 7001 7002 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, 7003 ArrayRef<int> ShuffleMask, 7004 SelectionDAG &DAG) { 7005 // Check to see if we can use the VTBL instruction. 7006 SDValue V1 = Op.getOperand(0); 7007 SDValue V2 = Op.getOperand(1); 7008 SDLoc DL(Op); 7009 7010 SmallVector<SDValue, 8> VTBLMask; 7011 for (ArrayRef<int>::iterator 7012 I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I) 7013 VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32)); 7014 7015 if (V2.getNode()->isUndef()) 7016 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1, 7017 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask)); 7018 7019 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2, 7020 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask)); 7021 } 7022 7023 static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op, 7024 SelectionDAG &DAG) { 7025 SDLoc DL(Op); 7026 SDValue OpLHS = Op.getOperand(0); 7027 EVT VT = OpLHS.getValueType(); 7028 7029 assert((VT == MVT::v8i16 || VT == MVT::v16i8) && 7030 "Expect an v8i16/v16i8 type"); 7031 OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, OpLHS); 7032 // For a v16i8 type: After the VREV, we have got <8, ...15, 8, ..., 0>. Now, 7033 // extract the first 8 bytes into the top double word and the last 8 bytes 7034 // into the bottom double word. The v8i16 case is similar. 7035 unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4; 7036 return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS, 7037 DAG.getConstant(ExtractNum, DL, MVT::i32)); 7038 } 7039 7040 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { 7041 SDValue V1 = Op.getOperand(0); 7042 SDValue V2 = Op.getOperand(1); 7043 SDLoc dl(Op); 7044 EVT VT = Op.getValueType(); 7045 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode()); 7046 7047 // Convert shuffles that are directly supported on NEON to target-specific 7048 // DAG nodes, instead of keeping them as shuffles and matching them again 7049 // during code selection. This is more efficient and avoids the possibility 7050 // of inconsistencies between legalization and selection. 7051 // FIXME: floating-point vectors should be canonicalized to integer vectors 7052 // of the same time so that they get CSEd properly. 7053 ArrayRef<int> ShuffleMask = SVN->getMask(); 7054 7055 unsigned EltSize = VT.getScalarSizeInBits(); 7056 if (EltSize <= 32) { 7057 if (SVN->isSplat()) { 7058 int Lane = SVN->getSplatIndex(); 7059 // If this is undef splat, generate it via "just" vdup, if possible. 7060 if (Lane == -1) Lane = 0; 7061 7062 // Test if V1 is a SCALAR_TO_VECTOR. 7063 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) { 7064 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); 7065 } 7066 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR 7067 // (and probably will turn into a SCALAR_TO_VECTOR once legalization 7068 // reaches it). 7069 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR && 7070 !isa<ConstantSDNode>(V1.getOperand(0))) { 7071 bool IsScalarToVector = true; 7072 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i) 7073 if (!V1.getOperand(i).isUndef()) { 7074 IsScalarToVector = false; 7075 break; 7076 } 7077 if (IsScalarToVector) 7078 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); 7079 } 7080 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1, 7081 DAG.getConstant(Lane, dl, MVT::i32)); 7082 } 7083 7084 bool ReverseVEXT; 7085 unsigned Imm; 7086 if (isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) { 7087 if (ReverseVEXT) 7088 std::swap(V1, V2); 7089 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2, 7090 DAG.getConstant(Imm, dl, MVT::i32)); 7091 } 7092 7093 if (isVREVMask(ShuffleMask, VT, 64)) 7094 return DAG.getNode(ARMISD::VREV64, dl, VT, V1); 7095 if (isVREVMask(ShuffleMask, VT, 32)) 7096 return DAG.getNode(ARMISD::VREV32, dl, VT, V1); 7097 if (isVREVMask(ShuffleMask, VT, 16)) 7098 return DAG.getNode(ARMISD::VREV16, dl, VT, V1); 7099 7100 if (V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) { 7101 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1, 7102 DAG.getConstant(Imm, dl, MVT::i32)); 7103 } 7104 7105 // Check for Neon shuffles that modify both input vectors in place. 7106 // If both results are used, i.e., if there are two shuffles with the same 7107 // source operands and with masks corresponding to both results of one of 7108 // these operations, DAG memoization will ensure that a single node is 7109 // used for both shuffles. 7110 unsigned WhichResult; 7111 bool isV_UNDEF; 7112 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask( 7113 ShuffleMask, VT, WhichResult, isV_UNDEF)) { 7114 if (isV_UNDEF) 7115 V2 = V1; 7116 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2) 7117 .getValue(WhichResult); 7118 } 7119 7120 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize 7121 // shuffles that produce a result larger than their operands with: 7122 // shuffle(concat(v1, undef), concat(v2, undef)) 7123 // -> 7124 // shuffle(concat(v1, v2), undef) 7125 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine). 7126 // 7127 // This is useful in the general case, but there are special cases where 7128 // native shuffles produce larger results: the two-result ops. 7129 // 7130 // Look through the concat when lowering them: 7131 // shuffle(concat(v1, v2), undef) 7132 // -> 7133 // concat(VZIP(v1, v2):0, :1) 7134 // 7135 if (V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) { 7136 SDValue SubV1 = V1->getOperand(0); 7137 SDValue SubV2 = V1->getOperand(1); 7138 EVT SubVT = SubV1.getValueType(); 7139 7140 // We expect these to have been canonicalized to -1. 7141 assert(llvm::all_of(ShuffleMask, [&](int i) { 7142 return i < (int)VT.getVectorNumElements(); 7143 }) && "Unexpected shuffle index into UNDEF operand!"); 7144 7145 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask( 7146 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) { 7147 if (isV_UNDEF) 7148 SubV2 = SubV1; 7149 assert((WhichResult == 0) && 7150 "In-place shuffle of concat can only have one result!"); 7151 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT), 7152 SubV1, SubV2); 7153 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0), 7154 Res.getValue(1)); 7155 } 7156 } 7157 } 7158 7159 // If the shuffle is not directly supported and it has 4 elements, use 7160 // the PerfectShuffle-generated table to synthesize it from other shuffles. 7161 unsigned NumElts = VT.getVectorNumElements(); 7162 if (NumElts == 4) { 7163 unsigned PFIndexes[4]; 7164 for (unsigned i = 0; i != 4; ++i) { 7165 if (ShuffleMask[i] < 0) 7166 PFIndexes[i] = 8; 7167 else 7168 PFIndexes[i] = ShuffleMask[i]; 7169 } 7170 7171 // Compute the index in the perfect shuffle table. 7172 unsigned PFTableIndex = 7173 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 7174 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 7175 unsigned Cost = (PFEntry >> 30); 7176 7177 if (Cost <= 4) 7178 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 7179 } 7180 7181 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs. 7182 if (EltSize >= 32) { 7183 // Do the expansion with floating-point types, since that is what the VFP 7184 // registers are defined to use, and since i64 is not legal. 7185 EVT EltVT = EVT::getFloatingPointVT(EltSize); 7186 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts); 7187 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1); 7188 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2); 7189 SmallVector<SDValue, 8> Ops; 7190 for (unsigned i = 0; i < NumElts; ++i) { 7191 if (ShuffleMask[i] < 0) 7192 Ops.push_back(DAG.getUNDEF(EltVT)); 7193 else 7194 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 7195 ShuffleMask[i] < (int)NumElts ? V1 : V2, 7196 DAG.getConstant(ShuffleMask[i] & (NumElts-1), 7197 dl, MVT::i32))); 7198 } 7199 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops); 7200 return DAG.getNode(ISD::BITCAST, dl, VT, Val); 7201 } 7202 7203 if ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(ShuffleMask, VT)) 7204 return LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(Op, DAG); 7205 7206 if (VT == MVT::v8i8) 7207 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG)) 7208 return NewOp; 7209 7210 return SDValue(); 7211 } 7212 7213 static SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 7214 // INSERT_VECTOR_ELT is legal only for immediate indexes. 7215 SDValue Lane = Op.getOperand(2); 7216 if (!isa<ConstantSDNode>(Lane)) 7217 return SDValue(); 7218 7219 return Op; 7220 } 7221 7222 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 7223 // EXTRACT_VECTOR_ELT is legal only for immediate indexes. 7224 SDValue Lane = Op.getOperand(1); 7225 if (!isa<ConstantSDNode>(Lane)) 7226 return SDValue(); 7227 7228 SDValue Vec = Op.getOperand(0); 7229 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) { 7230 SDLoc dl(Op); 7231 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane); 7232 } 7233 7234 return Op; 7235 } 7236 7237 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { 7238 // The only time a CONCAT_VECTORS operation can have legal types is when 7239 // two 64-bit vectors are concatenated to a 128-bit vector. 7240 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 && 7241 "unexpected CONCAT_VECTORS"); 7242 SDLoc dl(Op); 7243 SDValue Val = DAG.getUNDEF(MVT::v2f64); 7244 SDValue Op0 = Op.getOperand(0); 7245 SDValue Op1 = Op.getOperand(1); 7246 if (!Op0.isUndef()) 7247 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val, 7248 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0), 7249 DAG.getIntPtrConstant(0, dl)); 7250 if (!Op1.isUndef()) 7251 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val, 7252 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1), 7253 DAG.getIntPtrConstant(1, dl)); 7254 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val); 7255 } 7256 7257 /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each 7258 /// element has been zero/sign-extended, depending on the isSigned parameter, 7259 /// from an integer type half its size. 7260 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, 7261 bool isSigned) { 7262 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32. 7263 EVT VT = N->getValueType(0); 7264 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { 7265 SDNode *BVN = N->getOperand(0).getNode(); 7266 if (BVN->getValueType(0) != MVT::v4i32 || 7267 BVN->getOpcode() != ISD::BUILD_VECTOR) 7268 return false; 7269 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0; 7270 unsigned HiElt = 1 - LoElt; 7271 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt)); 7272 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt)); 7273 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2)); 7274 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2)); 7275 if (!Lo0 || !Hi0 || !Lo1 || !Hi1) 7276 return false; 7277 if (isSigned) { 7278 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 && 7279 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32) 7280 return true; 7281 } else { 7282 if (Hi0->isNullValue() && Hi1->isNullValue()) 7283 return true; 7284 } 7285 return false; 7286 } 7287 7288 if (N->getOpcode() != ISD::BUILD_VECTOR) 7289 return false; 7290 7291 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 7292 SDNode *Elt = N->getOperand(i).getNode(); 7293 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) { 7294 unsigned EltSize = VT.getScalarSizeInBits(); 7295 unsigned HalfSize = EltSize / 2; 7296 if (isSigned) { 7297 if (!isIntN(HalfSize, C->getSExtValue())) 7298 return false; 7299 } else { 7300 if (!isUIntN(HalfSize, C->getZExtValue())) 7301 return false; 7302 } 7303 continue; 7304 } 7305 return false; 7306 } 7307 7308 return true; 7309 } 7310 7311 /// isSignExtended - Check if a node is a vector value that is sign-extended 7312 /// or a constant BUILD_VECTOR with sign-extended elements. 7313 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) { 7314 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N)) 7315 return true; 7316 if (isExtendedBUILD_VECTOR(N, DAG, true)) 7317 return true; 7318 return false; 7319 } 7320 7321 /// isZeroExtended - Check if a node is a vector value that is zero-extended 7322 /// or a constant BUILD_VECTOR with zero-extended elements. 7323 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) { 7324 if (N->getOpcode() == ISD::ZERO_EXTEND || ISD::isZEXTLoad(N)) 7325 return true; 7326 if (isExtendedBUILD_VECTOR(N, DAG, false)) 7327 return true; 7328 return false; 7329 } 7330 7331 static EVT getExtensionTo64Bits(const EVT &OrigVT) { 7332 if (OrigVT.getSizeInBits() >= 64) 7333 return OrigVT; 7334 7335 assert(OrigVT.isSimple() && "Expecting a simple value type"); 7336 7337 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy; 7338 switch (OrigSimpleTy) { 7339 default: llvm_unreachable("Unexpected Vector Type"); 7340 case MVT::v2i8: 7341 case MVT::v2i16: 7342 return MVT::v2i32; 7343 case MVT::v4i8: 7344 return MVT::v4i16; 7345 } 7346 } 7347 7348 /// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total 7349 /// value size to 64 bits. We need a 64-bit D register as an operand to VMULL. 7350 /// We insert the required extension here to get the vector to fill a D register. 7351 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, 7352 const EVT &OrigTy, 7353 const EVT &ExtTy, 7354 unsigned ExtOpcode) { 7355 // The vector originally had a size of OrigTy. It was then extended to ExtTy. 7356 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than 7357 // 64-bits we need to insert a new extension so that it will be 64-bits. 7358 assert(ExtTy.is128BitVector() && "Unexpected extension size"); 7359 if (OrigTy.getSizeInBits() >= 64) 7360 return N; 7361 7362 // Must extend size to at least 64 bits to be used as an operand for VMULL. 7363 EVT NewVT = getExtensionTo64Bits(OrigTy); 7364 7365 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N); 7366 } 7367 7368 /// SkipLoadExtensionForVMULL - return a load of the original vector size that 7369 /// does not do any sign/zero extension. If the original vector is less 7370 /// than 64 bits, an appropriate extension will be added after the load to 7371 /// reach a total size of 64 bits. We have to add the extension separately 7372 /// because ARM does not have a sign/zero extending load for vectors. 7373 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) { 7374 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT()); 7375 7376 // The load already has the right type. 7377 if (ExtendedTy == LD->getMemoryVT()) 7378 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(), 7379 LD->getBasePtr(), LD->getPointerInfo(), 7380 LD->getAlignment(), LD->getMemOperand()->getFlags()); 7381 7382 // We need to create a zextload/sextload. We cannot just create a load 7383 // followed by a zext/zext node because LowerMUL is also run during normal 7384 // operation legalization where we can't create illegal types. 7385 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy, 7386 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(), 7387 LD->getMemoryVT(), LD->getAlignment(), 7388 LD->getMemOperand()->getFlags()); 7389 } 7390 7391 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, 7392 /// extending load, or BUILD_VECTOR with extended elements, return the 7393 /// unextended value. The unextended vector should be 64 bits so that it can 7394 /// be used as an operand to a VMULL instruction. If the original vector size 7395 /// before extension is less than 64 bits we add a an extension to resize 7396 /// the vector to 64 bits. 7397 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) { 7398 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) 7399 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG, 7400 N->getOperand(0)->getValueType(0), 7401 N->getValueType(0), 7402 N->getOpcode()); 7403 7404 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 7405 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) && 7406 "Expected extending load"); 7407 7408 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG); 7409 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1)); 7410 unsigned Opcode = ISD::isSEXTLoad(LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 7411 SDValue extLoad = 7412 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad); 7413 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad); 7414 7415 return newLoad; 7416 } 7417 7418 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will 7419 // have been legalized as a BITCAST from v4i32. 7420 if (N->getOpcode() == ISD::BITCAST) { 7421 SDNode *BVN = N->getOperand(0).getNode(); 7422 assert(BVN->getOpcode() == ISD::BUILD_VECTOR && 7423 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR"); 7424 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0; 7425 return DAG.getBuildVector( 7426 MVT::v2i32, SDLoc(N), 7427 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)}); 7428 } 7429 // Construct a new BUILD_VECTOR with elements truncated to half the size. 7430 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); 7431 EVT VT = N->getValueType(0); 7432 unsigned EltSize = VT.getScalarSizeInBits() / 2; 7433 unsigned NumElts = VT.getVectorNumElements(); 7434 MVT TruncVT = MVT::getIntegerVT(EltSize); 7435 SmallVector<SDValue, 8> Ops; 7436 SDLoc dl(N); 7437 for (unsigned i = 0; i != NumElts; ++i) { 7438 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i)); 7439 const APInt &CInt = C->getAPIntValue(); 7440 // Element types smaller than 32 bits are not legal, so use i32 elements. 7441 // The values are implicitly truncated so sext vs. zext doesn't matter. 7442 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32)); 7443 } 7444 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops); 7445 } 7446 7447 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) { 7448 unsigned Opcode = N->getOpcode(); 7449 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { 7450 SDNode *N0 = N->getOperand(0).getNode(); 7451 SDNode *N1 = N->getOperand(1).getNode(); 7452 return N0->hasOneUse() && N1->hasOneUse() && 7453 isSignExtended(N0, DAG) && isSignExtended(N1, DAG); 7454 } 7455 return false; 7456 } 7457 7458 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) { 7459 unsigned Opcode = N->getOpcode(); 7460 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { 7461 SDNode *N0 = N->getOperand(0).getNode(); 7462 SDNode *N1 = N->getOperand(1).getNode(); 7463 return N0->hasOneUse() && N1->hasOneUse() && 7464 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG); 7465 } 7466 return false; 7467 } 7468 7469 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) { 7470 // Multiplications are only custom-lowered for 128-bit vectors so that 7471 // VMULL can be detected. Otherwise v2i64 multiplications are not legal. 7472 EVT VT = Op.getValueType(); 7473 assert(VT.is128BitVector() && VT.isInteger() && 7474 "unexpected type for custom-lowering ISD::MUL"); 7475 SDNode *N0 = Op.getOperand(0).getNode(); 7476 SDNode *N1 = Op.getOperand(1).getNode(); 7477 unsigned NewOpc = 0; 7478 bool isMLA = false; 7479 bool isN0SExt = isSignExtended(N0, DAG); 7480 bool isN1SExt = isSignExtended(N1, DAG); 7481 if (isN0SExt && isN1SExt) 7482 NewOpc = ARMISD::VMULLs; 7483 else { 7484 bool isN0ZExt = isZeroExtended(N0, DAG); 7485 bool isN1ZExt = isZeroExtended(N1, DAG); 7486 if (isN0ZExt && isN1ZExt) 7487 NewOpc = ARMISD::VMULLu; 7488 else if (isN1SExt || isN1ZExt) { 7489 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these 7490 // into (s/zext A * s/zext C) + (s/zext B * s/zext C) 7491 if (isN1SExt && isAddSubSExt(N0, DAG)) { 7492 NewOpc = ARMISD::VMULLs; 7493 isMLA = true; 7494 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) { 7495 NewOpc = ARMISD::VMULLu; 7496 isMLA = true; 7497 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) { 7498 std::swap(N0, N1); 7499 NewOpc = ARMISD::VMULLu; 7500 isMLA = true; 7501 } 7502 } 7503 7504 if (!NewOpc) { 7505 if (VT == MVT::v2i64) 7506 // Fall through to expand this. It is not legal. 7507 return SDValue(); 7508 else 7509 // Other vector multiplications are legal. 7510 return Op; 7511 } 7512 } 7513 7514 // Legalize to a VMULL instruction. 7515 SDLoc DL(Op); 7516 SDValue Op0; 7517 SDValue Op1 = SkipExtensionForVMULL(N1, DAG); 7518 if (!isMLA) { 7519 Op0 = SkipExtensionForVMULL(N0, DAG); 7520 assert(Op0.getValueType().is64BitVector() && 7521 Op1.getValueType().is64BitVector() && 7522 "unexpected types for extended operands to VMULL"); 7523 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); 7524 } 7525 7526 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during 7527 // isel lowering to take advantage of no-stall back to back vmul + vmla. 7528 // vmull q0, d4, d6 7529 // vmlal q0, d5, d6 7530 // is faster than 7531 // vaddl q0, d4, d5 7532 // vmovl q1, d6 7533 // vmul q0, q0, q1 7534 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG); 7535 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG); 7536 EVT Op1VT = Op1.getValueType(); 7537 return DAG.getNode(N0->getOpcode(), DL, VT, 7538 DAG.getNode(NewOpc, DL, VT, 7539 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), 7540 DAG.getNode(NewOpc, DL, VT, 7541 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); 7542 } 7543 7544 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, 7545 SelectionDAG &DAG) { 7546 // TODO: Should this propagate fast-math-flags? 7547 7548 // Convert to float 7549 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo)); 7550 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo)); 7551 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X); 7552 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y); 7553 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X); 7554 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y); 7555 // Get reciprocal estimate. 7556 // float4 recip = vrecpeq_f32(yf); 7557 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7558 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32), 7559 Y); 7560 // Because char has a smaller range than uchar, we can actually get away 7561 // without any newton steps. This requires that we use a weird bias 7562 // of 0xb000, however (again, this has been exhaustively tested). 7563 // float4 result = as_float4(as_int4(xf*recip) + 0xb000); 7564 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y); 7565 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X); 7566 Y = DAG.getConstant(0xb000, dl, MVT::v4i32); 7567 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y); 7568 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X); 7569 // Convert back to short. 7570 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X); 7571 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X); 7572 return X; 7573 } 7574 7575 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, 7576 SelectionDAG &DAG) { 7577 // TODO: Should this propagate fast-math-flags? 7578 7579 SDValue N2; 7580 // Convert to float. 7581 // float4 yf = vcvt_f32_s32(vmovl_s16(y)); 7582 // float4 xf = vcvt_f32_s32(vmovl_s16(x)); 7583 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0); 7584 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1); 7585 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0); 7586 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1); 7587 7588 // Use reciprocal estimate and one refinement step. 7589 // float4 recip = vrecpeq_f32(yf); 7590 // recip *= vrecpsq_f32(yf, recip); 7591 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7592 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32), 7593 N1); 7594 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7595 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32), 7596 N1, N2); 7597 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); 7598 // Because short has a smaller range than ushort, we can actually get away 7599 // with only a single newton step. This requires that we use a weird bias 7600 // of 89, however (again, this has been exhaustively tested). 7601 // float4 result = as_float4(as_int4(xf*recip) + 0x89); 7602 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2); 7603 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0); 7604 N1 = DAG.getConstant(0x89, dl, MVT::v4i32); 7605 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1); 7606 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0); 7607 // Convert back to integer and return. 7608 // return vmovn_s32(vcvt_s32_f32(result)); 7609 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0); 7610 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0); 7611 return N0; 7612 } 7613 7614 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) { 7615 EVT VT = Op.getValueType(); 7616 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && 7617 "unexpected type for custom-lowering ISD::SDIV"); 7618 7619 SDLoc dl(Op); 7620 SDValue N0 = Op.getOperand(0); 7621 SDValue N1 = Op.getOperand(1); 7622 SDValue N2, N3; 7623 7624 if (VT == MVT::v8i8) { 7625 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0); 7626 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1); 7627 7628 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7629 DAG.getIntPtrConstant(4, dl)); 7630 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7631 DAG.getIntPtrConstant(4, dl)); 7632 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7633 DAG.getIntPtrConstant(0, dl)); 7634 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7635 DAG.getIntPtrConstant(0, dl)); 7636 7637 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16 7638 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16 7639 7640 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2); 7641 N0 = LowerCONCAT_VECTORS(N0, DAG); 7642 7643 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0); 7644 return N0; 7645 } 7646 return LowerSDIV_v4i16(N0, N1, dl, DAG); 7647 } 7648 7649 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) { 7650 // TODO: Should this propagate fast-math-flags? 7651 EVT VT = Op.getValueType(); 7652 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && 7653 "unexpected type for custom-lowering ISD::UDIV"); 7654 7655 SDLoc dl(Op); 7656 SDValue N0 = Op.getOperand(0); 7657 SDValue N1 = Op.getOperand(1); 7658 SDValue N2, N3; 7659 7660 if (VT == MVT::v8i8) { 7661 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0); 7662 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1); 7663 7664 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7665 DAG.getIntPtrConstant(4, dl)); 7666 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7667 DAG.getIntPtrConstant(4, dl)); 7668 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, 7669 DAG.getIntPtrConstant(0, dl)); 7670 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, 7671 DAG.getIntPtrConstant(0, dl)); 7672 7673 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16 7674 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16 7675 7676 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2); 7677 N0 = LowerCONCAT_VECTORS(N0, DAG); 7678 7679 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8, 7680 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl, 7681 MVT::i32), 7682 N0); 7683 return N0; 7684 } 7685 7686 // v4i16 sdiv ... Convert to float. 7687 // float4 yf = vcvt_f32_s32(vmovl_u16(y)); 7688 // float4 xf = vcvt_f32_s32(vmovl_u16(x)); 7689 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0); 7690 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1); 7691 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0); 7692 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1); 7693 7694 // Use reciprocal estimate and two refinement steps. 7695 // float4 recip = vrecpeq_f32(yf); 7696 // recip *= vrecpsq_f32(yf, recip); 7697 // recip *= vrecpsq_f32(yf, recip); 7698 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7699 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32), 7700 BN1); 7701 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7702 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32), 7703 BN1, N2); 7704 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); 7705 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, 7706 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32), 7707 BN1, N2); 7708 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); 7709 // Simply multiplying by the reciprocal estimate can leave us a few ulps 7710 // too low, so we add 2 ulps (exhaustive testing shows that this is enough, 7711 // and that it will never cause us to return an answer too large). 7712 // float4 result = as_float4(as_int4(xf*recip) + 2); 7713 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2); 7714 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0); 7715 N1 = DAG.getConstant(2, dl, MVT::v4i32); 7716 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1); 7717 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0); 7718 // Convert back to integer and return. 7719 // return vmovn_u32(vcvt_s32_f32(result)); 7720 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0); 7721 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0); 7722 return N0; 7723 } 7724 7725 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) { 7726 SDNode *N = Op.getNode(); 7727 EVT VT = N->getValueType(0); 7728 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 7729 7730 SDValue Carry = Op.getOperand(2); 7731 7732 SDLoc DL(Op); 7733 7734 SDValue Result; 7735 if (Op.getOpcode() == ISD::ADDCARRY) { 7736 // This converts the boolean value carry into the carry flag. 7737 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG); 7738 7739 // Do the addition proper using the carry flag we wanted. 7740 Result = DAG.getNode(ARMISD::ADDE, DL, VTs, Op.getOperand(0), 7741 Op.getOperand(1), Carry); 7742 7743 // Now convert the carry flag into a boolean value. 7744 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); 7745 } else { 7746 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we 7747 // have to invert the carry first. 7748 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, 7749 DAG.getConstant(1, DL, MVT::i32), Carry); 7750 // This converts the boolean value carry into the carry flag. 7751 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG); 7752 7753 // Do the subtraction proper using the carry flag we wanted. 7754 Result = DAG.getNode(ARMISD::SUBE, DL, VTs, Op.getOperand(0), 7755 Op.getOperand(1), Carry); 7756 7757 // Now convert the carry flag into a boolean value. 7758 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); 7759 // But the carry returned by ARMISD::SUBE is not a borrow as expected 7760 // by ISD::SUBCARRY, so compute 1 - C. 7761 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, 7762 DAG.getConstant(1, DL, MVT::i32), Carry); 7763 } 7764 7765 // Return both values. 7766 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry); 7767 } 7768 7769 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const { 7770 assert(Subtarget->isTargetDarwin()); 7771 7772 // For iOS, we want to call an alternative entry point: __sincos_stret, 7773 // return values are passed via sret. 7774 SDLoc dl(Op); 7775 SDValue Arg = Op.getOperand(0); 7776 EVT ArgVT = Arg.getValueType(); 7777 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 7778 auto PtrVT = getPointerTy(DAG.getDataLayout()); 7779 7780 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 7781 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7782 7783 // Pair of floats / doubles used to pass the result. 7784 Type *RetTy = StructType::get(ArgTy, ArgTy); 7785 auto &DL = DAG.getDataLayout(); 7786 7787 ArgListTy Args; 7788 bool ShouldUseSRet = Subtarget->isAPCS_ABI(); 7789 SDValue SRet; 7790 if (ShouldUseSRet) { 7791 // Create stack object for sret. 7792 const uint64_t ByteSize = DL.getTypeAllocSize(RetTy); 7793 const unsigned StackAlign = DL.getPrefTypeAlignment(RetTy); 7794 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false); 7795 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL)); 7796 7797 ArgListEntry Entry; 7798 Entry.Node = SRet; 7799 Entry.Ty = RetTy->getPointerTo(); 7800 Entry.IsSExt = false; 7801 Entry.IsZExt = false; 7802 Entry.IsSRet = true; 7803 Args.push_back(Entry); 7804 RetTy = Type::getVoidTy(*DAG.getContext()); 7805 } 7806 7807 ArgListEntry Entry; 7808 Entry.Node = Arg; 7809 Entry.Ty = ArgTy; 7810 Entry.IsSExt = false; 7811 Entry.IsZExt = false; 7812 Args.push_back(Entry); 7813 7814 RTLIB::Libcall LC = 7815 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32; 7816 const char *LibcallName = getLibcallName(LC); 7817 CallingConv::ID CC = getLibcallCallingConv(LC); 7818 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL)); 7819 7820 TargetLowering::CallLoweringInfo CLI(DAG); 7821 CLI.setDebugLoc(dl) 7822 .setChain(DAG.getEntryNode()) 7823 .setCallee(CC, RetTy, Callee, std::move(Args)) 7824 .setDiscardResult(ShouldUseSRet); 7825 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 7826 7827 if (!ShouldUseSRet) 7828 return CallResult.first; 7829 7830 SDValue LoadSin = 7831 DAG.getLoad(ArgVT, dl, CallResult.second, SRet, MachinePointerInfo()); 7832 7833 // Address of cos field. 7834 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet, 7835 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl)); 7836 SDValue LoadCos = 7837 DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add, MachinePointerInfo()); 7838 7839 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT); 7840 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, 7841 LoadSin.getValue(0), LoadCos.getValue(0)); 7842 } 7843 7844 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG, 7845 bool Signed, 7846 SDValue &Chain) const { 7847 EVT VT = Op.getValueType(); 7848 assert((VT == MVT::i32 || VT == MVT::i64) && 7849 "unexpected type for custom lowering DIV"); 7850 SDLoc dl(Op); 7851 7852 const auto &DL = DAG.getDataLayout(); 7853 const auto &TLI = DAG.getTargetLoweringInfo(); 7854 7855 const char *Name = nullptr; 7856 if (Signed) 7857 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64"; 7858 else 7859 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64"; 7860 7861 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL)); 7862 7863 ARMTargetLowering::ArgListTy Args; 7864 7865 for (auto AI : {1, 0}) { 7866 ArgListEntry Arg; 7867 Arg.Node = Op.getOperand(AI); 7868 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext()); 7869 Args.push_back(Arg); 7870 } 7871 7872 CallLoweringInfo CLI(DAG); 7873 CLI.setDebugLoc(dl) 7874 .setChain(Chain) 7875 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()), 7876 ES, std::move(Args)); 7877 7878 return LowerCallTo(CLI).first; 7879 } 7880 7881 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG, 7882 bool Signed) const { 7883 assert(Op.getValueType() == MVT::i32 && 7884 "unexpected type for custom lowering DIV"); 7885 SDLoc dl(Op); 7886 7887 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other, 7888 DAG.getEntryNode(), Op.getOperand(1)); 7889 7890 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK); 7891 } 7892 7893 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) { 7894 SDLoc DL(N); 7895 SDValue Op = N->getOperand(1); 7896 if (N->getValueType(0) == MVT::i32) 7897 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, Op); 7898 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op, 7899 DAG.getConstant(0, DL, MVT::i32)); 7900 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op, 7901 DAG.getConstant(1, DL, MVT::i32)); 7902 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, 7903 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi)); 7904 } 7905 7906 void ARMTargetLowering::ExpandDIV_Windows( 7907 SDValue Op, SelectionDAG &DAG, bool Signed, 7908 SmallVectorImpl<SDValue> &Results) const { 7909 const auto &DL = DAG.getDataLayout(); 7910 const auto &TLI = DAG.getTargetLoweringInfo(); 7911 7912 assert(Op.getValueType() == MVT::i64 && 7913 "unexpected type for custom lowering DIV"); 7914 SDLoc dl(Op); 7915 7916 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode()); 7917 7918 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK); 7919 7920 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result); 7921 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result, 7922 DAG.getConstant(32, dl, TLI.getPointerTy(DL))); 7923 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper); 7924 7925 Results.push_back(Lower); 7926 Results.push_back(Upper); 7927 } 7928 7929 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) { 7930 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering())) 7931 // Acquire/Release load/store is not legal for targets without a dmb or 7932 // equivalent available. 7933 return SDValue(); 7934 7935 // Monotonic load/store is legal for all targets. 7936 return Op; 7937 } 7938 7939 static void ReplaceREADCYCLECOUNTER(SDNode *N, 7940 SmallVectorImpl<SDValue> &Results, 7941 SelectionDAG &DAG, 7942 const ARMSubtarget *Subtarget) { 7943 SDLoc DL(N); 7944 // Under Power Management extensions, the cycle-count is: 7945 // mrc p15, #0, <Rt>, c9, c13, #0 7946 SDValue Ops[] = { N->getOperand(0), // Chain 7947 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32), 7948 DAG.getConstant(15, DL, MVT::i32), 7949 DAG.getConstant(0, DL, MVT::i32), 7950 DAG.getConstant(9, DL, MVT::i32), 7951 DAG.getConstant(13, DL, MVT::i32), 7952 DAG.getConstant(0, DL, MVT::i32) 7953 }; 7954 7955 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, 7956 DAG.getVTList(MVT::i32, MVT::Other), Ops); 7957 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32, 7958 DAG.getConstant(0, DL, MVT::i32))); 7959 Results.push_back(Cycles32.getValue(1)); 7960 } 7961 7962 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) { 7963 SDLoc dl(V.getNode()); 7964 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32); 7965 SDValue VHi = DAG.getAnyExtOrTrunc( 7966 DAG.getNode(ISD::SRL, dl, MVT::i64, V, DAG.getConstant(32, dl, MVT::i32)), 7967 dl, MVT::i32); 7968 bool isBigEndian = DAG.getDataLayout().isBigEndian(); 7969 if (isBigEndian) 7970 std::swap (VLo, VHi); 7971 SDValue RegClass = 7972 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32); 7973 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32); 7974 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32); 7975 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 }; 7976 return SDValue( 7977 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0); 7978 } 7979 7980 static void ReplaceCMP_SWAP_64Results(SDNode *N, 7981 SmallVectorImpl<SDValue> & Results, 7982 SelectionDAG &DAG) { 7983 assert(N->getValueType(0) == MVT::i64 && 7984 "AtomicCmpSwap on types less than 64 should be legal"); 7985 SDValue Ops[] = {N->getOperand(1), 7986 createGPRPairNode(DAG, N->getOperand(2)), 7987 createGPRPairNode(DAG, N->getOperand(3)), 7988 N->getOperand(0)}; 7989 SDNode *CmpSwap = DAG.getMachineNode( 7990 ARM::CMP_SWAP_64, SDLoc(N), 7991 DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops); 7992 7993 MachineFunction &MF = DAG.getMachineFunction(); 7994 MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); 7995 MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); 7996 cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); 7997 7998 bool isBigEndian = DAG.getDataLayout().isBigEndian(); 7999 8000 Results.push_back( 8001 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_1 : ARM::gsub_0, 8002 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0))); 8003 Results.push_back( 8004 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_0 : ARM::gsub_1, 8005 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0))); 8006 Results.push_back(SDValue(CmpSwap, 2)); 8007 } 8008 8009 static SDValue LowerFPOWI(SDValue Op, const ARMSubtarget &Subtarget, 8010 SelectionDAG &DAG) { 8011 const auto &TLI = DAG.getTargetLoweringInfo(); 8012 8013 assert(Subtarget.getTargetTriple().isOSMSVCRT() && 8014 "Custom lowering is MSVCRT specific!"); 8015 8016 SDLoc dl(Op); 8017 SDValue Val = Op.getOperand(0); 8018 MVT Ty = Val->getSimpleValueType(0); 8019 SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, dl, Ty, Op.getOperand(1)); 8020 SDValue Callee = DAG.getExternalSymbol(Ty == MVT::f32 ? "powf" : "pow", 8021 TLI.getPointerTy(DAG.getDataLayout())); 8022 8023 TargetLowering::ArgListTy Args; 8024 TargetLowering::ArgListEntry Entry; 8025 8026 Entry.Node = Val; 8027 Entry.Ty = Val.getValueType().getTypeForEVT(*DAG.getContext()); 8028 Entry.IsZExt = true; 8029 Args.push_back(Entry); 8030 8031 Entry.Node = Exponent; 8032 Entry.Ty = Exponent.getValueType().getTypeForEVT(*DAG.getContext()); 8033 Entry.IsZExt = true; 8034 Args.push_back(Entry); 8035 8036 Type *LCRTy = Val.getValueType().getTypeForEVT(*DAG.getContext()); 8037 8038 // In the in-chain to the call is the entry node If we are emitting a 8039 // tailcall, the chain will be mutated if the node has a non-entry input 8040 // chain. 8041 SDValue InChain = DAG.getEntryNode(); 8042 SDValue TCChain = InChain; 8043 8044 const Function &F = DAG.getMachineFunction().getFunction(); 8045 bool IsTC = TLI.isInTailCallPosition(DAG, Op.getNode(), TCChain) && 8046 F.getReturnType() == LCRTy; 8047 if (IsTC) 8048 InChain = TCChain; 8049 8050 TargetLowering::CallLoweringInfo CLI(DAG); 8051 CLI.setDebugLoc(dl) 8052 .setChain(InChain) 8053 .setCallee(CallingConv::ARM_AAPCS_VFP, LCRTy, Callee, std::move(Args)) 8054 .setTailCall(IsTC); 8055 std::pair<SDValue, SDValue> CI = TLI.LowerCallTo(CLI); 8056 8057 // Return the chain (the DAG root) if it is a tail call 8058 return !CI.second.getNode() ? DAG.getRoot() : CI.first; 8059 } 8060 8061 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 8062 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump()); 8063 switch (Op.getOpcode()) { 8064 default: llvm_unreachable("Don't know how to custom lower this!"); 8065 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG); 8066 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 8067 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 8068 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 8069 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 8070 case ISD::SELECT: return LowerSELECT(Op, DAG); 8071 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 8072 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 8073 case ISD::BR_CC: return LowerBR_CC(Op, DAG); 8074 case ISD::BR_JT: return LowerBR_JT(Op, DAG); 8075 case ISD::VASTART: return LowerVASTART(Op, DAG); 8076 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget); 8077 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget); 8078 case ISD::SINT_TO_FP: 8079 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 8080 case ISD::FP_TO_SINT: 8081 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); 8082 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 8083 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 8084 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 8085 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG); 8086 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG); 8087 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG); 8088 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG, 8089 Subtarget); 8090 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget); 8091 case ISD::SHL: 8092 case ISD::SRL: 8093 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget); 8094 case ISD::SREM: return LowerREM(Op.getNode(), DAG); 8095 case ISD::UREM: return LowerREM(Op.getNode(), DAG); 8096 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG); 8097 case ISD::SRL_PARTS: 8098 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG); 8099 case ISD::CTTZ: 8100 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget); 8101 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget); 8102 case ISD::SETCC: return LowerVSETCC(Op, DAG); 8103 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG); 8104 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget); 8105 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget); 8106 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 8107 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 8108 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 8109 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 8110 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 8111 case ISD::MUL: return LowerMUL(Op, DAG); 8112 case ISD::SDIV: 8113 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) 8114 return LowerDIV_Windows(Op, DAG, /* Signed */ true); 8115 return LowerSDIV(Op, DAG); 8116 case ISD::UDIV: 8117 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) 8118 return LowerDIV_Windows(Op, DAG, /* Signed */ false); 8119 return LowerUDIV(Op, DAG); 8120 case ISD::ADDCARRY: 8121 case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG); 8122 case ISD::SADDO: 8123 case ISD::SSUBO: 8124 return LowerSignedALUO(Op, DAG); 8125 case ISD::UADDO: 8126 case ISD::USUBO: 8127 return LowerUnsignedALUO(Op, DAG); 8128 case ISD::ATOMIC_LOAD: 8129 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG); 8130 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG); 8131 case ISD::SDIVREM: 8132 case ISD::UDIVREM: return LowerDivRem(Op, DAG); 8133 case ISD::DYNAMIC_STACKALLOC: 8134 if (Subtarget->isTargetWindows()) 8135 return LowerDYNAMIC_STACKALLOC(Op, DAG); 8136 llvm_unreachable("Don't know how to custom lower this!"); 8137 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG); 8138 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); 8139 case ISD::FPOWI: return LowerFPOWI(Op, *Subtarget, DAG); 8140 case ARMISD::WIN__DBZCHK: return SDValue(); 8141 } 8142 } 8143 8144 static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results, 8145 SelectionDAG &DAG) { 8146 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 8147 unsigned Opc = 0; 8148 if (IntNo == Intrinsic::arm_smlald) 8149 Opc = ARMISD::SMLALD; 8150 else if (IntNo == Intrinsic::arm_smlaldx) 8151 Opc = ARMISD::SMLALDX; 8152 else if (IntNo == Intrinsic::arm_smlsld) 8153 Opc = ARMISD::SMLSLD; 8154 else if (IntNo == Intrinsic::arm_smlsldx) 8155 Opc = ARMISD::SMLSLDX; 8156 else 8157 return; 8158 8159 SDLoc dl(N); 8160 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 8161 N->getOperand(3), 8162 DAG.getConstant(0, dl, MVT::i32)); 8163 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 8164 N->getOperand(3), 8165 DAG.getConstant(1, dl, MVT::i32)); 8166 8167 SDValue LongMul = DAG.getNode(Opc, dl, 8168 DAG.getVTList(MVT::i32, MVT::i32), 8169 N->getOperand(1), N->getOperand(2), 8170 Lo, Hi); 8171 Results.push_back(LongMul.getValue(0)); 8172 Results.push_back(LongMul.getValue(1)); 8173 } 8174 8175 /// ReplaceNodeResults - Replace the results of node with an illegal result 8176 /// type with new values built out of custom code. 8177 void ARMTargetLowering::ReplaceNodeResults(SDNode *N, 8178 SmallVectorImpl<SDValue> &Results, 8179 SelectionDAG &DAG) const { 8180 SDValue Res; 8181 switch (N->getOpcode()) { 8182 default: 8183 llvm_unreachable("Don't know how to custom expand this!"); 8184 case ISD::READ_REGISTER: 8185 ExpandREAD_REGISTER(N, Results, DAG); 8186 break; 8187 case ISD::BITCAST: 8188 Res = ExpandBITCAST(N, DAG, Subtarget); 8189 break; 8190 case ISD::SRL: 8191 case ISD::SRA: 8192 Res = Expand64BitShift(N, DAG, Subtarget); 8193 break; 8194 case ISD::SREM: 8195 case ISD::UREM: 8196 Res = LowerREM(N, DAG); 8197 break; 8198 case ISD::SDIVREM: 8199 case ISD::UDIVREM: 8200 Res = LowerDivRem(SDValue(N, 0), DAG); 8201 assert(Res.getNumOperands() == 2 && "DivRem needs two values"); 8202 Results.push_back(Res.getValue(0)); 8203 Results.push_back(Res.getValue(1)); 8204 return; 8205 case ISD::READCYCLECOUNTER: 8206 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget); 8207 return; 8208 case ISD::UDIV: 8209 case ISD::SDIV: 8210 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows"); 8211 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV, 8212 Results); 8213 case ISD::ATOMIC_CMP_SWAP: 8214 ReplaceCMP_SWAP_64Results(N, Results, DAG); 8215 return; 8216 case ISD::INTRINSIC_WO_CHAIN: 8217 return ReplaceLongIntrinsic(N, Results, DAG); 8218 } 8219 if (Res.getNode()) 8220 Results.push_back(Res); 8221 } 8222 8223 //===----------------------------------------------------------------------===// 8224 // ARM Scheduler Hooks 8225 //===----------------------------------------------------------------------===// 8226 8227 /// SetupEntryBlockForSjLj - Insert code into the entry block that creates and 8228 /// registers the function context. 8229 void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI, 8230 MachineBasicBlock *MBB, 8231 MachineBasicBlock *DispatchBB, 8232 int FI) const { 8233 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && 8234 "ROPI/RWPI not currently supported with SjLj"); 8235 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 8236 DebugLoc dl = MI.getDebugLoc(); 8237 MachineFunction *MF = MBB->getParent(); 8238 MachineRegisterInfo *MRI = &MF->getRegInfo(); 8239 MachineConstantPool *MCP = MF->getConstantPool(); 8240 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>(); 8241 const Function &F = MF->getFunction(); 8242 8243 bool isThumb = Subtarget->isThumb(); 8244 bool isThumb2 = Subtarget->isThumb2(); 8245 8246 unsigned PCLabelId = AFI->createPICLabelUId(); 8247 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8; 8248 ARMConstantPoolValue *CPV = 8249 ARMConstantPoolMBB::Create(F.getContext(), DispatchBB, PCLabelId, PCAdj); 8250 unsigned CPI = MCP->getConstantPoolIndex(CPV, 4); 8251 8252 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass 8253 : &ARM::GPRRegClass; 8254 8255 // Grab constant pool and fixed stack memory operands. 8256 MachineMemOperand *CPMMO = 8257 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF), 8258 MachineMemOperand::MOLoad, 4, 4); 8259 8260 MachineMemOperand *FIMMOSt = 8261 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI), 8262 MachineMemOperand::MOStore, 4, 4); 8263 8264 // Load the address of the dispatch MBB into the jump buffer. 8265 if (isThumb2) { 8266 // Incoming value: jbuf 8267 // ldr.n r5, LCPI1_1 8268 // orr r5, r5, #1 8269 // add r5, pc 8270 // str r5, [$jbuf, #+4] ; &jbuf[1] 8271 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8272 BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1) 8273 .addConstantPoolIndex(CPI) 8274 .addMemOperand(CPMMO) 8275 .add(predOps(ARMCC::AL)); 8276 // Set the low bit because of thumb mode. 8277 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8278 BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2) 8279 .addReg(NewVReg1, RegState::Kill) 8280 .addImm(0x01) 8281 .add(predOps(ARMCC::AL)) 8282 .add(condCodeOp()); 8283 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8284 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3) 8285 .addReg(NewVReg2, RegState::Kill) 8286 .addImm(PCLabelId); 8287 BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12)) 8288 .addReg(NewVReg3, RegState::Kill) 8289 .addFrameIndex(FI) 8290 .addImm(36) // &jbuf[1] :: pc 8291 .addMemOperand(FIMMOSt) 8292 .add(predOps(ARMCC::AL)); 8293 } else if (isThumb) { 8294 // Incoming value: jbuf 8295 // ldr.n r1, LCPI1_4 8296 // add r1, pc 8297 // mov r2, #1 8298 // orrs r1, r2 8299 // add r2, $jbuf, #+4 ; &jbuf[1] 8300 // str r1, [r2] 8301 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8302 BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1) 8303 .addConstantPoolIndex(CPI) 8304 .addMemOperand(CPMMO) 8305 .add(predOps(ARMCC::AL)); 8306 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8307 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2) 8308 .addReg(NewVReg1, RegState::Kill) 8309 .addImm(PCLabelId); 8310 // Set the low bit because of thumb mode. 8311 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8312 BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3) 8313 .addReg(ARM::CPSR, RegState::Define) 8314 .addImm(1) 8315 .add(predOps(ARMCC::AL)); 8316 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8317 BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4) 8318 .addReg(ARM::CPSR, RegState::Define) 8319 .addReg(NewVReg2, RegState::Kill) 8320 .addReg(NewVReg3, RegState::Kill) 8321 .add(predOps(ARMCC::AL)); 8322 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); 8323 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5) 8324 .addFrameIndex(FI) 8325 .addImm(36); // &jbuf[1] :: pc 8326 BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi)) 8327 .addReg(NewVReg4, RegState::Kill) 8328 .addReg(NewVReg5, RegState::Kill) 8329 .addImm(0) 8330 .addMemOperand(FIMMOSt) 8331 .add(predOps(ARMCC::AL)); 8332 } else { 8333 // Incoming value: jbuf 8334 // ldr r1, LCPI1_1 8335 // add r1, pc, r1 8336 // str r1, [$jbuf, #+4] ; &jbuf[1] 8337 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8338 BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1) 8339 .addConstantPoolIndex(CPI) 8340 .addImm(0) 8341 .addMemOperand(CPMMO) 8342 .add(predOps(ARMCC::AL)); 8343 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8344 BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2) 8345 .addReg(NewVReg1, RegState::Kill) 8346 .addImm(PCLabelId) 8347 .add(predOps(ARMCC::AL)); 8348 BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12)) 8349 .addReg(NewVReg2, RegState::Kill) 8350 .addFrameIndex(FI) 8351 .addImm(36) // &jbuf[1] :: pc 8352 .addMemOperand(FIMMOSt) 8353 .add(predOps(ARMCC::AL)); 8354 } 8355 } 8356 8357 void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI, 8358 MachineBasicBlock *MBB) const { 8359 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 8360 DebugLoc dl = MI.getDebugLoc(); 8361 MachineFunction *MF = MBB->getParent(); 8362 MachineRegisterInfo *MRI = &MF->getRegInfo(); 8363 MachineFrameInfo &MFI = MF->getFrameInfo(); 8364 int FI = MFI.getFunctionContextIndex(); 8365 8366 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass 8367 : &ARM::GPRnopcRegClass; 8368 8369 // Get a mapping of the call site numbers to all of the landing pads they're 8370 // associated with. 8371 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad; 8372 unsigned MaxCSNum = 0; 8373 for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E; 8374 ++BB) { 8375 if (!BB->isEHPad()) continue; 8376 8377 // FIXME: We should assert that the EH_LABEL is the first MI in the landing 8378 // pad. 8379 for (MachineBasicBlock::iterator 8380 II = BB->begin(), IE = BB->end(); II != IE; ++II) { 8381 if (!II->isEHLabel()) continue; 8382 8383 MCSymbol *Sym = II->getOperand(0).getMCSymbol(); 8384 if (!MF->hasCallSiteLandingPad(Sym)) continue; 8385 8386 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym); 8387 for (SmallVectorImpl<unsigned>::iterator 8388 CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end(); 8389 CSI != CSE; ++CSI) { 8390 CallSiteNumToLPad[*CSI].push_back(&*BB); 8391 MaxCSNum = std::max(MaxCSNum, *CSI); 8392 } 8393 break; 8394 } 8395 } 8396 8397 // Get an ordered list of the machine basic blocks for the jump table. 8398 std::vector<MachineBasicBlock*> LPadList; 8399 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs; 8400 LPadList.reserve(CallSiteNumToLPad.size()); 8401 for (unsigned I = 1; I <= MaxCSNum; ++I) { 8402 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I]; 8403 for (SmallVectorImpl<MachineBasicBlock*>::iterator 8404 II = MBBList.begin(), IE = MBBList.end(); II != IE; ++II) { 8405 LPadList.push_back(*II); 8406 InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end()); 8407 } 8408 } 8409 8410 assert(!LPadList.empty() && 8411 "No landing pad destinations for the dispatch jump table!"); 8412 8413 // Create the jump table and associated information. 8414 MachineJumpTableInfo *JTI = 8415 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline); 8416 unsigned MJTI = JTI->createJumpTableIndex(LPadList); 8417 8418 // Create the MBBs for the dispatch code. 8419 8420 // Shove the dispatch's address into the return slot in the function context. 8421 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock(); 8422 DispatchBB->setIsEHPad(); 8423 8424 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); 8425 unsigned trap_opcode; 8426 if (Subtarget->isThumb()) 8427 trap_opcode = ARM::tTRAP; 8428 else 8429 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP; 8430 8431 BuildMI(TrapBB, dl, TII->get(trap_opcode)); 8432 DispatchBB->addSuccessor(TrapBB); 8433 8434 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock(); 8435 DispatchBB->addSuccessor(DispContBB); 8436 8437 // Insert and MBBs. 8438 MF->insert(MF->end(), DispatchBB); 8439 MF->insert(MF->end(), DispContBB); 8440 MF->insert(MF->end(), TrapBB); 8441 8442 // Insert code into the entry block that creates and registers the function 8443 // context. 8444 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI); 8445 8446 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand( 8447 MachinePointerInfo::getFixedStack(*MF, FI), 8448 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, 4); 8449 8450 MachineInstrBuilder MIB; 8451 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup)); 8452 8453 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII); 8454 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo(); 8455 8456 // Add a register mask with no preserved registers. This results in all 8457 // registers being marked as clobbered. This can't work if the dispatch block 8458 // is in a Thumb1 function and is linked with ARM code which uses the FP 8459 // registers, as there is no way to preserve the FP registers in Thumb1 mode. 8460 MIB.addRegMask(RI.getSjLjDispatchPreservedMask(*MF)); 8461 8462 bool IsPositionIndependent = isPositionIndependent(); 8463 unsigned NumLPads = LPadList.size(); 8464 if (Subtarget->isThumb2()) { 8465 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8466 BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1) 8467 .addFrameIndex(FI) 8468 .addImm(4) 8469 .addMemOperand(FIMMOLd) 8470 .add(predOps(ARMCC::AL)); 8471 8472 if (NumLPads < 256) { 8473 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri)) 8474 .addReg(NewVReg1) 8475 .addImm(LPadList.size()) 8476 .add(predOps(ARMCC::AL)); 8477 } else { 8478 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8479 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1) 8480 .addImm(NumLPads & 0xFFFF) 8481 .add(predOps(ARMCC::AL)); 8482 8483 unsigned VReg2 = VReg1; 8484 if ((NumLPads & 0xFFFF0000) != 0) { 8485 VReg2 = MRI->createVirtualRegister(TRC); 8486 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2) 8487 .addReg(VReg1) 8488 .addImm(NumLPads >> 16) 8489 .add(predOps(ARMCC::AL)); 8490 } 8491 8492 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr)) 8493 .addReg(NewVReg1) 8494 .addReg(VReg2) 8495 .add(predOps(ARMCC::AL)); 8496 } 8497 8498 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc)) 8499 .addMBB(TrapBB) 8500 .addImm(ARMCC::HI) 8501 .addReg(ARM::CPSR); 8502 8503 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8504 BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT), NewVReg3) 8505 .addJumpTableIndex(MJTI) 8506 .add(predOps(ARMCC::AL)); 8507 8508 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8509 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4) 8510 .addReg(NewVReg3, RegState::Kill) 8511 .addReg(NewVReg1) 8512 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2)) 8513 .add(predOps(ARMCC::AL)) 8514 .add(condCodeOp()); 8515 8516 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT)) 8517 .addReg(NewVReg4, RegState::Kill) 8518 .addReg(NewVReg1) 8519 .addJumpTableIndex(MJTI); 8520 } else if (Subtarget->isThumb()) { 8521 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8522 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1) 8523 .addFrameIndex(FI) 8524 .addImm(1) 8525 .addMemOperand(FIMMOLd) 8526 .add(predOps(ARMCC::AL)); 8527 8528 if (NumLPads < 256) { 8529 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8)) 8530 .addReg(NewVReg1) 8531 .addImm(NumLPads) 8532 .add(predOps(ARMCC::AL)); 8533 } else { 8534 MachineConstantPool *ConstantPool = MF->getConstantPool(); 8535 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); 8536 const Constant *C = ConstantInt::get(Int32Ty, NumLPads); 8537 8538 // MachineConstantPool wants an explicit alignment. 8539 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); 8540 if (Align == 0) 8541 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); 8542 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); 8543 8544 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8545 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci)) 8546 .addReg(VReg1, RegState::Define) 8547 .addConstantPoolIndex(Idx) 8548 .add(predOps(ARMCC::AL)); 8549 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr)) 8550 .addReg(NewVReg1) 8551 .addReg(VReg1) 8552 .add(predOps(ARMCC::AL)); 8553 } 8554 8555 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc)) 8556 .addMBB(TrapBB) 8557 .addImm(ARMCC::HI) 8558 .addReg(ARM::CPSR); 8559 8560 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); 8561 BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2) 8562 .addReg(ARM::CPSR, RegState::Define) 8563 .addReg(NewVReg1) 8564 .addImm(2) 8565 .add(predOps(ARMCC::AL)); 8566 8567 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8568 BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3) 8569 .addJumpTableIndex(MJTI) 8570 .add(predOps(ARMCC::AL)); 8571 8572 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8573 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4) 8574 .addReg(ARM::CPSR, RegState::Define) 8575 .addReg(NewVReg2, RegState::Kill) 8576 .addReg(NewVReg3) 8577 .add(predOps(ARMCC::AL)); 8578 8579 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand( 8580 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4); 8581 8582 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); 8583 BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5) 8584 .addReg(NewVReg4, RegState::Kill) 8585 .addImm(0) 8586 .addMemOperand(JTMMOLd) 8587 .add(predOps(ARMCC::AL)); 8588 8589 unsigned NewVReg6 = NewVReg5; 8590 if (IsPositionIndependent) { 8591 NewVReg6 = MRI->createVirtualRegister(TRC); 8592 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6) 8593 .addReg(ARM::CPSR, RegState::Define) 8594 .addReg(NewVReg5, RegState::Kill) 8595 .addReg(NewVReg3) 8596 .add(predOps(ARMCC::AL)); 8597 } 8598 8599 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr)) 8600 .addReg(NewVReg6, RegState::Kill) 8601 .addJumpTableIndex(MJTI); 8602 } else { 8603 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); 8604 BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1) 8605 .addFrameIndex(FI) 8606 .addImm(4) 8607 .addMemOperand(FIMMOLd) 8608 .add(predOps(ARMCC::AL)); 8609 8610 if (NumLPads < 256) { 8611 BuildMI(DispatchBB, dl, TII->get(ARM::CMPri)) 8612 .addReg(NewVReg1) 8613 .addImm(NumLPads) 8614 .add(predOps(ARMCC::AL)); 8615 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) { 8616 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8617 BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1) 8618 .addImm(NumLPads & 0xFFFF) 8619 .add(predOps(ARMCC::AL)); 8620 8621 unsigned VReg2 = VReg1; 8622 if ((NumLPads & 0xFFFF0000) != 0) { 8623 VReg2 = MRI->createVirtualRegister(TRC); 8624 BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2) 8625 .addReg(VReg1) 8626 .addImm(NumLPads >> 16) 8627 .add(predOps(ARMCC::AL)); 8628 } 8629 8630 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) 8631 .addReg(NewVReg1) 8632 .addReg(VReg2) 8633 .add(predOps(ARMCC::AL)); 8634 } else { 8635 MachineConstantPool *ConstantPool = MF->getConstantPool(); 8636 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); 8637 const Constant *C = ConstantInt::get(Int32Ty, NumLPads); 8638 8639 // MachineConstantPool wants an explicit alignment. 8640 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); 8641 if (Align == 0) 8642 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); 8643 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); 8644 8645 unsigned VReg1 = MRI->createVirtualRegister(TRC); 8646 BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp)) 8647 .addReg(VReg1, RegState::Define) 8648 .addConstantPoolIndex(Idx) 8649 .addImm(0) 8650 .add(predOps(ARMCC::AL)); 8651 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) 8652 .addReg(NewVReg1) 8653 .addReg(VReg1, RegState::Kill) 8654 .add(predOps(ARMCC::AL)); 8655 } 8656 8657 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc)) 8658 .addMBB(TrapBB) 8659 .addImm(ARMCC::HI) 8660 .addReg(ARM::CPSR); 8661 8662 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); 8663 BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3) 8664 .addReg(NewVReg1) 8665 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2)) 8666 .add(predOps(ARMCC::AL)) 8667 .add(condCodeOp()); 8668 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); 8669 BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4) 8670 .addJumpTableIndex(MJTI) 8671 .add(predOps(ARMCC::AL)); 8672 8673 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand( 8674 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4); 8675 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); 8676 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5) 8677 .addReg(NewVReg3, RegState::Kill) 8678 .addReg(NewVReg4) 8679 .addImm(0) 8680 .addMemOperand(JTMMOLd) 8681 .add(predOps(ARMCC::AL)); 8682 8683 if (IsPositionIndependent) { 8684 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd)) 8685 .addReg(NewVReg5, RegState::Kill) 8686 .addReg(NewVReg4) 8687 .addJumpTableIndex(MJTI); 8688 } else { 8689 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr)) 8690 .addReg(NewVReg5, RegState::Kill) 8691 .addJumpTableIndex(MJTI); 8692 } 8693 } 8694 8695 // Add the jump table entries as successors to the MBB. 8696 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs; 8697 for (std::vector<MachineBasicBlock*>::iterator 8698 I = LPadList.begin(), E = LPadList.end(); I != E; ++I) { 8699 MachineBasicBlock *CurMBB = *I; 8700 if (SeenMBBs.insert(CurMBB).second) 8701 DispContBB->addSuccessor(CurMBB); 8702 } 8703 8704 // N.B. the order the invoke BBs are processed in doesn't matter here. 8705 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF); 8706 SmallVector<MachineBasicBlock*, 64> MBBLPads; 8707 for (MachineBasicBlock *BB : InvokeBBs) { 8708 8709 // Remove the landing pad successor from the invoke block and replace it 8710 // with the new dispatch block. 8711 SmallVector<MachineBasicBlock*, 4> Successors(BB->succ_begin(), 8712 BB->succ_end()); 8713 while (!Successors.empty()) { 8714 MachineBasicBlock *SMBB = Successors.pop_back_val(); 8715 if (SMBB->isEHPad()) { 8716 BB->removeSuccessor(SMBB); 8717 MBBLPads.push_back(SMBB); 8718 } 8719 } 8720 8721 BB->addSuccessor(DispatchBB, BranchProbability::getZero()); 8722 BB->normalizeSuccProbs(); 8723 8724 // Find the invoke call and mark all of the callee-saved registers as 8725 // 'implicit defined' so that they're spilled. This prevents code from 8726 // moving instructions to before the EH block, where they will never be 8727 // executed. 8728 for (MachineBasicBlock::reverse_iterator 8729 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) { 8730 if (!II->isCall()) continue; 8731 8732 DenseMap<unsigned, bool> DefRegs; 8733 for (MachineInstr::mop_iterator 8734 OI = II->operands_begin(), OE = II->operands_end(); 8735 OI != OE; ++OI) { 8736 if (!OI->isReg()) continue; 8737 DefRegs[OI->getReg()] = true; 8738 } 8739 8740 MachineInstrBuilder MIB(*MF, &*II); 8741 8742 for (unsigned i = 0; SavedRegs[i] != 0; ++i) { 8743 unsigned Reg = SavedRegs[i]; 8744 if (Subtarget->isThumb2() && 8745 !ARM::tGPRRegClass.contains(Reg) && 8746 !ARM::hGPRRegClass.contains(Reg)) 8747 continue; 8748 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg)) 8749 continue; 8750 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg)) 8751 continue; 8752 if (!DefRegs[Reg]) 8753 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead); 8754 } 8755 8756 break; 8757 } 8758 } 8759 8760 // Mark all former landing pads as non-landing pads. The dispatch is the only 8761 // landing pad now. 8762 for (SmallVectorImpl<MachineBasicBlock*>::iterator 8763 I = MBBLPads.begin(), E = MBBLPads.end(); I != E; ++I) 8764 (*I)->setIsEHPad(false); 8765 8766 // The instruction is gone now. 8767 MI.eraseFromParent(); 8768 } 8769 8770 static 8771 MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) { 8772 for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(), 8773 E = MBB->succ_end(); I != E; ++I) 8774 if (*I != Succ) 8775 return *I; 8776 llvm_unreachable("Expecting a BB with two successors!"); 8777 } 8778 8779 /// Return the load opcode for a given load size. If load size >= 8, 8780 /// neon opcode will be returned. 8781 static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) { 8782 if (LdSize >= 8) 8783 return LdSize == 16 ? ARM::VLD1q32wb_fixed 8784 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0; 8785 if (IsThumb1) 8786 return LdSize == 4 ? ARM::tLDRi 8787 : LdSize == 2 ? ARM::tLDRHi 8788 : LdSize == 1 ? ARM::tLDRBi : 0; 8789 if (IsThumb2) 8790 return LdSize == 4 ? ARM::t2LDR_POST 8791 : LdSize == 2 ? ARM::t2LDRH_POST 8792 : LdSize == 1 ? ARM::t2LDRB_POST : 0; 8793 return LdSize == 4 ? ARM::LDR_POST_IMM 8794 : LdSize == 2 ? ARM::LDRH_POST 8795 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0; 8796 } 8797 8798 /// Return the store opcode for a given store size. If store size >= 8, 8799 /// neon opcode will be returned. 8800 static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) { 8801 if (StSize >= 8) 8802 return StSize == 16 ? ARM::VST1q32wb_fixed 8803 : StSize == 8 ? ARM::VST1d32wb_fixed : 0; 8804 if (IsThumb1) 8805 return StSize == 4 ? ARM::tSTRi 8806 : StSize == 2 ? ARM::tSTRHi 8807 : StSize == 1 ? ARM::tSTRBi : 0; 8808 if (IsThumb2) 8809 return StSize == 4 ? ARM::t2STR_POST 8810 : StSize == 2 ? ARM::t2STRH_POST 8811 : StSize == 1 ? ARM::t2STRB_POST : 0; 8812 return StSize == 4 ? ARM::STR_POST_IMM 8813 : StSize == 2 ? ARM::STRH_POST 8814 : StSize == 1 ? ARM::STRB_POST_IMM : 0; 8815 } 8816 8817 /// Emit a post-increment load operation with given size. The instructions 8818 /// will be added to BB at Pos. 8819 static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, 8820 const TargetInstrInfo *TII, const DebugLoc &dl, 8821 unsigned LdSize, unsigned Data, unsigned AddrIn, 8822 unsigned AddrOut, bool IsThumb1, bool IsThumb2) { 8823 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2); 8824 assert(LdOpc != 0 && "Should have a load opcode"); 8825 if (LdSize >= 8) { 8826 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8827 .addReg(AddrOut, RegState::Define) 8828 .addReg(AddrIn) 8829 .addImm(0) 8830 .add(predOps(ARMCC::AL)); 8831 } else if (IsThumb1) { 8832 // load + update AddrIn 8833 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8834 .addReg(AddrIn) 8835 .addImm(0) 8836 .add(predOps(ARMCC::AL)); 8837 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut) 8838 .add(t1CondCodeOp()) 8839 .addReg(AddrIn) 8840 .addImm(LdSize) 8841 .add(predOps(ARMCC::AL)); 8842 } else if (IsThumb2) { 8843 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8844 .addReg(AddrOut, RegState::Define) 8845 .addReg(AddrIn) 8846 .addImm(LdSize) 8847 .add(predOps(ARMCC::AL)); 8848 } else { // arm 8849 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) 8850 .addReg(AddrOut, RegState::Define) 8851 .addReg(AddrIn) 8852 .addReg(0) 8853 .addImm(LdSize) 8854 .add(predOps(ARMCC::AL)); 8855 } 8856 } 8857 8858 /// Emit a post-increment store operation with given size. The instructions 8859 /// will be added to BB at Pos. 8860 static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, 8861 const TargetInstrInfo *TII, const DebugLoc &dl, 8862 unsigned StSize, unsigned Data, unsigned AddrIn, 8863 unsigned AddrOut, bool IsThumb1, bool IsThumb2) { 8864 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2); 8865 assert(StOpc != 0 && "Should have a store opcode"); 8866 if (StSize >= 8) { 8867 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) 8868 .addReg(AddrIn) 8869 .addImm(0) 8870 .addReg(Data) 8871 .add(predOps(ARMCC::AL)); 8872 } else if (IsThumb1) { 8873 // store + update AddrIn 8874 BuildMI(*BB, Pos, dl, TII->get(StOpc)) 8875 .addReg(Data) 8876 .addReg(AddrIn) 8877 .addImm(0) 8878 .add(predOps(ARMCC::AL)); 8879 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut) 8880 .add(t1CondCodeOp()) 8881 .addReg(AddrIn) 8882 .addImm(StSize) 8883 .add(predOps(ARMCC::AL)); 8884 } else if (IsThumb2) { 8885 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) 8886 .addReg(Data) 8887 .addReg(AddrIn) 8888 .addImm(StSize) 8889 .add(predOps(ARMCC::AL)); 8890 } else { // arm 8891 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) 8892 .addReg(Data) 8893 .addReg(AddrIn) 8894 .addReg(0) 8895 .addImm(StSize) 8896 .add(predOps(ARMCC::AL)); 8897 } 8898 } 8899 8900 MachineBasicBlock * 8901 ARMTargetLowering::EmitStructByval(MachineInstr &MI, 8902 MachineBasicBlock *BB) const { 8903 // This pseudo instruction has 3 operands: dst, src, size 8904 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold(). 8905 // Otherwise, we will generate unrolled scalar copies. 8906 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 8907 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 8908 MachineFunction::iterator It = ++BB->getIterator(); 8909 8910 unsigned dest = MI.getOperand(0).getReg(); 8911 unsigned src = MI.getOperand(1).getReg(); 8912 unsigned SizeVal = MI.getOperand(2).getImm(); 8913 unsigned Align = MI.getOperand(3).getImm(); 8914 DebugLoc dl = MI.getDebugLoc(); 8915 8916 MachineFunction *MF = BB->getParent(); 8917 MachineRegisterInfo &MRI = MF->getRegInfo(); 8918 unsigned UnitSize = 0; 8919 const TargetRegisterClass *TRC = nullptr; 8920 const TargetRegisterClass *VecTRC = nullptr; 8921 8922 bool IsThumb1 = Subtarget->isThumb1Only(); 8923 bool IsThumb2 = Subtarget->isThumb2(); 8924 bool IsThumb = Subtarget->isThumb(); 8925 8926 if (Align & 1) { 8927 UnitSize = 1; 8928 } else if (Align & 2) { 8929 UnitSize = 2; 8930 } else { 8931 // Check whether we can use NEON instructions. 8932 if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) && 8933 Subtarget->hasNEON()) { 8934 if ((Align % 16 == 0) && SizeVal >= 16) 8935 UnitSize = 16; 8936 else if ((Align % 8 == 0) && SizeVal >= 8) 8937 UnitSize = 8; 8938 } 8939 // Can't use NEON instructions. 8940 if (UnitSize == 0) 8941 UnitSize = 4; 8942 } 8943 8944 // Select the correct opcode and register class for unit size load/store 8945 bool IsNeon = UnitSize >= 8; 8946 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass; 8947 if (IsNeon) 8948 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass 8949 : UnitSize == 8 ? &ARM::DPRRegClass 8950 : nullptr; 8951 8952 unsigned BytesLeft = SizeVal % UnitSize; 8953 unsigned LoopSize = SizeVal - BytesLeft; 8954 8955 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) { 8956 // Use LDR and STR to copy. 8957 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize) 8958 // [destOut] = STR_POST(scratch, destIn, UnitSize) 8959 unsigned srcIn = src; 8960 unsigned destIn = dest; 8961 for (unsigned i = 0; i < LoopSize; i+=UnitSize) { 8962 unsigned srcOut = MRI.createVirtualRegister(TRC); 8963 unsigned destOut = MRI.createVirtualRegister(TRC); 8964 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC); 8965 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut, 8966 IsThumb1, IsThumb2); 8967 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut, 8968 IsThumb1, IsThumb2); 8969 srcIn = srcOut; 8970 destIn = destOut; 8971 } 8972 8973 // Handle the leftover bytes with LDRB and STRB. 8974 // [scratch, srcOut] = LDRB_POST(srcIn, 1) 8975 // [destOut] = STRB_POST(scratch, destIn, 1) 8976 for (unsigned i = 0; i < BytesLeft; i++) { 8977 unsigned srcOut = MRI.createVirtualRegister(TRC); 8978 unsigned destOut = MRI.createVirtualRegister(TRC); 8979 unsigned scratch = MRI.createVirtualRegister(TRC); 8980 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut, 8981 IsThumb1, IsThumb2); 8982 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut, 8983 IsThumb1, IsThumb2); 8984 srcIn = srcOut; 8985 destIn = destOut; 8986 } 8987 MI.eraseFromParent(); // The instruction is gone now. 8988 return BB; 8989 } 8990 8991 // Expand the pseudo op to a loop. 8992 // thisMBB: 8993 // ... 8994 // movw varEnd, # --> with thumb2 8995 // movt varEnd, # 8996 // ldrcp varEnd, idx --> without thumb2 8997 // fallthrough --> loopMBB 8998 // loopMBB: 8999 // PHI varPhi, varEnd, varLoop 9000 // PHI srcPhi, src, srcLoop 9001 // PHI destPhi, dst, destLoop 9002 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize) 9003 // [destLoop] = STR_POST(scratch, destPhi, UnitSize) 9004 // subs varLoop, varPhi, #UnitSize 9005 // bne loopMBB 9006 // fallthrough --> exitMBB 9007 // exitMBB: 9008 // epilogue to handle left-over bytes 9009 // [scratch, srcOut] = LDRB_POST(srcLoop, 1) 9010 // [destOut] = STRB_POST(scratch, destLoop, 1) 9011 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); 9012 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); 9013 MF->insert(It, loopMBB); 9014 MF->insert(It, exitMBB); 9015 9016 // Transfer the remainder of BB and its successor edges to exitMBB. 9017 exitMBB->splice(exitMBB->begin(), BB, 9018 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9019 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 9020 9021 // Load an immediate to varEnd. 9022 unsigned varEnd = MRI.createVirtualRegister(TRC); 9023 if (Subtarget->useMovt(*MF)) { 9024 unsigned Vtmp = varEnd; 9025 if ((LoopSize & 0xFFFF0000) != 0) 9026 Vtmp = MRI.createVirtualRegister(TRC); 9027 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVi16 : ARM::MOVi16), Vtmp) 9028 .addImm(LoopSize & 0xFFFF) 9029 .add(predOps(ARMCC::AL)); 9030 9031 if ((LoopSize & 0xFFFF0000) != 0) 9032 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVTi16 : ARM::MOVTi16), varEnd) 9033 .addReg(Vtmp) 9034 .addImm(LoopSize >> 16) 9035 .add(predOps(ARMCC::AL)); 9036 } else { 9037 MachineConstantPool *ConstantPool = MF->getConstantPool(); 9038 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); 9039 const Constant *C = ConstantInt::get(Int32Ty, LoopSize); 9040 9041 // MachineConstantPool wants an explicit alignment. 9042 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); 9043 if (Align == 0) 9044 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); 9045 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); 9046 9047 if (IsThumb) 9048 BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci)) 9049 .addReg(varEnd, RegState::Define) 9050 .addConstantPoolIndex(Idx) 9051 .add(predOps(ARMCC::AL)); 9052 else 9053 BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp)) 9054 .addReg(varEnd, RegState::Define) 9055 .addConstantPoolIndex(Idx) 9056 .addImm(0) 9057 .add(predOps(ARMCC::AL)); 9058 } 9059 BB->addSuccessor(loopMBB); 9060 9061 // Generate the loop body: 9062 // varPhi = PHI(varLoop, varEnd) 9063 // srcPhi = PHI(srcLoop, src) 9064 // destPhi = PHI(destLoop, dst) 9065 MachineBasicBlock *entryBB = BB; 9066 BB = loopMBB; 9067 unsigned varLoop = MRI.createVirtualRegister(TRC); 9068 unsigned varPhi = MRI.createVirtualRegister(TRC); 9069 unsigned srcLoop = MRI.createVirtualRegister(TRC); 9070 unsigned srcPhi = MRI.createVirtualRegister(TRC); 9071 unsigned destLoop = MRI.createVirtualRegister(TRC); 9072 unsigned destPhi = MRI.createVirtualRegister(TRC); 9073 9074 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi) 9075 .addReg(varLoop).addMBB(loopMBB) 9076 .addReg(varEnd).addMBB(entryBB); 9077 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi) 9078 .addReg(srcLoop).addMBB(loopMBB) 9079 .addReg(src).addMBB(entryBB); 9080 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi) 9081 .addReg(destLoop).addMBB(loopMBB) 9082 .addReg(dest).addMBB(entryBB); 9083 9084 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize) 9085 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz) 9086 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC); 9087 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop, 9088 IsThumb1, IsThumb2); 9089 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop, 9090 IsThumb1, IsThumb2); 9091 9092 // Decrement loop variable by UnitSize. 9093 if (IsThumb1) { 9094 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop) 9095 .add(t1CondCodeOp()) 9096 .addReg(varPhi) 9097 .addImm(UnitSize) 9098 .add(predOps(ARMCC::AL)); 9099 } else { 9100 MachineInstrBuilder MIB = 9101 BuildMI(*BB, BB->end(), dl, 9102 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop); 9103 MIB.addReg(varPhi) 9104 .addImm(UnitSize) 9105 .add(predOps(ARMCC::AL)) 9106 .add(condCodeOp()); 9107 MIB->getOperand(5).setReg(ARM::CPSR); 9108 MIB->getOperand(5).setIsDef(true); 9109 } 9110 BuildMI(*BB, BB->end(), dl, 9111 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc)) 9112 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR); 9113 9114 // loopMBB can loop back to loopMBB or fall through to exitMBB. 9115 BB->addSuccessor(loopMBB); 9116 BB->addSuccessor(exitMBB); 9117 9118 // Add epilogue to handle BytesLeft. 9119 BB = exitMBB; 9120 auto StartOfExit = exitMBB->begin(); 9121 9122 // [scratch, srcOut] = LDRB_POST(srcLoop, 1) 9123 // [destOut] = STRB_POST(scratch, destLoop, 1) 9124 unsigned srcIn = srcLoop; 9125 unsigned destIn = destLoop; 9126 for (unsigned i = 0; i < BytesLeft; i++) { 9127 unsigned srcOut = MRI.createVirtualRegister(TRC); 9128 unsigned destOut = MRI.createVirtualRegister(TRC); 9129 unsigned scratch = MRI.createVirtualRegister(TRC); 9130 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut, 9131 IsThumb1, IsThumb2); 9132 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut, 9133 IsThumb1, IsThumb2); 9134 srcIn = srcOut; 9135 destIn = destOut; 9136 } 9137 9138 MI.eraseFromParent(); // The instruction is gone now. 9139 return BB; 9140 } 9141 9142 MachineBasicBlock * 9143 ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI, 9144 MachineBasicBlock *MBB) const { 9145 const TargetMachine &TM = getTargetMachine(); 9146 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); 9147 DebugLoc DL = MI.getDebugLoc(); 9148 9149 assert(Subtarget->isTargetWindows() && 9150 "__chkstk is only supported on Windows"); 9151 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode"); 9152 9153 // __chkstk takes the number of words to allocate on the stack in R4, and 9154 // returns the stack adjustment in number of bytes in R4. This will not 9155 // clober any other registers (other than the obvious lr). 9156 // 9157 // Although, technically, IP should be considered a register which may be 9158 // clobbered, the call itself will not touch it. Windows on ARM is a pure 9159 // thumb-2 environment, so there is no interworking required. As a result, we 9160 // do not expect a veneer to be emitted by the linker, clobbering IP. 9161 // 9162 // Each module receives its own copy of __chkstk, so no import thunk is 9163 // required, again, ensuring that IP is not clobbered. 9164 // 9165 // Finally, although some linkers may theoretically provide a trampoline for 9166 // out of range calls (which is quite common due to a 32M range limitation of 9167 // branches for Thumb), we can generate the long-call version via 9168 // -mcmodel=large, alleviating the need for the trampoline which may clobber 9169 // IP. 9170 9171 switch (TM.getCodeModel()) { 9172 case CodeModel::Small: 9173 case CodeModel::Medium: 9174 case CodeModel::Kernel: 9175 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL)) 9176 .add(predOps(ARMCC::AL)) 9177 .addExternalSymbol("__chkstk") 9178 .addReg(ARM::R4, RegState::Implicit | RegState::Kill) 9179 .addReg(ARM::R4, RegState::Implicit | RegState::Define) 9180 .addReg(ARM::R12, 9181 RegState::Implicit | RegState::Define | RegState::Dead) 9182 .addReg(ARM::CPSR, 9183 RegState::Implicit | RegState::Define | RegState::Dead); 9184 break; 9185 case CodeModel::Large: { 9186 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 9187 unsigned Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass); 9188 9189 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg) 9190 .addExternalSymbol("__chkstk"); 9191 BuildMI(*MBB, MI, DL, TII.get(ARM::tBLXr)) 9192 .add(predOps(ARMCC::AL)) 9193 .addReg(Reg, RegState::Kill) 9194 .addReg(ARM::R4, RegState::Implicit | RegState::Kill) 9195 .addReg(ARM::R4, RegState::Implicit | RegState::Define) 9196 .addReg(ARM::R12, 9197 RegState::Implicit | RegState::Define | RegState::Dead) 9198 .addReg(ARM::CPSR, 9199 RegState::Implicit | RegState::Define | RegState::Dead); 9200 break; 9201 } 9202 } 9203 9204 BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr), ARM::SP) 9205 .addReg(ARM::SP, RegState::Kill) 9206 .addReg(ARM::R4, RegState::Kill) 9207 .setMIFlags(MachineInstr::FrameSetup) 9208 .add(predOps(ARMCC::AL)) 9209 .add(condCodeOp()); 9210 9211 MI.eraseFromParent(); 9212 return MBB; 9213 } 9214 9215 MachineBasicBlock * 9216 ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI, 9217 MachineBasicBlock *MBB) const { 9218 DebugLoc DL = MI.getDebugLoc(); 9219 MachineFunction *MF = MBB->getParent(); 9220 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 9221 9222 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock(); 9223 MF->insert(++MBB->getIterator(), ContBB); 9224 ContBB->splice(ContBB->begin(), MBB, 9225 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 9226 ContBB->transferSuccessorsAndUpdatePHIs(MBB); 9227 MBB->addSuccessor(ContBB); 9228 9229 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); 9230 BuildMI(TrapBB, DL, TII->get(ARM::t__brkdiv0)); 9231 MF->push_back(TrapBB); 9232 MBB->addSuccessor(TrapBB); 9233 9234 BuildMI(*MBB, MI, DL, TII->get(ARM::tCMPi8)) 9235 .addReg(MI.getOperand(0).getReg()) 9236 .addImm(0) 9237 .add(predOps(ARMCC::AL)); 9238 BuildMI(*MBB, MI, DL, TII->get(ARM::t2Bcc)) 9239 .addMBB(TrapBB) 9240 .addImm(ARMCC::EQ) 9241 .addReg(ARM::CPSR); 9242 9243 MI.eraseFromParent(); 9244 return ContBB; 9245 } 9246 9247 MachineBasicBlock * 9248 ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 9249 MachineBasicBlock *BB) const { 9250 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 9251 DebugLoc dl = MI.getDebugLoc(); 9252 bool isThumb2 = Subtarget->isThumb2(); 9253 switch (MI.getOpcode()) { 9254 default: { 9255 MI.print(errs()); 9256 llvm_unreachable("Unexpected instr type to insert"); 9257 } 9258 9259 // Thumb1 post-indexed loads are really just single-register LDMs. 9260 case ARM::tLDR_postidx: { 9261 MachineOperand Def(MI.getOperand(1)); 9262 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD)) 9263 .add(Def) // Rn_wb 9264 .add(MI.getOperand(2)) // Rn 9265 .add(MI.getOperand(3)) // PredImm 9266 .add(MI.getOperand(4)) // PredReg 9267 .add(MI.getOperand(0)); // Rt 9268 MI.eraseFromParent(); 9269 return BB; 9270 } 9271 9272 // The Thumb2 pre-indexed stores have the same MI operands, they just 9273 // define them differently in the .td files from the isel patterns, so 9274 // they need pseudos. 9275 case ARM::t2STR_preidx: 9276 MI.setDesc(TII->get(ARM::t2STR_PRE)); 9277 return BB; 9278 case ARM::t2STRB_preidx: 9279 MI.setDesc(TII->get(ARM::t2STRB_PRE)); 9280 return BB; 9281 case ARM::t2STRH_preidx: 9282 MI.setDesc(TII->get(ARM::t2STRH_PRE)); 9283 return BB; 9284 9285 case ARM::STRi_preidx: 9286 case ARM::STRBi_preidx: { 9287 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM 9288 : ARM::STRB_PRE_IMM; 9289 // Decode the offset. 9290 unsigned Offset = MI.getOperand(4).getImm(); 9291 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub; 9292 Offset = ARM_AM::getAM2Offset(Offset); 9293 if (isSub) 9294 Offset = -Offset; 9295 9296 MachineMemOperand *MMO = *MI.memoperands_begin(); 9297 BuildMI(*BB, MI, dl, TII->get(NewOpc)) 9298 .add(MI.getOperand(0)) // Rn_wb 9299 .add(MI.getOperand(1)) // Rt 9300 .add(MI.getOperand(2)) // Rn 9301 .addImm(Offset) // offset (skip GPR==zero_reg) 9302 .add(MI.getOperand(5)) // pred 9303 .add(MI.getOperand(6)) 9304 .addMemOperand(MMO); 9305 MI.eraseFromParent(); 9306 return BB; 9307 } 9308 case ARM::STRr_preidx: 9309 case ARM::STRBr_preidx: 9310 case ARM::STRH_preidx: { 9311 unsigned NewOpc; 9312 switch (MI.getOpcode()) { 9313 default: llvm_unreachable("unexpected opcode!"); 9314 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break; 9315 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break; 9316 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break; 9317 } 9318 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc)); 9319 for (unsigned i = 0; i < MI.getNumOperands(); ++i) 9320 MIB.add(MI.getOperand(i)); 9321 MI.eraseFromParent(); 9322 return BB; 9323 } 9324 9325 case ARM::tMOVCCr_pseudo: { 9326 // To "insert" a SELECT_CC instruction, we actually have to insert the 9327 // diamond control-flow pattern. The incoming instruction knows the 9328 // destination vreg to set, the condition code register to branch on, the 9329 // true/false values to select between, and a branch opcode to use. 9330 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 9331 MachineFunction::iterator It = ++BB->getIterator(); 9332 9333 // thisMBB: 9334 // ... 9335 // TrueVal = ... 9336 // cmpTY ccX, r1, r2 9337 // bCC copy1MBB 9338 // fallthrough --> copy0MBB 9339 MachineBasicBlock *thisMBB = BB; 9340 MachineFunction *F = BB->getParent(); 9341 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 9342 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 9343 F->insert(It, copy0MBB); 9344 F->insert(It, sinkMBB); 9345 9346 // Transfer the remainder of BB and its successor edges to sinkMBB. 9347 sinkMBB->splice(sinkMBB->begin(), BB, 9348 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9349 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 9350 9351 BB->addSuccessor(copy0MBB); 9352 BB->addSuccessor(sinkMBB); 9353 9354 BuildMI(BB, dl, TII->get(ARM::tBcc)) 9355 .addMBB(sinkMBB) 9356 .addImm(MI.getOperand(3).getImm()) 9357 .addReg(MI.getOperand(4).getReg()); 9358 9359 // copy0MBB: 9360 // %FalseValue = ... 9361 // # fallthrough to sinkMBB 9362 BB = copy0MBB; 9363 9364 // Update machine-CFG edges 9365 BB->addSuccessor(sinkMBB); 9366 9367 // sinkMBB: 9368 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 9369 // ... 9370 BB = sinkMBB; 9371 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg()) 9372 .addReg(MI.getOperand(1).getReg()) 9373 .addMBB(copy0MBB) 9374 .addReg(MI.getOperand(2).getReg()) 9375 .addMBB(thisMBB); 9376 9377 MI.eraseFromParent(); // The pseudo instruction is gone now. 9378 return BB; 9379 } 9380 9381 case ARM::BCCi64: 9382 case ARM::BCCZi64: { 9383 // If there is an unconditional branch to the other successor, remove it. 9384 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9385 9386 // Compare both parts that make up the double comparison separately for 9387 // equality. 9388 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64; 9389 9390 unsigned LHS1 = MI.getOperand(1).getReg(); 9391 unsigned LHS2 = MI.getOperand(2).getReg(); 9392 if (RHSisZero) { 9393 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) 9394 .addReg(LHS1) 9395 .addImm(0) 9396 .add(predOps(ARMCC::AL)); 9397 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) 9398 .addReg(LHS2).addImm(0) 9399 .addImm(ARMCC::EQ).addReg(ARM::CPSR); 9400 } else { 9401 unsigned RHS1 = MI.getOperand(3).getReg(); 9402 unsigned RHS2 = MI.getOperand(4).getReg(); 9403 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) 9404 .addReg(LHS1) 9405 .addReg(RHS1) 9406 .add(predOps(ARMCC::AL)); 9407 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) 9408 .addReg(LHS2).addReg(RHS2) 9409 .addImm(ARMCC::EQ).addReg(ARM::CPSR); 9410 } 9411 9412 MachineBasicBlock *destMBB = MI.getOperand(RHSisZero ? 3 : 5).getMBB(); 9413 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB); 9414 if (MI.getOperand(0).getImm() == ARMCC::NE) 9415 std::swap(destMBB, exitMBB); 9416 9417 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)) 9418 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR); 9419 if (isThumb2) 9420 BuildMI(BB, dl, TII->get(ARM::t2B)) 9421 .addMBB(exitMBB) 9422 .add(predOps(ARMCC::AL)); 9423 else 9424 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB); 9425 9426 MI.eraseFromParent(); // The pseudo instruction is gone now. 9427 return BB; 9428 } 9429 9430 case ARM::Int_eh_sjlj_setjmp: 9431 case ARM::Int_eh_sjlj_setjmp_nofp: 9432 case ARM::tInt_eh_sjlj_setjmp: 9433 case ARM::t2Int_eh_sjlj_setjmp: 9434 case ARM::t2Int_eh_sjlj_setjmp_nofp: 9435 return BB; 9436 9437 case ARM::Int_eh_sjlj_setup_dispatch: 9438 EmitSjLjDispatchBlock(MI, BB); 9439 return BB; 9440 9441 case ARM::ABS: 9442 case ARM::t2ABS: { 9443 // To insert an ABS instruction, we have to insert the 9444 // diamond control-flow pattern. The incoming instruction knows the 9445 // source vreg to test against 0, the destination vreg to set, 9446 // the condition code register to branch on, the 9447 // true/false values to select between, and a branch opcode to use. 9448 // It transforms 9449 // V1 = ABS V0 9450 // into 9451 // V2 = MOVS V0 9452 // BCC (branch to SinkBB if V0 >= 0) 9453 // RSBBB: V3 = RSBri V2, 0 (compute ABS if V2 < 0) 9454 // SinkBB: V1 = PHI(V2, V3) 9455 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 9456 MachineFunction::iterator BBI = ++BB->getIterator(); 9457 MachineFunction *Fn = BB->getParent(); 9458 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB); 9459 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB); 9460 Fn->insert(BBI, RSBBB); 9461 Fn->insert(BBI, SinkBB); 9462 9463 unsigned int ABSSrcReg = MI.getOperand(1).getReg(); 9464 unsigned int ABSDstReg = MI.getOperand(0).getReg(); 9465 bool ABSSrcKIll = MI.getOperand(1).isKill(); 9466 bool isThumb2 = Subtarget->isThumb2(); 9467 MachineRegisterInfo &MRI = Fn->getRegInfo(); 9468 // In Thumb mode S must not be specified if source register is the SP or 9469 // PC and if destination register is the SP, so restrict register class 9470 unsigned NewRsbDstReg = 9471 MRI.createVirtualRegister(isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass); 9472 9473 // Transfer the remainder of BB and its successor edges to sinkMBB. 9474 SinkBB->splice(SinkBB->begin(), BB, 9475 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 9476 SinkBB->transferSuccessorsAndUpdatePHIs(BB); 9477 9478 BB->addSuccessor(RSBBB); 9479 BB->addSuccessor(SinkBB); 9480 9481 // fall through to SinkMBB 9482 RSBBB->addSuccessor(SinkBB); 9483 9484 // insert a cmp at the end of BB 9485 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) 9486 .addReg(ABSSrcReg) 9487 .addImm(0) 9488 .add(predOps(ARMCC::AL)); 9489 9490 // insert a bcc with opposite CC to ARMCC::MI at the end of BB 9491 BuildMI(BB, dl, 9492 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB) 9493 .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR); 9494 9495 // insert rsbri in RSBBB 9496 // Note: BCC and rsbri will be converted into predicated rsbmi 9497 // by if-conversion pass 9498 BuildMI(*RSBBB, RSBBB->begin(), dl, 9499 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg) 9500 .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0) 9501 .addImm(0) 9502 .add(predOps(ARMCC::AL)) 9503 .add(condCodeOp()); 9504 9505 // insert PHI in SinkBB, 9506 // reuse ABSDstReg to not change uses of ABS instruction 9507 BuildMI(*SinkBB, SinkBB->begin(), dl, 9508 TII->get(ARM::PHI), ABSDstReg) 9509 .addReg(NewRsbDstReg).addMBB(RSBBB) 9510 .addReg(ABSSrcReg).addMBB(BB); 9511 9512 // remove ABS instruction 9513 MI.eraseFromParent(); 9514 9515 // return last added BB 9516 return SinkBB; 9517 } 9518 case ARM::COPY_STRUCT_BYVAL_I32: 9519 ++NumLoopByVals; 9520 return EmitStructByval(MI, BB); 9521 case ARM::WIN__CHKSTK: 9522 return EmitLowered__chkstk(MI, BB); 9523 case ARM::WIN__DBZCHK: 9524 return EmitLowered__dbzchk(MI, BB); 9525 } 9526 } 9527 9528 /// Attaches vregs to MEMCPY that it will use as scratch registers 9529 /// when it is expanded into LDM/STM. This is done as a post-isel lowering 9530 /// instead of as a custom inserter because we need the use list from the SDNode. 9531 static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, 9532 MachineInstr &MI, const SDNode *Node) { 9533 bool isThumb1 = Subtarget->isThumb1Only(); 9534 9535 DebugLoc DL = MI.getDebugLoc(); 9536 MachineFunction *MF = MI.getParent()->getParent(); 9537 MachineRegisterInfo &MRI = MF->getRegInfo(); 9538 MachineInstrBuilder MIB(*MF, MI); 9539 9540 // If the new dst/src is unused mark it as dead. 9541 if (!Node->hasAnyUseOfValue(0)) { 9542 MI.getOperand(0).setIsDead(true); 9543 } 9544 if (!Node->hasAnyUseOfValue(1)) { 9545 MI.getOperand(1).setIsDead(true); 9546 } 9547 9548 // The MEMCPY both defines and kills the scratch registers. 9549 for (unsigned I = 0; I != MI.getOperand(4).getImm(); ++I) { 9550 unsigned TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass 9551 : &ARM::GPRRegClass); 9552 MIB.addReg(TmpReg, RegState::Define|RegState::Dead); 9553 } 9554 } 9555 9556 void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, 9557 SDNode *Node) const { 9558 if (MI.getOpcode() == ARM::MEMCPY) { 9559 attachMEMCPYScratchRegs(Subtarget, MI, Node); 9560 return; 9561 } 9562 9563 const MCInstrDesc *MCID = &MI.getDesc(); 9564 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB, 9565 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional 9566 // operand is still set to noreg. If needed, set the optional operand's 9567 // register to CPSR, and remove the redundant implicit def. 9568 // 9569 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR). 9570 9571 // Rename pseudo opcodes. 9572 unsigned NewOpc = convertAddSubFlagsOpcode(MI.getOpcode()); 9573 unsigned ccOutIdx; 9574 if (NewOpc) { 9575 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo(); 9576 MCID = &TII->get(NewOpc); 9577 9578 assert(MCID->getNumOperands() == 9579 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() 9580 && "converted opcode should be the same except for cc_out" 9581 " (and, on Thumb1, pred)"); 9582 9583 MI.setDesc(*MCID); 9584 9585 // Add the optional cc_out operand 9586 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/true)); 9587 9588 // On Thumb1, move all input operands to the end, then add the predicate 9589 if (Subtarget->isThumb1Only()) { 9590 for (unsigned c = MCID->getNumOperands() - 4; c--;) { 9591 MI.addOperand(MI.getOperand(1)); 9592 MI.RemoveOperand(1); 9593 } 9594 9595 // Restore the ties 9596 for (unsigned i = MI.getNumOperands(); i--;) { 9597 const MachineOperand& op = MI.getOperand(i); 9598 if (op.isReg() && op.isUse()) { 9599 int DefIdx = MCID->getOperandConstraint(i, MCOI::TIED_TO); 9600 if (DefIdx != -1) 9601 MI.tieOperands(DefIdx, i); 9602 } 9603 } 9604 9605 MI.addOperand(MachineOperand::CreateImm(ARMCC::AL)); 9606 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/false)); 9607 ccOutIdx = 1; 9608 } else 9609 ccOutIdx = MCID->getNumOperands() - 1; 9610 } else 9611 ccOutIdx = MCID->getNumOperands() - 1; 9612 9613 // Any ARM instruction that sets the 's' bit should specify an optional 9614 // "cc_out" operand in the last operand position. 9615 if (!MI.hasOptionalDef() || !MCID->OpInfo[ccOutIdx].isOptionalDef()) { 9616 assert(!NewOpc && "Optional cc_out operand required"); 9617 return; 9618 } 9619 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it 9620 // since we already have an optional CPSR def. 9621 bool definesCPSR = false; 9622 bool deadCPSR = false; 9623 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e; 9624 ++i) { 9625 const MachineOperand &MO = MI.getOperand(i); 9626 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) { 9627 definesCPSR = true; 9628 if (MO.isDead()) 9629 deadCPSR = true; 9630 MI.RemoveOperand(i); 9631 break; 9632 } 9633 } 9634 if (!definesCPSR) { 9635 assert(!NewOpc && "Optional cc_out operand required"); 9636 return; 9637 } 9638 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag"); 9639 if (deadCPSR) { 9640 assert(!MI.getOperand(ccOutIdx).getReg() && 9641 "expect uninitialized optional cc_out operand"); 9642 // Thumb1 instructions must have the S bit even if the CPSR is dead. 9643 if (!Subtarget->isThumb1Only()) 9644 return; 9645 } 9646 9647 // If this instruction was defined with an optional CPSR def and its dag node 9648 // had a live implicit CPSR def, then activate the optional CPSR def. 9649 MachineOperand &MO = MI.getOperand(ccOutIdx); 9650 MO.setReg(ARM::CPSR); 9651 MO.setIsDef(true); 9652 } 9653 9654 //===----------------------------------------------------------------------===// 9655 // ARM Optimization Hooks 9656 //===----------------------------------------------------------------------===// 9657 9658 // Helper function that checks if N is a null or all ones constant. 9659 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) { 9660 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N); 9661 } 9662 9663 // Return true if N is conditionally 0 or all ones. 9664 // Detects these expressions where cc is an i1 value: 9665 // 9666 // (select cc 0, y) [AllOnes=0] 9667 // (select cc y, 0) [AllOnes=0] 9668 // (zext cc) [AllOnes=0] 9669 // (sext cc) [AllOnes=0/1] 9670 // (select cc -1, y) [AllOnes=1] 9671 // (select cc y, -1) [AllOnes=1] 9672 // 9673 // Invert is set when N is the null/all ones constant when CC is false. 9674 // OtherOp is set to the alternative value of N. 9675 static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, 9676 SDValue &CC, bool &Invert, 9677 SDValue &OtherOp, 9678 SelectionDAG &DAG) { 9679 switch (N->getOpcode()) { 9680 default: return false; 9681 case ISD::SELECT: { 9682 CC = N->getOperand(0); 9683 SDValue N1 = N->getOperand(1); 9684 SDValue N2 = N->getOperand(2); 9685 if (isZeroOrAllOnes(N1, AllOnes)) { 9686 Invert = false; 9687 OtherOp = N2; 9688 return true; 9689 } 9690 if (isZeroOrAllOnes(N2, AllOnes)) { 9691 Invert = true; 9692 OtherOp = N1; 9693 return true; 9694 } 9695 return false; 9696 } 9697 case ISD::ZERO_EXTEND: 9698 // (zext cc) can never be the all ones value. 9699 if (AllOnes) 9700 return false; 9701 LLVM_FALLTHROUGH; 9702 case ISD::SIGN_EXTEND: { 9703 SDLoc dl(N); 9704 EVT VT = N->getValueType(0); 9705 CC = N->getOperand(0); 9706 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC) 9707 return false; 9708 Invert = !AllOnes; 9709 if (AllOnes) 9710 // When looking for an AllOnes constant, N is an sext, and the 'other' 9711 // value is 0. 9712 OtherOp = DAG.getConstant(0, dl, VT); 9713 else if (N->getOpcode() == ISD::ZERO_EXTEND) 9714 // When looking for a 0 constant, N can be zext or sext. 9715 OtherOp = DAG.getConstant(1, dl, VT); 9716 else 9717 OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, 9718 VT); 9719 return true; 9720 } 9721 } 9722 } 9723 9724 // Combine a constant select operand into its use: 9725 // 9726 // (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) 9727 // (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) 9728 // (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1] 9729 // (or (select cc, 0, c), x) -> (select cc, x, (or, x, c)) 9730 // (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c)) 9731 // 9732 // The transform is rejected if the select doesn't have a constant operand that 9733 // is null, or all ones when AllOnes is set. 9734 // 9735 // Also recognize sext/zext from i1: 9736 // 9737 // (add (zext cc), x) -> (select cc (add x, 1), x) 9738 // (add (sext cc), x) -> (select cc (add x, -1), x) 9739 // 9740 // These transformations eventually create predicated instructions. 9741 // 9742 // @param N The node to transform. 9743 // @param Slct The N operand that is a select. 9744 // @param OtherOp The other N operand (x above). 9745 // @param DCI Context. 9746 // @param AllOnes Require the select constant to be all ones instead of null. 9747 // @returns The new node, or SDValue() on failure. 9748 static 9749 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, 9750 TargetLowering::DAGCombinerInfo &DCI, 9751 bool AllOnes = false) { 9752 SelectionDAG &DAG = DCI.DAG; 9753 EVT VT = N->getValueType(0); 9754 SDValue NonConstantVal; 9755 SDValue CCOp; 9756 bool SwapSelectOps; 9757 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps, 9758 NonConstantVal, DAG)) 9759 return SDValue(); 9760 9761 // Slct is now know to be the desired identity constant when CC is true. 9762 SDValue TrueVal = OtherOp; 9763 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, 9764 OtherOp, NonConstantVal); 9765 // Unless SwapSelectOps says CC should be false. 9766 if (SwapSelectOps) 9767 std::swap(TrueVal, FalseVal); 9768 9769 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, 9770 CCOp, TrueVal, FalseVal); 9771 } 9772 9773 // Attempt combineSelectAndUse on each operand of a commutative operator N. 9774 static 9775 SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, 9776 TargetLowering::DAGCombinerInfo &DCI) { 9777 SDValue N0 = N->getOperand(0); 9778 SDValue N1 = N->getOperand(1); 9779 if (N0.getNode()->hasOneUse()) 9780 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes)) 9781 return Result; 9782 if (N1.getNode()->hasOneUse()) 9783 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes)) 9784 return Result; 9785 return SDValue(); 9786 } 9787 9788 static bool IsVUZPShuffleNode(SDNode *N) { 9789 // VUZP shuffle node. 9790 if (N->getOpcode() == ARMISD::VUZP) 9791 return true; 9792 9793 // "VUZP" on i32 is an alias for VTRN. 9794 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32) 9795 return true; 9796 9797 return false; 9798 } 9799 9800 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, 9801 TargetLowering::DAGCombinerInfo &DCI, 9802 const ARMSubtarget *Subtarget) { 9803 // Look for ADD(VUZP.0, VUZP.1). 9804 if (!IsVUZPShuffleNode(N0.getNode()) || N0.getNode() != N1.getNode() || 9805 N0 == N1) 9806 return SDValue(); 9807 9808 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD. 9809 if (!N->getValueType(0).is64BitVector()) 9810 return SDValue(); 9811 9812 // Generate vpadd. 9813 SelectionDAG &DAG = DCI.DAG; 9814 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9815 SDLoc dl(N); 9816 SDNode *Unzip = N0.getNode(); 9817 EVT VT = N->getValueType(0); 9818 9819 SmallVector<SDValue, 8> Ops; 9820 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl, 9821 TLI.getPointerTy(DAG.getDataLayout()))); 9822 Ops.push_back(Unzip->getOperand(0)); 9823 Ops.push_back(Unzip->getOperand(1)); 9824 9825 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); 9826 } 9827 9828 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, 9829 TargetLowering::DAGCombinerInfo &DCI, 9830 const ARMSubtarget *Subtarget) { 9831 // Check for two extended operands. 9832 if (!(N0.getOpcode() == ISD::SIGN_EXTEND && 9833 N1.getOpcode() == ISD::SIGN_EXTEND) && 9834 !(N0.getOpcode() == ISD::ZERO_EXTEND && 9835 N1.getOpcode() == ISD::ZERO_EXTEND)) 9836 return SDValue(); 9837 9838 SDValue N00 = N0.getOperand(0); 9839 SDValue N10 = N1.getOperand(0); 9840 9841 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1)) 9842 if (!IsVUZPShuffleNode(N00.getNode()) || N00.getNode() != N10.getNode() || 9843 N00 == N10) 9844 return SDValue(); 9845 9846 // We only recognize Q register paddl here; this can't be reached until 9847 // after type legalization. 9848 if (!N00.getValueType().is64BitVector() || 9849 !N0.getValueType().is128BitVector()) 9850 return SDValue(); 9851 9852 // Generate vpaddl. 9853 SelectionDAG &DAG = DCI.DAG; 9854 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9855 SDLoc dl(N); 9856 EVT VT = N->getValueType(0); 9857 9858 SmallVector<SDValue, 8> Ops; 9859 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension. 9860 unsigned Opcode; 9861 if (N0.getOpcode() == ISD::SIGN_EXTEND) 9862 Opcode = Intrinsic::arm_neon_vpaddls; 9863 else 9864 Opcode = Intrinsic::arm_neon_vpaddlu; 9865 Ops.push_back(DAG.getConstant(Opcode, dl, 9866 TLI.getPointerTy(DAG.getDataLayout()))); 9867 EVT ElemTy = N00.getValueType().getVectorElementType(); 9868 unsigned NumElts = VT.getVectorNumElements(); 9869 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2); 9870 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT, 9871 N00.getOperand(0), N00.getOperand(1)); 9872 Ops.push_back(Concat); 9873 9874 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); 9875 } 9876 9877 // FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in 9878 // an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is 9879 // much easier to match. 9880 static SDValue 9881 AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, 9882 TargetLowering::DAGCombinerInfo &DCI, 9883 const ARMSubtarget *Subtarget) { 9884 // Only perform optimization if after legalize, and if NEON is available. We 9885 // also expected both operands to be BUILD_VECTORs. 9886 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON() 9887 || N0.getOpcode() != ISD::BUILD_VECTOR 9888 || N1.getOpcode() != ISD::BUILD_VECTOR) 9889 return SDValue(); 9890 9891 // Check output type since VPADDL operand elements can only be 8, 16, or 32. 9892 EVT VT = N->getValueType(0); 9893 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64) 9894 return SDValue(); 9895 9896 // Check that the vector operands are of the right form. 9897 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR 9898 // operands, where N is the size of the formed vector. 9899 // Each EXTRACT_VECTOR should have the same input vector and odd or even 9900 // index such that we have a pair wise add pattern. 9901 9902 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing. 9903 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 9904 return SDValue(); 9905 SDValue Vec = N0->getOperand(0)->getOperand(0); 9906 SDNode *V = Vec.getNode(); 9907 unsigned nextIndex = 0; 9908 9909 // For each operands to the ADD which are BUILD_VECTORs, 9910 // check to see if each of their operands are an EXTRACT_VECTOR with 9911 // the same vector and appropriate index. 9912 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) { 9913 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT 9914 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 9915 9916 SDValue ExtVec0 = N0->getOperand(i); 9917 SDValue ExtVec1 = N1->getOperand(i); 9918 9919 // First operand is the vector, verify its the same. 9920 if (V != ExtVec0->getOperand(0).getNode() || 9921 V != ExtVec1->getOperand(0).getNode()) 9922 return SDValue(); 9923 9924 // Second is the constant, verify its correct. 9925 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1)); 9926 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1)); 9927 9928 // For the constant, we want to see all the even or all the odd. 9929 if (!C0 || !C1 || C0->getZExtValue() != nextIndex 9930 || C1->getZExtValue() != nextIndex+1) 9931 return SDValue(); 9932 9933 // Increment index. 9934 nextIndex+=2; 9935 } else 9936 return SDValue(); 9937 } 9938 9939 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure 9940 // we're using the entire input vector, otherwise there's a size/legality 9941 // mismatch somewhere. 9942 if (nextIndex != Vec.getValueType().getVectorNumElements() || 9943 Vec.getValueType().getVectorElementType() == VT.getVectorElementType()) 9944 return SDValue(); 9945 9946 // Create VPADDL node. 9947 SelectionDAG &DAG = DCI.DAG; 9948 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9949 9950 SDLoc dl(N); 9951 9952 // Build operand list. 9953 SmallVector<SDValue, 8> Ops; 9954 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl, 9955 TLI.getPointerTy(DAG.getDataLayout()))); 9956 9957 // Input is the vector. 9958 Ops.push_back(Vec); 9959 9960 // Get widened type and narrowed type. 9961 MVT widenType; 9962 unsigned numElem = VT.getVectorNumElements(); 9963 9964 EVT inputLaneType = Vec.getValueType().getVectorElementType(); 9965 switch (inputLaneType.getSimpleVT().SimpleTy) { 9966 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break; 9967 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break; 9968 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break; 9969 default: 9970 llvm_unreachable("Invalid vector element type for padd optimization."); 9971 } 9972 9973 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops); 9974 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE; 9975 return DAG.getNode(ExtOp, dl, VT, tmp); 9976 } 9977 9978 static SDValue findMUL_LOHI(SDValue V) { 9979 if (V->getOpcode() == ISD::UMUL_LOHI || 9980 V->getOpcode() == ISD::SMUL_LOHI) 9981 return V; 9982 return SDValue(); 9983 } 9984 9985 static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, 9986 TargetLowering::DAGCombinerInfo &DCI, 9987 const ARMSubtarget *Subtarget) { 9988 if (Subtarget->isThumb()) { 9989 if (!Subtarget->hasDSP()) 9990 return SDValue(); 9991 } else if (!Subtarget->hasV5TEOps()) 9992 return SDValue(); 9993 9994 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and 9995 // accumulates the product into a 64-bit value. The 16-bit values will 9996 // be sign extended somehow or SRA'd into 32-bit values 9997 // (addc (adde (mul 16bit, 16bit), lo), hi) 9998 SDValue Mul = AddcNode->getOperand(0); 9999 SDValue Lo = AddcNode->getOperand(1); 10000 if (Mul.getOpcode() != ISD::MUL) { 10001 Lo = AddcNode->getOperand(0); 10002 Mul = AddcNode->getOperand(1); 10003 if (Mul.getOpcode() != ISD::MUL) 10004 return SDValue(); 10005 } 10006 10007 SDValue SRA = AddeNode->getOperand(0); 10008 SDValue Hi = AddeNode->getOperand(1); 10009 if (SRA.getOpcode() != ISD::SRA) { 10010 SRA = AddeNode->getOperand(1); 10011 Hi = AddeNode->getOperand(0); 10012 if (SRA.getOpcode() != ISD::SRA) 10013 return SDValue(); 10014 } 10015 if (auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) { 10016 if (Const->getZExtValue() != 31) 10017 return SDValue(); 10018 } else 10019 return SDValue(); 10020 10021 if (SRA.getOperand(0) != Mul) 10022 return SDValue(); 10023 10024 SelectionDAG &DAG = DCI.DAG; 10025 SDLoc dl(AddcNode); 10026 unsigned Opcode = 0; 10027 SDValue Op0; 10028 SDValue Op1; 10029 10030 if (isS16(Mul.getOperand(0), DAG) && isS16(Mul.getOperand(1), DAG)) { 10031 Opcode = ARMISD::SMLALBB; 10032 Op0 = Mul.getOperand(0); 10033 Op1 = Mul.getOperand(1); 10034 } else if (isS16(Mul.getOperand(0), DAG) && isSRA16(Mul.getOperand(1))) { 10035 Opcode = ARMISD::SMLALBT; 10036 Op0 = Mul.getOperand(0); 10037 Op1 = Mul.getOperand(1).getOperand(0); 10038 } else if (isSRA16(Mul.getOperand(0)) && isS16(Mul.getOperand(1), DAG)) { 10039 Opcode = ARMISD::SMLALTB; 10040 Op0 = Mul.getOperand(0).getOperand(0); 10041 Op1 = Mul.getOperand(1); 10042 } else if (isSRA16(Mul.getOperand(0)) && isSRA16(Mul.getOperand(1))) { 10043 Opcode = ARMISD::SMLALTT; 10044 Op0 = Mul->getOperand(0).getOperand(0); 10045 Op1 = Mul->getOperand(1).getOperand(0); 10046 } 10047 10048 if (!Op0 || !Op1) 10049 return SDValue(); 10050 10051 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32), 10052 Op0, Op1, Lo, Hi); 10053 // Replace the ADDs' nodes uses by the MLA node's values. 10054 SDValue HiMLALResult(SMLAL.getNode(), 1); 10055 SDValue LoMLALResult(SMLAL.getNode(), 0); 10056 10057 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult); 10058 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult); 10059 10060 // Return original node to notify the driver to stop replacing. 10061 SDValue resNode(AddcNode, 0); 10062 return resNode; 10063 } 10064 10065 static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, 10066 TargetLowering::DAGCombinerInfo &DCI, 10067 const ARMSubtarget *Subtarget) { 10068 // Look for multiply add opportunities. 10069 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where 10070 // each add nodes consumes a value from ISD::UMUL_LOHI and there is 10071 // a glue link from the first add to the second add. 10072 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by 10073 // a S/UMLAL instruction. 10074 // UMUL_LOHI 10075 // / :lo \ :hi 10076 // V \ [no multiline comment] 10077 // loAdd -> ADDC | 10078 // \ :carry / 10079 // V V 10080 // ADDE <- hiAdd 10081 // 10082 // In the special case where only the higher part of a signed result is used 10083 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts 10084 // a constant with the exact value of 0x80000000, we recognize we are dealing 10085 // with a "rounded multiply and add" (or subtract) and transform it into 10086 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively. 10087 10088 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE || 10089 AddeSubeNode->getOpcode() == ARMISD::SUBE) && 10090 "Expect an ADDE or SUBE"); 10091 10092 assert(AddeSubeNode->getNumOperands() == 3 && 10093 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 && 10094 "ADDE node has the wrong inputs"); 10095 10096 // Check that we are chained to the right ADDC or SUBC node. 10097 SDNode *AddcSubcNode = AddeSubeNode->getOperand(2).getNode(); 10098 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE && 10099 AddcSubcNode->getOpcode() != ARMISD::ADDC) || 10100 (AddeSubeNode->getOpcode() == ARMISD::SUBE && 10101 AddcSubcNode->getOpcode() != ARMISD::SUBC)) 10102 return SDValue(); 10103 10104 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0); 10105 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1); 10106 10107 // Check if the two operands are from the same mul_lohi node. 10108 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode()) 10109 return SDValue(); 10110 10111 assert(AddcSubcNode->getNumValues() == 2 && 10112 AddcSubcNode->getValueType(0) == MVT::i32 && 10113 "Expect ADDC with two result values. First: i32"); 10114 10115 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it 10116 // maybe a SMLAL which multiplies two 16-bit values. 10117 if (AddeSubeNode->getOpcode() == ARMISD::ADDE && 10118 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI && 10119 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI && 10120 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI && 10121 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI) 10122 return AddCombineTo64BitSMLAL16(AddcSubcNode, AddeSubeNode, DCI, Subtarget); 10123 10124 // Check for the triangle shape. 10125 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0); 10126 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1); 10127 10128 // Make sure that the ADDE/SUBE operands are not coming from the same node. 10129 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode()) 10130 return SDValue(); 10131 10132 // Find the MUL_LOHI node walking up ADDE/SUBE's operands. 10133 bool IsLeftOperandMUL = false; 10134 SDValue MULOp = findMUL_LOHI(AddeSubeOp0); 10135 if (MULOp == SDValue()) 10136 MULOp = findMUL_LOHI(AddeSubeOp1); 10137 else 10138 IsLeftOperandMUL = true; 10139 if (MULOp == SDValue()) 10140 return SDValue(); 10141 10142 // Figure out the right opcode. 10143 unsigned Opc = MULOp->getOpcode(); 10144 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL; 10145 10146 // Figure out the high and low input values to the MLAL node. 10147 SDValue *HiAddSub = nullptr; 10148 SDValue *LoMul = nullptr; 10149 SDValue *LowAddSub = nullptr; 10150 10151 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI. 10152 if ((AddeSubeOp0 != MULOp.getValue(1)) && (AddeSubeOp1 != MULOp.getValue(1))) 10153 return SDValue(); 10154 10155 if (IsLeftOperandMUL) 10156 HiAddSub = &AddeSubeOp1; 10157 else 10158 HiAddSub = &AddeSubeOp0; 10159 10160 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node 10161 // whose low result is fed to the ADDC/SUBC we are checking. 10162 10163 if (AddcSubcOp0 == MULOp.getValue(0)) { 10164 LoMul = &AddcSubcOp0; 10165 LowAddSub = &AddcSubcOp1; 10166 } 10167 if (AddcSubcOp1 == MULOp.getValue(0)) { 10168 LoMul = &AddcSubcOp1; 10169 LowAddSub = &AddcSubcOp0; 10170 } 10171 10172 if (!LoMul) 10173 return SDValue(); 10174 10175 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC 10176 // the replacement below will create a cycle. 10177 if (AddcSubcNode == HiAddSub->getNode() || 10178 AddcSubcNode->isPredecessorOf(HiAddSub->getNode())) 10179 return SDValue(); 10180 10181 // Create the merged node. 10182 SelectionDAG &DAG = DCI.DAG; 10183 10184 // Start building operand list. 10185 SmallVector<SDValue, 8> Ops; 10186 Ops.push_back(LoMul->getOperand(0)); 10187 Ops.push_back(LoMul->getOperand(1)); 10188 10189 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be 10190 // the case, we must be doing signed multiplication and only use the higher 10191 // part of the result of the MLAL, furthermore the LowAddSub must be a constant 10192 // addition or subtraction with the value of 0x800000. 10193 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() && 10194 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(1) && 10195 LowAddSub->getNode()->getOpcode() == ISD::Constant && 10196 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() == 10197 0x80000000) { 10198 Ops.push_back(*HiAddSub); 10199 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) { 10200 FinalOpc = ARMISD::SMMLSR; 10201 } else { 10202 FinalOpc = ARMISD::SMMLAR; 10203 } 10204 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops); 10205 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode); 10206 10207 return SDValue(AddeSubeNode, 0); 10208 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC) 10209 // SMMLS is generated during instruction selection and the rest of this 10210 // function can not handle the case where AddcSubcNode is a SUBC. 10211 return SDValue(); 10212 10213 // Finish building the operand list for {U/S}MLAL 10214 Ops.push_back(*LowAddSub); 10215 Ops.push_back(*HiAddSub); 10216 10217 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), 10218 DAG.getVTList(MVT::i32, MVT::i32), Ops); 10219 10220 // Replace the ADDs' nodes uses by the MLA node's values. 10221 SDValue HiMLALResult(MLALNode.getNode(), 1); 10222 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult); 10223 10224 SDValue LoMLALResult(MLALNode.getNode(), 0); 10225 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult); 10226 10227 // Return original node to notify the driver to stop replacing. 10228 return SDValue(AddeSubeNode, 0); 10229 } 10230 10231 static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, 10232 TargetLowering::DAGCombinerInfo &DCI, 10233 const ARMSubtarget *Subtarget) { 10234 // UMAAL is similar to UMLAL except that it adds two unsigned values. 10235 // While trying to combine for the other MLAL nodes, first search for the 10236 // chance to use UMAAL. Check if Addc uses a node which has already 10237 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde 10238 // as the addend, and it's handled in PerformUMLALCombine. 10239 10240 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) 10241 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget); 10242 10243 // Check that we have a glued ADDC node. 10244 SDNode* AddcNode = AddeNode->getOperand(2).getNode(); 10245 if (AddcNode->getOpcode() != ARMISD::ADDC) 10246 return SDValue(); 10247 10248 // Find the converted UMAAL or quit if it doesn't exist. 10249 SDNode *UmlalNode = nullptr; 10250 SDValue AddHi; 10251 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) { 10252 UmlalNode = AddcNode->getOperand(0).getNode(); 10253 AddHi = AddcNode->getOperand(1); 10254 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) { 10255 UmlalNode = AddcNode->getOperand(1).getNode(); 10256 AddHi = AddcNode->getOperand(0); 10257 } else { 10258 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget); 10259 } 10260 10261 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as 10262 // the ADDC as well as Zero. 10263 if (!isNullConstant(UmlalNode->getOperand(3))) 10264 return SDValue(); 10265 10266 if ((isNullConstant(AddeNode->getOperand(0)) && 10267 AddeNode->getOperand(1).getNode() == UmlalNode) || 10268 (AddeNode->getOperand(0).getNode() == UmlalNode && 10269 isNullConstant(AddeNode->getOperand(1)))) { 10270 SelectionDAG &DAG = DCI.DAG; 10271 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1), 10272 UmlalNode->getOperand(2), AddHi }; 10273 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode), 10274 DAG.getVTList(MVT::i32, MVT::i32), Ops); 10275 10276 // Replace the ADDs' nodes uses by the UMAAL node's values. 10277 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1)); 10278 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0)); 10279 10280 // Return original node to notify the driver to stop replacing. 10281 return SDValue(AddeNode, 0); 10282 } 10283 return SDValue(); 10284 } 10285 10286 static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, 10287 const ARMSubtarget *Subtarget) { 10288 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) 10289 return SDValue(); 10290 10291 // Check that we have a pair of ADDC and ADDE as operands. 10292 // Both addends of the ADDE must be zero. 10293 SDNode* AddcNode = N->getOperand(2).getNode(); 10294 SDNode* AddeNode = N->getOperand(3).getNode(); 10295 if ((AddcNode->getOpcode() == ARMISD::ADDC) && 10296 (AddeNode->getOpcode() == ARMISD::ADDE) && 10297 isNullConstant(AddeNode->getOperand(0)) && 10298 isNullConstant(AddeNode->getOperand(1)) && 10299 (AddeNode->getOperand(2).getNode() == AddcNode)) 10300 return DAG.getNode(ARMISD::UMAAL, SDLoc(N), 10301 DAG.getVTList(MVT::i32, MVT::i32), 10302 {N->getOperand(0), N->getOperand(1), 10303 AddcNode->getOperand(0), AddcNode->getOperand(1)}); 10304 else 10305 return SDValue(); 10306 } 10307 10308 static SDValue PerformAddcSubcCombine(SDNode *N, 10309 TargetLowering::DAGCombinerInfo &DCI, 10310 const ARMSubtarget *Subtarget) { 10311 SelectionDAG &DAG(DCI.DAG); 10312 10313 if (N->getOpcode() == ARMISD::SUBC) { 10314 // (SUBC (ADDE 0, 0, C), 1) -> C 10315 SDValue LHS = N->getOperand(0); 10316 SDValue RHS = N->getOperand(1); 10317 if (LHS->getOpcode() == ARMISD::ADDE && 10318 isNullConstant(LHS->getOperand(0)) && 10319 isNullConstant(LHS->getOperand(1)) && isOneConstant(RHS)) { 10320 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2)); 10321 } 10322 } 10323 10324 if (Subtarget->isThumb1Only()) { 10325 SDValue RHS = N->getOperand(1); 10326 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { 10327 int32_t imm = C->getSExtValue(); 10328 if (imm < 0 && imm > std::numeric_limits<int>::min()) { 10329 SDLoc DL(N); 10330 RHS = DAG.getConstant(-imm, DL, MVT::i32); 10331 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC 10332 : ARMISD::ADDC; 10333 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS); 10334 } 10335 } 10336 } 10337 10338 return SDValue(); 10339 } 10340 10341 static SDValue PerformAddeSubeCombine(SDNode *N, 10342 TargetLowering::DAGCombinerInfo &DCI, 10343 const ARMSubtarget *Subtarget) { 10344 if (Subtarget->isThumb1Only()) { 10345 SelectionDAG &DAG = DCI.DAG; 10346 SDValue RHS = N->getOperand(1); 10347 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { 10348 int64_t imm = C->getSExtValue(); 10349 if (imm < 0) { 10350 SDLoc DL(N); 10351 10352 // The with-carry-in form matches bitwise not instead of the negation. 10353 // Effectively, the inverse interpretation of the carry flag already 10354 // accounts for part of the negation. 10355 RHS = DAG.getConstant(~imm, DL, MVT::i32); 10356 10357 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE 10358 : ARMISD::ADDE; 10359 return DAG.getNode(Opcode, DL, N->getVTList(), 10360 N->getOperand(0), RHS, N->getOperand(2)); 10361 } 10362 } 10363 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) { 10364 return AddCombineTo64bitMLAL(N, DCI, Subtarget); 10365 } 10366 return SDValue(); 10367 } 10368 10369 /// PerformADDECombine - Target-specific dag combine transform from 10370 /// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or 10371 /// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL 10372 static SDValue PerformADDECombine(SDNode *N, 10373 TargetLowering::DAGCombinerInfo &DCI, 10374 const ARMSubtarget *Subtarget) { 10375 // Only ARM and Thumb2 support UMLAL/SMLAL. 10376 if (Subtarget->isThumb1Only()) 10377 return PerformAddeSubeCombine(N, DCI, Subtarget); 10378 10379 // Only perform the checks after legalize when the pattern is available. 10380 if (DCI.isBeforeLegalize()) return SDValue(); 10381 10382 return AddCombineTo64bitUMAAL(N, DCI, Subtarget); 10383 } 10384 10385 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with 10386 /// operands N0 and N1. This is a helper for PerformADDCombine that is 10387 /// called with the default operands, and if that fails, with commuted 10388 /// operands. 10389 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, 10390 TargetLowering::DAGCombinerInfo &DCI, 10391 const ARMSubtarget *Subtarget){ 10392 // Attempt to create vpadd for this add. 10393 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget)) 10394 return Result; 10395 10396 // Attempt to create vpaddl for this add. 10397 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget)) 10398 return Result; 10399 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI, 10400 Subtarget)) 10401 return Result; 10402 10403 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) 10404 if (N0.getNode()->hasOneUse()) 10405 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI)) 10406 return Result; 10407 return SDValue(); 10408 } 10409 10410 static SDValue PerformSHLSimplify(SDNode *N, 10411 TargetLowering::DAGCombinerInfo &DCI, 10412 const ARMSubtarget *ST) { 10413 // Allow the generic combiner to identify potential bswaps. 10414 if (DCI.isBeforeLegalize()) 10415 return SDValue(); 10416 10417 // DAG combiner will fold: 10418 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2) 10419 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2 10420 // Other code patterns that can be also be modified have the following form: 10421 // b + ((a << 1) | 510) 10422 // b + ((a << 1) & 510) 10423 // b + ((a << 1) ^ 510) 10424 // b + ((a << 1) + 510) 10425 10426 // Many instructions can perform the shift for free, but it requires both 10427 // the operands to be registers. If c1 << c2 is too large, a mov immediate 10428 // instruction will needed. So, unfold back to the original pattern if: 10429 // - if c1 and c2 are small enough that they don't require mov imms. 10430 // - the user(s) of the node can perform an shl 10431 10432 // No shifted operands for 16-bit instructions. 10433 if (ST->isThumb() && ST->isThumb1Only()) 10434 return SDValue(); 10435 10436 // Check that all the users could perform the shl themselves. 10437 for (auto U : N->uses()) { 10438 switch(U->getOpcode()) { 10439 default: 10440 return SDValue(); 10441 case ISD::SUB: 10442 case ISD::ADD: 10443 case ISD::AND: 10444 case ISD::OR: 10445 case ISD::XOR: 10446 case ISD::SETCC: 10447 case ARMISD::CMP: 10448 // Check that the user isn't already using a constant because there 10449 // aren't any instructions that support an immediate operand and a 10450 // shifted operand. 10451 if (isa<ConstantSDNode>(U->getOperand(0)) || 10452 isa<ConstantSDNode>(U->getOperand(1))) 10453 return SDValue(); 10454 10455 // Check that it's not already using a shift. 10456 if (U->getOperand(0).getOpcode() == ISD::SHL || 10457 U->getOperand(1).getOpcode() == ISD::SHL) 10458 return SDValue(); 10459 break; 10460 } 10461 } 10462 10463 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR && 10464 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND) 10465 return SDValue(); 10466 10467 if (N->getOperand(0).getOpcode() != ISD::SHL) 10468 return SDValue(); 10469 10470 SDValue SHL = N->getOperand(0); 10471 10472 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10473 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1)); 10474 if (!C1ShlC2 || !C2) 10475 return SDValue(); 10476 10477 APInt C2Int = C2->getAPIntValue(); 10478 APInt C1Int = C1ShlC2->getAPIntValue(); 10479 10480 // Check that performing a lshr will not lose any information. 10481 APInt Mask = APInt::getHighBitsSet(C2Int.getBitWidth(), 10482 C2Int.getBitWidth() - C2->getZExtValue()); 10483 if ((C1Int & Mask) != C1Int) 10484 return SDValue(); 10485 10486 // Shift the first constant. 10487 C1Int.lshrInPlace(C2Int); 10488 10489 // The immediates are encoded as an 8-bit value that can be rotated. 10490 auto LargeImm = [](const APInt &Imm) { 10491 unsigned Zeros = Imm.countLeadingZeros() + Imm.countTrailingZeros(); 10492 return Imm.getBitWidth() - Zeros > 8; 10493 }; 10494 10495 if (LargeImm(C1Int) || LargeImm(C2Int)) 10496 return SDValue(); 10497 10498 SelectionDAG &DAG = DCI.DAG; 10499 SDLoc dl(N); 10500 SDValue X = SHL.getOperand(0); 10501 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X, 10502 DAG.getConstant(C1Int, dl, MVT::i32)); 10503 // Shift left to compensate for the lshr of C1Int. 10504 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1)); 10505 10506 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump(); 10507 SHL.dump(); N->dump()); 10508 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump()); 10509 10510 DAG.ReplaceAllUsesWith(SDValue(N, 0), Res); 10511 return SDValue(N, 0); 10512 } 10513 10514 10515 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD. 10516 /// 10517 static SDValue PerformADDCombine(SDNode *N, 10518 TargetLowering::DAGCombinerInfo &DCI, 10519 const ARMSubtarget *Subtarget) { 10520 SDValue N0 = N->getOperand(0); 10521 SDValue N1 = N->getOperand(1); 10522 10523 // Only works one way, because it needs an immediate operand. 10524 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget)) 10525 return Result; 10526 10527 // First try with the default operand order. 10528 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget)) 10529 return Result; 10530 10531 // If that didn't work, try again with the operands commuted. 10532 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget); 10533 } 10534 10535 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB. 10536 /// 10537 static SDValue PerformSUBCombine(SDNode *N, 10538 TargetLowering::DAGCombinerInfo &DCI) { 10539 SDValue N0 = N->getOperand(0); 10540 SDValue N1 = N->getOperand(1); 10541 10542 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) 10543 if (N1.getNode()->hasOneUse()) 10544 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI)) 10545 return Result; 10546 10547 return SDValue(); 10548 } 10549 10550 /// PerformVMULCombine 10551 /// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the 10552 /// special multiplier accumulator forwarding. 10553 /// vmul d3, d0, d2 10554 /// vmla d3, d1, d2 10555 /// is faster than 10556 /// vadd d3, d0, d1 10557 /// vmul d3, d3, d2 10558 // However, for (A + B) * (A + B), 10559 // vadd d2, d0, d1 10560 // vmul d3, d0, d2 10561 // vmla d3, d1, d2 10562 // is slower than 10563 // vadd d2, d0, d1 10564 // vmul d3, d2, d2 10565 static SDValue PerformVMULCombine(SDNode *N, 10566 TargetLowering::DAGCombinerInfo &DCI, 10567 const ARMSubtarget *Subtarget) { 10568 if (!Subtarget->hasVMLxForwarding()) 10569 return SDValue(); 10570 10571 SelectionDAG &DAG = DCI.DAG; 10572 SDValue N0 = N->getOperand(0); 10573 SDValue N1 = N->getOperand(1); 10574 unsigned Opcode = N0.getOpcode(); 10575 if (Opcode != ISD::ADD && Opcode != ISD::SUB && 10576 Opcode != ISD::FADD && Opcode != ISD::FSUB) { 10577 Opcode = N1.getOpcode(); 10578 if (Opcode != ISD::ADD && Opcode != ISD::SUB && 10579 Opcode != ISD::FADD && Opcode != ISD::FSUB) 10580 return SDValue(); 10581 std::swap(N0, N1); 10582 } 10583 10584 if (N0 == N1) 10585 return SDValue(); 10586 10587 EVT VT = N->getValueType(0); 10588 SDLoc DL(N); 10589 SDValue N00 = N0->getOperand(0); 10590 SDValue N01 = N0->getOperand(1); 10591 return DAG.getNode(Opcode, DL, VT, 10592 DAG.getNode(ISD::MUL, DL, VT, N00, N1), 10593 DAG.getNode(ISD::MUL, DL, VT, N01, N1)); 10594 } 10595 10596 static SDValue PerformMULCombine(SDNode *N, 10597 TargetLowering::DAGCombinerInfo &DCI, 10598 const ARMSubtarget *Subtarget) { 10599 SelectionDAG &DAG = DCI.DAG; 10600 10601 if (Subtarget->isThumb1Only()) 10602 return SDValue(); 10603 10604 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 10605 return SDValue(); 10606 10607 EVT VT = N->getValueType(0); 10608 if (VT.is64BitVector() || VT.is128BitVector()) 10609 return PerformVMULCombine(N, DCI, Subtarget); 10610 if (VT != MVT::i32) 10611 return SDValue(); 10612 10613 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10614 if (!C) 10615 return SDValue(); 10616 10617 int64_t MulAmt = C->getSExtValue(); 10618 unsigned ShiftAmt = countTrailingZeros<uint64_t>(MulAmt); 10619 10620 ShiftAmt = ShiftAmt & (32 - 1); 10621 SDValue V = N->getOperand(0); 10622 SDLoc DL(N); 10623 10624 SDValue Res; 10625 MulAmt >>= ShiftAmt; 10626 10627 if (MulAmt >= 0) { 10628 if (isPowerOf2_32(MulAmt - 1)) { 10629 // (mul x, 2^N + 1) => (add (shl x, N), x) 10630 Res = DAG.getNode(ISD::ADD, DL, VT, 10631 V, 10632 DAG.getNode(ISD::SHL, DL, VT, 10633 V, 10634 DAG.getConstant(Log2_32(MulAmt - 1), DL, 10635 MVT::i32))); 10636 } else if (isPowerOf2_32(MulAmt + 1)) { 10637 // (mul x, 2^N - 1) => (sub (shl x, N), x) 10638 Res = DAG.getNode(ISD::SUB, DL, VT, 10639 DAG.getNode(ISD::SHL, DL, VT, 10640 V, 10641 DAG.getConstant(Log2_32(MulAmt + 1), DL, 10642 MVT::i32)), 10643 V); 10644 } else 10645 return SDValue(); 10646 } else { 10647 uint64_t MulAmtAbs = -MulAmt; 10648 if (isPowerOf2_32(MulAmtAbs + 1)) { 10649 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) 10650 Res = DAG.getNode(ISD::SUB, DL, VT, 10651 V, 10652 DAG.getNode(ISD::SHL, DL, VT, 10653 V, 10654 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL, 10655 MVT::i32))); 10656 } else if (isPowerOf2_32(MulAmtAbs - 1)) { 10657 // (mul x, -(2^N + 1)) => - (add (shl x, N), x) 10658 Res = DAG.getNode(ISD::ADD, DL, VT, 10659 V, 10660 DAG.getNode(ISD::SHL, DL, VT, 10661 V, 10662 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL, 10663 MVT::i32))); 10664 Res = DAG.getNode(ISD::SUB, DL, VT, 10665 DAG.getConstant(0, DL, MVT::i32), Res); 10666 } else 10667 return SDValue(); 10668 } 10669 10670 if (ShiftAmt != 0) 10671 Res = DAG.getNode(ISD::SHL, DL, VT, 10672 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32)); 10673 10674 // Do not add new nodes to DAG combiner worklist. 10675 DCI.CombineTo(N, Res, false); 10676 return SDValue(); 10677 } 10678 10679 static SDValue CombineANDShift(SDNode *N, 10680 TargetLowering::DAGCombinerInfo &DCI, 10681 const ARMSubtarget *Subtarget) { 10682 // Allow DAGCombine to pattern-match before we touch the canonical form. 10683 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 10684 return SDValue(); 10685 10686 if (N->getValueType(0) != MVT::i32) 10687 return SDValue(); 10688 10689 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10690 if (!N1C) 10691 return SDValue(); 10692 10693 uint32_t C1 = (uint32_t)N1C->getZExtValue(); 10694 // Don't transform uxtb/uxth. 10695 if (C1 == 255 || C1 == 65535) 10696 return SDValue(); 10697 10698 SDNode *N0 = N->getOperand(0).getNode(); 10699 if (!N0->hasOneUse()) 10700 return SDValue(); 10701 10702 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL) 10703 return SDValue(); 10704 10705 bool LeftShift = N0->getOpcode() == ISD::SHL; 10706 10707 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 10708 if (!N01C) 10709 return SDValue(); 10710 10711 uint32_t C2 = (uint32_t)N01C->getZExtValue(); 10712 if (!C2 || C2 >= 32) 10713 return SDValue(); 10714 10715 SelectionDAG &DAG = DCI.DAG; 10716 SDLoc DL(N); 10717 10718 // We have a pattern of the form "(and (shl x, c2) c1)" or 10719 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to 10720 // transform to a pair of shifts, to save materializing c1. 10721 10722 // First pattern: right shift, and c1+1 is a power of two. 10723 // FIXME: Also check reversed pattern (left shift, and ~c1+1 is a power 10724 // of two). 10725 // FIXME: Use demanded bits? 10726 if (!LeftShift && isMask_32(C1)) { 10727 uint32_t C3 = countLeadingZeros(C1); 10728 if (C2 < C3) { 10729 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), 10730 DAG.getConstant(C3 - C2, DL, MVT::i32)); 10731 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL, 10732 DAG.getConstant(C3, DL, MVT::i32)); 10733 } 10734 } 10735 10736 // Second pattern: left shift, and (c1>>c2)+1 is a power of two. 10737 // FIXME: Also check reversed pattern (right shift, and ~(c1<<c2)+1 10738 // is a power of two). 10739 // FIXME: Use demanded bits? 10740 if (LeftShift && isShiftedMask_32(C1)) { 10741 uint32_t C3 = countLeadingZeros(C1); 10742 if (C2 + C3 < 32 && C1 == ((-1U << (C2 + C3)) >> C3)) { 10743 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), 10744 DAG.getConstant(C2 + C3, DL, MVT::i32)); 10745 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL, 10746 DAG.getConstant(C3, DL, MVT::i32)); 10747 } 10748 } 10749 10750 // FIXME: Transform "(and (shl x, c2) c1)" -> 10751 // "(shl (and x, c1>>c2), c2)" if "c1 >> c2" is a cheaper immediate than 10752 // c1. 10753 return SDValue(); 10754 } 10755 10756 static SDValue PerformANDCombine(SDNode *N, 10757 TargetLowering::DAGCombinerInfo &DCI, 10758 const ARMSubtarget *Subtarget) { 10759 // Attempt to use immediate-form VBIC 10760 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); 10761 SDLoc dl(N); 10762 EVT VT = N->getValueType(0); 10763 SelectionDAG &DAG = DCI.DAG; 10764 10765 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 10766 return SDValue(); 10767 10768 APInt SplatBits, SplatUndef; 10769 unsigned SplatBitSize; 10770 bool HasAnyUndefs; 10771 if (BVN && 10772 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { 10773 if (SplatBitSize <= 64) { 10774 EVT VbicVT; 10775 SDValue Val = isNEONModifiedImm((~SplatBits).getZExtValue(), 10776 SplatUndef.getZExtValue(), SplatBitSize, 10777 DAG, dl, VbicVT, VT.is128BitVector(), 10778 OtherModImm); 10779 if (Val.getNode()) { 10780 SDValue Input = 10781 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0)); 10782 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val); 10783 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic); 10784 } 10785 } 10786 } 10787 10788 if (!Subtarget->isThumb1Only()) { 10789 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) 10790 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI)) 10791 return Result; 10792 10793 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget)) 10794 return Result; 10795 } 10796 10797 if (Subtarget->isThumb1Only()) 10798 if (SDValue Result = CombineANDShift(N, DCI, Subtarget)) 10799 return Result; 10800 10801 return SDValue(); 10802 } 10803 10804 // Try combining OR nodes to SMULWB, SMULWT. 10805 static SDValue PerformORCombineToSMULWBT(SDNode *OR, 10806 TargetLowering::DAGCombinerInfo &DCI, 10807 const ARMSubtarget *Subtarget) { 10808 if (!Subtarget->hasV6Ops() || 10809 (Subtarget->isThumb() && 10810 (!Subtarget->hasThumb2() || !Subtarget->hasDSP()))) 10811 return SDValue(); 10812 10813 SDValue SRL = OR->getOperand(0); 10814 SDValue SHL = OR->getOperand(1); 10815 10816 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) { 10817 SRL = OR->getOperand(1); 10818 SHL = OR->getOperand(0); 10819 } 10820 if (!isSRL16(SRL) || !isSHL16(SHL)) 10821 return SDValue(); 10822 10823 // The first operands to the shifts need to be the two results from the 10824 // same smul_lohi node. 10825 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) || 10826 SRL.getOperand(0).getOpcode() != ISD::SMUL_LOHI) 10827 return SDValue(); 10828 10829 SDNode *SMULLOHI = SRL.getOperand(0).getNode(); 10830 if (SRL.getOperand(0) != SDValue(SMULLOHI, 0) || 10831 SHL.getOperand(0) != SDValue(SMULLOHI, 1)) 10832 return SDValue(); 10833 10834 // Now we have: 10835 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16))) 10836 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments. 10837 // For SMUWB the 16-bit value will signed extended somehow. 10838 // For SMULWT only the SRA is required. 10839 // Check both sides of SMUL_LOHI 10840 SDValue OpS16 = SMULLOHI->getOperand(0); 10841 SDValue OpS32 = SMULLOHI->getOperand(1); 10842 10843 SelectionDAG &DAG = DCI.DAG; 10844 if (!isS16(OpS16, DAG) && !isSRA16(OpS16)) { 10845 OpS16 = OpS32; 10846 OpS32 = SMULLOHI->getOperand(0); 10847 } 10848 10849 SDLoc dl(OR); 10850 unsigned Opcode = 0; 10851 if (isS16(OpS16, DAG)) 10852 Opcode = ARMISD::SMULWB; 10853 else if (isSRA16(OpS16)) { 10854 Opcode = ARMISD::SMULWT; 10855 OpS16 = OpS16->getOperand(0); 10856 } 10857 else 10858 return SDValue(); 10859 10860 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16); 10861 DAG.ReplaceAllUsesOfValueWith(SDValue(OR, 0), Res); 10862 return SDValue(OR, 0); 10863 } 10864 10865 static SDValue PerformORCombineToBFI(SDNode *N, 10866 TargetLowering::DAGCombinerInfo &DCI, 10867 const ARMSubtarget *Subtarget) { 10868 // BFI is only available on V6T2+ 10869 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops()) 10870 return SDValue(); 10871 10872 EVT VT = N->getValueType(0); 10873 SDValue N0 = N->getOperand(0); 10874 SDValue N1 = N->getOperand(1); 10875 SelectionDAG &DAG = DCI.DAG; 10876 SDLoc DL(N); 10877 // 1) or (and A, mask), val => ARMbfi A, val, mask 10878 // iff (val & mask) == val 10879 // 10880 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask 10881 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2) 10882 // && mask == ~mask2 10883 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2) 10884 // && ~mask == mask2 10885 // (i.e., copy a bitfield value into another bitfield of the same width) 10886 10887 if (VT != MVT::i32) 10888 return SDValue(); 10889 10890 SDValue N00 = N0.getOperand(0); 10891 10892 // The value and the mask need to be constants so we can verify this is 10893 // actually a bitfield set. If the mask is 0xffff, we can do better 10894 // via a movt instruction, so don't use BFI in that case. 10895 SDValue MaskOp = N0.getOperand(1); 10896 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(MaskOp); 10897 if (!MaskC) 10898 return SDValue(); 10899 unsigned Mask = MaskC->getZExtValue(); 10900 if (Mask == 0xffff) 10901 return SDValue(); 10902 SDValue Res; 10903 // Case (1): or (and A, mask), val => ARMbfi A, val, mask 10904 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 10905 if (N1C) { 10906 unsigned Val = N1C->getZExtValue(); 10907 if ((Val & ~Mask) != Val) 10908 return SDValue(); 10909 10910 if (ARM::isBitFieldInvertedMask(Mask)) { 10911 Val >>= countTrailingZeros(~Mask); 10912 10913 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, 10914 DAG.getConstant(Val, DL, MVT::i32), 10915 DAG.getConstant(Mask, DL, MVT::i32)); 10916 10917 DCI.CombineTo(N, Res, false); 10918 // Return value from the original node to inform the combiner than N is 10919 // now dead. 10920 return SDValue(N, 0); 10921 } 10922 } else if (N1.getOpcode() == ISD::AND) { 10923 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask 10924 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1)); 10925 if (!N11C) 10926 return SDValue(); 10927 unsigned Mask2 = N11C->getZExtValue(); 10928 10929 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern 10930 // as is to match. 10931 if (ARM::isBitFieldInvertedMask(Mask) && 10932 (Mask == ~Mask2)) { 10933 // The pack halfword instruction works better for masks that fit it, 10934 // so use that when it's available. 10935 if (Subtarget->hasDSP() && 10936 (Mask == 0xffff || Mask == 0xffff0000)) 10937 return SDValue(); 10938 // 2a 10939 unsigned amt = countTrailingZeros(Mask2); 10940 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0), 10941 DAG.getConstant(amt, DL, MVT::i32)); 10942 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res, 10943 DAG.getConstant(Mask, DL, MVT::i32)); 10944 DCI.CombineTo(N, Res, false); 10945 // Return value from the original node to inform the combiner than N is 10946 // now dead. 10947 return SDValue(N, 0); 10948 } else if (ARM::isBitFieldInvertedMask(~Mask) && 10949 (~Mask == Mask2)) { 10950 // The pack halfword instruction works better for masks that fit it, 10951 // so use that when it's available. 10952 if (Subtarget->hasDSP() && 10953 (Mask2 == 0xffff || Mask2 == 0xffff0000)) 10954 return SDValue(); 10955 // 2b 10956 unsigned lsb = countTrailingZeros(Mask); 10957 Res = DAG.getNode(ISD::SRL, DL, VT, N00, 10958 DAG.getConstant(lsb, DL, MVT::i32)); 10959 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res, 10960 DAG.getConstant(Mask2, DL, MVT::i32)); 10961 DCI.CombineTo(N, Res, false); 10962 // Return value from the original node to inform the combiner than N is 10963 // now dead. 10964 return SDValue(N, 0); 10965 } 10966 } 10967 10968 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) && 10969 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) && 10970 ARM::isBitFieldInvertedMask(~Mask)) { 10971 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask 10972 // where lsb(mask) == #shamt and masked bits of B are known zero. 10973 SDValue ShAmt = N00.getOperand(1); 10974 unsigned ShAmtC = cast<ConstantSDNode>(ShAmt)->getZExtValue(); 10975 unsigned LSB = countTrailingZeros(Mask); 10976 if (ShAmtC != LSB) 10977 return SDValue(); 10978 10979 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0), 10980 DAG.getConstant(~Mask, DL, MVT::i32)); 10981 10982 DCI.CombineTo(N, Res, false); 10983 // Return value from the original node to inform the combiner than N is 10984 // now dead. 10985 return SDValue(N, 0); 10986 } 10987 10988 return SDValue(); 10989 } 10990 10991 /// PerformORCombine - Target-specific dag combine xforms for ISD::OR 10992 static SDValue PerformORCombine(SDNode *N, 10993 TargetLowering::DAGCombinerInfo &DCI, 10994 const ARMSubtarget *Subtarget) { 10995 // Attempt to use immediate-form VORR 10996 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); 10997 SDLoc dl(N); 10998 EVT VT = N->getValueType(0); 10999 SelectionDAG &DAG = DCI.DAG; 11000 11001 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 11002 return SDValue(); 11003 11004 APInt SplatBits, SplatUndef; 11005 unsigned SplatBitSize; 11006 bool HasAnyUndefs; 11007 if (BVN && Subtarget->hasNEON() && 11008 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { 11009 if (SplatBitSize <= 64) { 11010 EVT VorrVT; 11011 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(), 11012 SplatUndef.getZExtValue(), SplatBitSize, 11013 DAG, dl, VorrVT, VT.is128BitVector(), 11014 OtherModImm); 11015 if (Val.getNode()) { 11016 SDValue Input = 11017 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0)); 11018 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val); 11019 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr); 11020 } 11021 } 11022 } 11023 11024 if (!Subtarget->isThumb1Only()) { 11025 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c)) 11026 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI)) 11027 return Result; 11028 if (SDValue Result = PerformORCombineToSMULWBT(N, DCI, Subtarget)) 11029 return Result; 11030 } 11031 11032 SDValue N0 = N->getOperand(0); 11033 SDValue N1 = N->getOperand(1); 11034 11035 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant. 11036 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() && 11037 DAG.getTargetLoweringInfo().isTypeLegal(VT)) { 11038 11039 // The code below optimizes (or (and X, Y), Z). 11040 // The AND operand needs to have a single user to make these optimizations 11041 // profitable. 11042 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse()) 11043 return SDValue(); 11044 11045 APInt SplatUndef; 11046 unsigned SplatBitSize; 11047 bool HasAnyUndefs; 11048 11049 APInt SplatBits0, SplatBits1; 11050 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1)); 11051 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1)); 11052 // Ensure that the second operand of both ands are constants 11053 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize, 11054 HasAnyUndefs) && !HasAnyUndefs) { 11055 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize, 11056 HasAnyUndefs) && !HasAnyUndefs) { 11057 // Ensure that the bit width of the constants are the same and that 11058 // the splat arguments are logical inverses as per the pattern we 11059 // are trying to simplify. 11060 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() && 11061 SplatBits0 == ~SplatBits1) { 11062 // Canonicalize the vector type to make instruction selection 11063 // simpler. 11064 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; 11065 SDValue Result = DAG.getNode(ARMISD::VBSL, dl, CanonicalVT, 11066 N0->getOperand(1), 11067 N0->getOperand(0), 11068 N1->getOperand(0)); 11069 return DAG.getNode(ISD::BITCAST, dl, VT, Result); 11070 } 11071 } 11072 } 11073 } 11074 11075 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when 11076 // reasonable. 11077 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) { 11078 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget)) 11079 return Res; 11080 } 11081 11082 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget)) 11083 return Result; 11084 11085 return SDValue(); 11086 } 11087 11088 static SDValue PerformXORCombine(SDNode *N, 11089 TargetLowering::DAGCombinerInfo &DCI, 11090 const ARMSubtarget *Subtarget) { 11091 EVT VT = N->getValueType(0); 11092 SelectionDAG &DAG = DCI.DAG; 11093 11094 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 11095 return SDValue(); 11096 11097 if (!Subtarget->isThumb1Only()) { 11098 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c)) 11099 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI)) 11100 return Result; 11101 11102 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget)) 11103 return Result; 11104 } 11105 11106 return SDValue(); 11107 } 11108 11109 // ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it, 11110 // and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and 11111 // their position in "to" (Rd). 11112 static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) { 11113 assert(N->getOpcode() == ARMISD::BFI); 11114 11115 SDValue From = N->getOperand(1); 11116 ToMask = ~cast<ConstantSDNode>(N->getOperand(2))->getAPIntValue(); 11117 FromMask = APInt::getLowBitsSet(ToMask.getBitWidth(), ToMask.countPopulation()); 11118 11119 // If the Base came from a SHR #C, we can deduce that it is really testing bit 11120 // #C in the base of the SHR. 11121 if (From->getOpcode() == ISD::SRL && 11122 isa<ConstantSDNode>(From->getOperand(1))) { 11123 APInt Shift = cast<ConstantSDNode>(From->getOperand(1))->getAPIntValue(); 11124 assert(Shift.getLimitedValue() < 32 && "Shift too large!"); 11125 FromMask <<= Shift.getLimitedValue(31); 11126 From = From->getOperand(0); 11127 } 11128 11129 return From; 11130 } 11131 11132 // If A and B contain one contiguous set of bits, does A | B == A . B? 11133 // 11134 // Neither A nor B must be zero. 11135 static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) { 11136 unsigned LastActiveBitInA = A.countTrailingZeros(); 11137 unsigned FirstActiveBitInB = B.getBitWidth() - B.countLeadingZeros() - 1; 11138 return LastActiveBitInA - 1 == FirstActiveBitInB; 11139 } 11140 11141 static SDValue FindBFIToCombineWith(SDNode *N) { 11142 // We have a BFI in N. Follow a possible chain of BFIs and find a BFI it can combine with, 11143 // if one exists. 11144 APInt ToMask, FromMask; 11145 SDValue From = ParseBFI(N, ToMask, FromMask); 11146 SDValue To = N->getOperand(0); 11147 11148 // Now check for a compatible BFI to merge with. We can pass through BFIs that 11149 // aren't compatible, but not if they set the same bit in their destination as 11150 // we do (or that of any BFI we're going to combine with). 11151 SDValue V = To; 11152 APInt CombinedToMask = ToMask; 11153 while (V.getOpcode() == ARMISD::BFI) { 11154 APInt NewToMask, NewFromMask; 11155 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask); 11156 if (NewFrom != From) { 11157 // This BFI has a different base. Keep going. 11158 CombinedToMask |= NewToMask; 11159 V = V.getOperand(0); 11160 continue; 11161 } 11162 11163 // Do the written bits conflict with any we've seen so far? 11164 if ((NewToMask & CombinedToMask).getBoolValue()) 11165 // Conflicting bits - bail out because going further is unsafe. 11166 return SDValue(); 11167 11168 // Are the new bits contiguous when combined with the old bits? 11169 if (BitsProperlyConcatenate(ToMask, NewToMask) && 11170 BitsProperlyConcatenate(FromMask, NewFromMask)) 11171 return V; 11172 if (BitsProperlyConcatenate(NewToMask, ToMask) && 11173 BitsProperlyConcatenate(NewFromMask, FromMask)) 11174 return V; 11175 11176 // We've seen a write to some bits, so track it. 11177 CombinedToMask |= NewToMask; 11178 // Keep going... 11179 V = V.getOperand(0); 11180 } 11181 11182 return SDValue(); 11183 } 11184 11185 static SDValue PerformBFICombine(SDNode *N, 11186 TargetLowering::DAGCombinerInfo &DCI) { 11187 SDValue N1 = N->getOperand(1); 11188 if (N1.getOpcode() == ISD::AND) { 11189 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff 11190 // the bits being cleared by the AND are not demanded by the BFI. 11191 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1)); 11192 if (!N11C) 11193 return SDValue(); 11194 unsigned InvMask = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(); 11195 unsigned LSB = countTrailingZeros(~InvMask); 11196 unsigned Width = (32 - countLeadingZeros(~InvMask)) - LSB; 11197 assert(Width < 11198 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && 11199 "undefined behavior"); 11200 unsigned Mask = (1u << Width) - 1; 11201 unsigned Mask2 = N11C->getZExtValue(); 11202 if ((Mask & (~Mask2)) == 0) 11203 return DCI.DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0), 11204 N->getOperand(0), N1.getOperand(0), 11205 N->getOperand(2)); 11206 } else if (N->getOperand(0).getOpcode() == ARMISD::BFI) { 11207 // We have a BFI of a BFI. Walk up the BFI chain to see how long it goes. 11208 // Keep track of any consecutive bits set that all come from the same base 11209 // value. We can combine these together into a single BFI. 11210 SDValue CombineBFI = FindBFIToCombineWith(N); 11211 if (CombineBFI == SDValue()) 11212 return SDValue(); 11213 11214 // We've found a BFI. 11215 APInt ToMask1, FromMask1; 11216 SDValue From1 = ParseBFI(N, ToMask1, FromMask1); 11217 11218 APInt ToMask2, FromMask2; 11219 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2); 11220 assert(From1 == From2); 11221 (void)From2; 11222 11223 // First, unlink CombineBFI. 11224 DCI.DAG.ReplaceAllUsesWith(CombineBFI, CombineBFI.getOperand(0)); 11225 // Then create a new BFI, combining the two together. 11226 APInt NewFromMask = FromMask1 | FromMask2; 11227 APInt NewToMask = ToMask1 | ToMask2; 11228 11229 EVT VT = N->getValueType(0); 11230 SDLoc dl(N); 11231 11232 if (NewFromMask[0] == 0) 11233 From1 = DCI.DAG.getNode( 11234 ISD::SRL, dl, VT, From1, 11235 DCI.DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT)); 11236 return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1, 11237 DCI.DAG.getConstant(~NewToMask, dl, VT)); 11238 } 11239 return SDValue(); 11240 } 11241 11242 /// PerformVMOVRRDCombine - Target-specific dag combine xforms for 11243 /// ARMISD::VMOVRRD. 11244 static SDValue PerformVMOVRRDCombine(SDNode *N, 11245 TargetLowering::DAGCombinerInfo &DCI, 11246 const ARMSubtarget *Subtarget) { 11247 // vmovrrd(vmovdrr x, y) -> x,y 11248 SDValue InDouble = N->getOperand(0); 11249 if (InDouble.getOpcode() == ARMISD::VMOVDRR && !Subtarget->isFPOnlySP()) 11250 return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1)); 11251 11252 // vmovrrd(load f64) -> (load i32), (load i32) 11253 SDNode *InNode = InDouble.getNode(); 11254 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() && 11255 InNode->getValueType(0) == MVT::f64 && 11256 InNode->getOperand(1).getOpcode() == ISD::FrameIndex && 11257 !cast<LoadSDNode>(InNode)->isVolatile()) { 11258 // TODO: Should this be done for non-FrameIndex operands? 11259 LoadSDNode *LD = cast<LoadSDNode>(InNode); 11260 11261 SelectionDAG &DAG = DCI.DAG; 11262 SDLoc DL(LD); 11263 SDValue BasePtr = LD->getBasePtr(); 11264 SDValue NewLD1 = 11265 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(), 11266 LD->getAlignment(), LD->getMemOperand()->getFlags()); 11267 11268 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, 11269 DAG.getConstant(4, DL, MVT::i32)); 11270 SDValue NewLD2 = DAG.getLoad( 11271 MVT::i32, DL, NewLD1.getValue(1), OffsetPtr, LD->getPointerInfo(), 11272 std::min(4U, LD->getAlignment() / 2), LD->getMemOperand()->getFlags()); 11273 11274 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1)); 11275 if (DCI.DAG.getDataLayout().isBigEndian()) 11276 std::swap (NewLD1, NewLD2); 11277 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2); 11278 return Result; 11279 } 11280 11281 return SDValue(); 11282 } 11283 11284 /// PerformVMOVDRRCombine - Target-specific dag combine xforms for 11285 /// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands. 11286 static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) { 11287 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X) 11288 SDValue Op0 = N->getOperand(0); 11289 SDValue Op1 = N->getOperand(1); 11290 if (Op0.getOpcode() == ISD::BITCAST) 11291 Op0 = Op0.getOperand(0); 11292 if (Op1.getOpcode() == ISD::BITCAST) 11293 Op1 = Op1.getOperand(0); 11294 if (Op0.getOpcode() == ARMISD::VMOVRRD && 11295 Op0.getNode() == Op1.getNode() && 11296 Op0.getResNo() == 0 && Op1.getResNo() == 1) 11297 return DAG.getNode(ISD::BITCAST, SDLoc(N), 11298 N->getValueType(0), Op0.getOperand(0)); 11299 return SDValue(); 11300 } 11301 11302 /// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node 11303 /// are normal, non-volatile loads. If so, it is profitable to bitcast an 11304 /// i64 vector to have f64 elements, since the value can then be loaded 11305 /// directly into a VFP register. 11306 static bool hasNormalLoadOperand(SDNode *N) { 11307 unsigned NumElts = N->getValueType(0).getVectorNumElements(); 11308 for (unsigned i = 0; i < NumElts; ++i) { 11309 SDNode *Elt = N->getOperand(i).getNode(); 11310 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile()) 11311 return true; 11312 } 11313 return false; 11314 } 11315 11316 /// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for 11317 /// ISD::BUILD_VECTOR. 11318 static SDValue PerformBUILD_VECTORCombine(SDNode *N, 11319 TargetLowering::DAGCombinerInfo &DCI, 11320 const ARMSubtarget *Subtarget) { 11321 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X): 11322 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value 11323 // into a pair of GPRs, which is fine when the value is used as a scalar, 11324 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD. 11325 SelectionDAG &DAG = DCI.DAG; 11326 if (N->getNumOperands() == 2) 11327 if (SDValue RV = PerformVMOVDRRCombine(N, DAG)) 11328 return RV; 11329 11330 // Load i64 elements as f64 values so that type legalization does not split 11331 // them up into i32 values. 11332 EVT VT = N->getValueType(0); 11333 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N)) 11334 return SDValue(); 11335 SDLoc dl(N); 11336 SmallVector<SDValue, 8> Ops; 11337 unsigned NumElts = VT.getVectorNumElements(); 11338 for (unsigned i = 0; i < NumElts; ++i) { 11339 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i)); 11340 Ops.push_back(V); 11341 // Make the DAGCombiner fold the bitcast. 11342 DCI.AddToWorklist(V.getNode()); 11343 } 11344 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts); 11345 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops); 11346 return DAG.getNode(ISD::BITCAST, dl, VT, BV); 11347 } 11348 11349 /// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR. 11350 static SDValue 11351 PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { 11352 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR. 11353 // At that time, we may have inserted bitcasts from integer to float. 11354 // If these bitcasts have survived DAGCombine, change the lowering of this 11355 // BUILD_VECTOR in something more vector friendly, i.e., that does not 11356 // force to use floating point types. 11357 11358 // Make sure we can change the type of the vector. 11359 // This is possible iff: 11360 // 1. The vector is only used in a bitcast to a integer type. I.e., 11361 // 1.1. Vector is used only once. 11362 // 1.2. Use is a bit convert to an integer type. 11363 // 2. The size of its operands are 32-bits (64-bits are not legal). 11364 EVT VT = N->getValueType(0); 11365 EVT EltVT = VT.getVectorElementType(); 11366 11367 // Check 1.1. and 2. 11368 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse()) 11369 return SDValue(); 11370 11371 // By construction, the input type must be float. 11372 assert(EltVT == MVT::f32 && "Unexpected type!"); 11373 11374 // Check 1.2. 11375 SDNode *Use = *N->use_begin(); 11376 if (Use->getOpcode() != ISD::BITCAST || 11377 Use->getValueType(0).isFloatingPoint()) 11378 return SDValue(); 11379 11380 // Check profitability. 11381 // Model is, if more than half of the relevant operands are bitcast from 11382 // i32, turn the build_vector into a sequence of insert_vector_elt. 11383 // Relevant operands are everything that is not statically 11384 // (i.e., at compile time) bitcasted. 11385 unsigned NumOfBitCastedElts = 0; 11386 unsigned NumElts = VT.getVectorNumElements(); 11387 unsigned NumOfRelevantElts = NumElts; 11388 for (unsigned Idx = 0; Idx < NumElts; ++Idx) { 11389 SDValue Elt = N->getOperand(Idx); 11390 if (Elt->getOpcode() == ISD::BITCAST) { 11391 // Assume only bit cast to i32 will go away. 11392 if (Elt->getOperand(0).getValueType() == MVT::i32) 11393 ++NumOfBitCastedElts; 11394 } else if (Elt.isUndef() || isa<ConstantSDNode>(Elt)) 11395 // Constants are statically casted, thus do not count them as 11396 // relevant operands. 11397 --NumOfRelevantElts; 11398 } 11399 11400 // Check if more than half of the elements require a non-free bitcast. 11401 if (NumOfBitCastedElts <= NumOfRelevantElts / 2) 11402 return SDValue(); 11403 11404 SelectionDAG &DAG = DCI.DAG; 11405 // Create the new vector type. 11406 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts); 11407 // Check if the type is legal. 11408 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11409 if (!TLI.isTypeLegal(VecVT)) 11410 return SDValue(); 11411 11412 // Combine: 11413 // ARMISD::BUILD_VECTOR E1, E2, ..., EN. 11414 // => BITCAST INSERT_VECTOR_ELT 11415 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1), 11416 // (BITCAST EN), N. 11417 SDValue Vec = DAG.getUNDEF(VecVT); 11418 SDLoc dl(N); 11419 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) { 11420 SDValue V = N->getOperand(Idx); 11421 if (V.isUndef()) 11422 continue; 11423 if (V.getOpcode() == ISD::BITCAST && 11424 V->getOperand(0).getValueType() == MVT::i32) 11425 // Fold obvious case. 11426 V = V.getOperand(0); 11427 else { 11428 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V); 11429 // Make the DAGCombiner fold the bitcasts. 11430 DCI.AddToWorklist(V.getNode()); 11431 } 11432 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32); 11433 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx); 11434 } 11435 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec); 11436 // Make the DAGCombiner fold the bitcasts. 11437 DCI.AddToWorklist(Vec.getNode()); 11438 return Vec; 11439 } 11440 11441 /// PerformInsertEltCombine - Target-specific dag combine xforms for 11442 /// ISD::INSERT_VECTOR_ELT. 11443 static SDValue PerformInsertEltCombine(SDNode *N, 11444 TargetLowering::DAGCombinerInfo &DCI) { 11445 // Bitcast an i64 load inserted into a vector to f64. 11446 // Otherwise, the i64 value will be legalized to a pair of i32 values. 11447 EVT VT = N->getValueType(0); 11448 SDNode *Elt = N->getOperand(1).getNode(); 11449 if (VT.getVectorElementType() != MVT::i64 || 11450 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile()) 11451 return SDValue(); 11452 11453 SelectionDAG &DAG = DCI.DAG; 11454 SDLoc dl(N); 11455 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, 11456 VT.getVectorNumElements()); 11457 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0)); 11458 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1)); 11459 // Make the DAGCombiner fold the bitcasts. 11460 DCI.AddToWorklist(Vec.getNode()); 11461 DCI.AddToWorklist(V.getNode()); 11462 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT, 11463 Vec, V, N->getOperand(2)); 11464 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt); 11465 } 11466 11467 /// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for 11468 /// ISD::VECTOR_SHUFFLE. 11469 static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) { 11470 // The LLVM shufflevector instruction does not require the shuffle mask 11471 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does 11472 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the 11473 // operands do not match the mask length, they are extended by concatenating 11474 // them with undef vectors. That is probably the right thing for other 11475 // targets, but for NEON it is better to concatenate two double-register 11476 // size vector operands into a single quad-register size vector. Do that 11477 // transformation here: 11478 // shuffle(concat(v1, undef), concat(v2, undef)) -> 11479 // shuffle(concat(v1, v2), undef) 11480 SDValue Op0 = N->getOperand(0); 11481 SDValue Op1 = N->getOperand(1); 11482 if (Op0.getOpcode() != ISD::CONCAT_VECTORS || 11483 Op1.getOpcode() != ISD::CONCAT_VECTORS || 11484 Op0.getNumOperands() != 2 || 11485 Op1.getNumOperands() != 2) 11486 return SDValue(); 11487 SDValue Concat0Op1 = Op0.getOperand(1); 11488 SDValue Concat1Op1 = Op1.getOperand(1); 11489 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef()) 11490 return SDValue(); 11491 // Skip the transformation if any of the types are illegal. 11492 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11493 EVT VT = N->getValueType(0); 11494 if (!TLI.isTypeLegal(VT) || 11495 !TLI.isTypeLegal(Concat0Op1.getValueType()) || 11496 !TLI.isTypeLegal(Concat1Op1.getValueType())) 11497 return SDValue(); 11498 11499 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, 11500 Op0.getOperand(0), Op1.getOperand(0)); 11501 // Translate the shuffle mask. 11502 SmallVector<int, 16> NewMask; 11503 unsigned NumElts = VT.getVectorNumElements(); 11504 unsigned HalfElts = NumElts/2; 11505 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 11506 for (unsigned n = 0; n < NumElts; ++n) { 11507 int MaskElt = SVN->getMaskElt(n); 11508 int NewElt = -1; 11509 if (MaskElt < (int)HalfElts) 11510 NewElt = MaskElt; 11511 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts)) 11512 NewElt = HalfElts + MaskElt - NumElts; 11513 NewMask.push_back(NewElt); 11514 } 11515 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat, 11516 DAG.getUNDEF(VT), NewMask); 11517 } 11518 11519 /// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, 11520 /// NEON load/store intrinsics, and generic vector load/stores, to merge 11521 /// base address updates. 11522 /// For generic load/stores, the memory type is assumed to be a vector. 11523 /// The caller is assumed to have checked legality. 11524 static SDValue CombineBaseUpdate(SDNode *N, 11525 TargetLowering::DAGCombinerInfo &DCI) { 11526 SelectionDAG &DAG = DCI.DAG; 11527 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID || 11528 N->getOpcode() == ISD::INTRINSIC_W_CHAIN); 11529 const bool isStore = N->getOpcode() == ISD::STORE; 11530 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1); 11531 SDValue Addr = N->getOperand(AddrOpIdx); 11532 MemSDNode *MemN = cast<MemSDNode>(N); 11533 SDLoc dl(N); 11534 11535 // Search for a use of the address operand that is an increment. 11536 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), 11537 UE = Addr.getNode()->use_end(); UI != UE; ++UI) { 11538 SDNode *User = *UI; 11539 if (User->getOpcode() != ISD::ADD || 11540 UI.getUse().getResNo() != Addr.getResNo()) 11541 continue; 11542 11543 // Check that the add is independent of the load/store. Otherwise, folding 11544 // it would create a cycle. 11545 if (User->isPredecessorOf(N) || N->isPredecessorOf(User)) 11546 continue; 11547 11548 // Find the new opcode for the updating load/store. 11549 bool isLoadOp = true; 11550 bool isLaneOp = false; 11551 unsigned NewOpc = 0; 11552 unsigned NumVecs = 0; 11553 if (isIntrinsic) { 11554 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 11555 switch (IntNo) { 11556 default: llvm_unreachable("unexpected intrinsic for Neon base update"); 11557 case Intrinsic::arm_neon_vld1: NewOpc = ARMISD::VLD1_UPD; 11558 NumVecs = 1; break; 11559 case Intrinsic::arm_neon_vld2: NewOpc = ARMISD::VLD2_UPD; 11560 NumVecs = 2; break; 11561 case Intrinsic::arm_neon_vld3: NewOpc = ARMISD::VLD3_UPD; 11562 NumVecs = 3; break; 11563 case Intrinsic::arm_neon_vld4: NewOpc = ARMISD::VLD4_UPD; 11564 NumVecs = 4; break; 11565 case Intrinsic::arm_neon_vld2dup: 11566 case Intrinsic::arm_neon_vld3dup: 11567 case Intrinsic::arm_neon_vld4dup: 11568 // TODO: Support updating VLDxDUP nodes. For now, we just skip 11569 // combining base updates for such intrinsics. 11570 continue; 11571 case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD; 11572 NumVecs = 2; isLaneOp = true; break; 11573 case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD; 11574 NumVecs = 3; isLaneOp = true; break; 11575 case Intrinsic::arm_neon_vld4lane: NewOpc = ARMISD::VLD4LN_UPD; 11576 NumVecs = 4; isLaneOp = true; break; 11577 case Intrinsic::arm_neon_vst1: NewOpc = ARMISD::VST1_UPD; 11578 NumVecs = 1; isLoadOp = false; break; 11579 case Intrinsic::arm_neon_vst2: NewOpc = ARMISD::VST2_UPD; 11580 NumVecs = 2; isLoadOp = false; break; 11581 case Intrinsic::arm_neon_vst3: NewOpc = ARMISD::VST3_UPD; 11582 NumVecs = 3; isLoadOp = false; break; 11583 case Intrinsic::arm_neon_vst4: NewOpc = ARMISD::VST4_UPD; 11584 NumVecs = 4; isLoadOp = false; break; 11585 case Intrinsic::arm_neon_vst2lane: NewOpc = ARMISD::VST2LN_UPD; 11586 NumVecs = 2; isLoadOp = false; isLaneOp = true; break; 11587 case Intrinsic::arm_neon_vst3lane: NewOpc = ARMISD::VST3LN_UPD; 11588 NumVecs = 3; isLoadOp = false; isLaneOp = true; break; 11589 case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD; 11590 NumVecs = 4; isLoadOp = false; isLaneOp = true; break; 11591 } 11592 } else { 11593 isLaneOp = true; 11594 switch (N->getOpcode()) { 11595 default: llvm_unreachable("unexpected opcode for Neon base update"); 11596 case ARMISD::VLD1DUP: NewOpc = ARMISD::VLD1DUP_UPD; NumVecs = 1; break; 11597 case ARMISD::VLD2DUP: NewOpc = ARMISD::VLD2DUP_UPD; NumVecs = 2; break; 11598 case ARMISD::VLD3DUP: NewOpc = ARMISD::VLD3DUP_UPD; NumVecs = 3; break; 11599 case ARMISD::VLD4DUP: NewOpc = ARMISD::VLD4DUP_UPD; NumVecs = 4; break; 11600 case ISD::LOAD: NewOpc = ARMISD::VLD1_UPD; 11601 NumVecs = 1; isLaneOp = false; break; 11602 case ISD::STORE: NewOpc = ARMISD::VST1_UPD; 11603 NumVecs = 1; isLaneOp = false; isLoadOp = false; break; 11604 } 11605 } 11606 11607 // Find the size of memory referenced by the load/store. 11608 EVT VecTy; 11609 if (isLoadOp) { 11610 VecTy = N->getValueType(0); 11611 } else if (isIntrinsic) { 11612 VecTy = N->getOperand(AddrOpIdx+1).getValueType(); 11613 } else { 11614 assert(isStore && "Node has to be a load, a store, or an intrinsic!"); 11615 VecTy = N->getOperand(1).getValueType(); 11616 } 11617 11618 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8; 11619 if (isLaneOp) 11620 NumBytes /= VecTy.getVectorNumElements(); 11621 11622 // If the increment is a constant, it must match the memory ref size. 11623 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0); 11624 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode()); 11625 if (NumBytes >= 3 * 16 && (!CInc || CInc->getZExtValue() != NumBytes)) { 11626 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two 11627 // separate instructions that make it harder to use a non-constant update. 11628 continue; 11629 } 11630 11631 // OK, we found an ADD we can fold into the base update. 11632 // Now, create a _UPD node, taking care of not breaking alignment. 11633 11634 EVT AlignedVecTy = VecTy; 11635 unsigned Alignment = MemN->getAlignment(); 11636 11637 // If this is a less-than-standard-aligned load/store, change the type to 11638 // match the standard alignment. 11639 // The alignment is overlooked when selecting _UPD variants; and it's 11640 // easier to introduce bitcasts here than fix that. 11641 // There are 3 ways to get to this base-update combine: 11642 // - intrinsics: they are assumed to be properly aligned (to the standard 11643 // alignment of the memory type), so we don't need to do anything. 11644 // - ARMISD::VLDx nodes: they are only generated from the aforementioned 11645 // intrinsics, so, likewise, there's nothing to do. 11646 // - generic load/store instructions: the alignment is specified as an 11647 // explicit operand, rather than implicitly as the standard alignment 11648 // of the memory type (like the intrisics). We need to change the 11649 // memory type to match the explicit alignment. That way, we don't 11650 // generate non-standard-aligned ARMISD::VLDx nodes. 11651 if (isa<LSBaseSDNode>(N)) { 11652 if (Alignment == 0) 11653 Alignment = 1; 11654 if (Alignment < VecTy.getScalarSizeInBits() / 8) { 11655 MVT EltTy = MVT::getIntegerVT(Alignment * 8); 11656 assert(NumVecs == 1 && "Unexpected multi-element generic load/store."); 11657 assert(!isLaneOp && "Unexpected generic load/store lane."); 11658 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8); 11659 AlignedVecTy = MVT::getVectorVT(EltTy, NumElts); 11660 } 11661 // Don't set an explicit alignment on regular load/stores that we want 11662 // to transform to VLD/VST 1_UPD nodes. 11663 // This matches the behavior of regular load/stores, which only get an 11664 // explicit alignment if the MMO alignment is larger than the standard 11665 // alignment of the memory type. 11666 // Intrinsics, however, always get an explicit alignment, set to the 11667 // alignment of the MMO. 11668 Alignment = 1; 11669 } 11670 11671 // Create the new updating load/store node. 11672 // First, create an SDVTList for the new updating node's results. 11673 EVT Tys[6]; 11674 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0); 11675 unsigned n; 11676 for (n = 0; n < NumResultVecs; ++n) 11677 Tys[n] = AlignedVecTy; 11678 Tys[n++] = MVT::i32; 11679 Tys[n] = MVT::Other; 11680 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs+2)); 11681 11682 // Then, gather the new node's operands. 11683 SmallVector<SDValue, 8> Ops; 11684 Ops.push_back(N->getOperand(0)); // incoming chain 11685 Ops.push_back(N->getOperand(AddrOpIdx)); 11686 Ops.push_back(Inc); 11687 11688 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) { 11689 // Try to match the intrinsic's signature 11690 Ops.push_back(StN->getValue()); 11691 } else { 11692 // Loads (and of course intrinsics) match the intrinsics' signature, 11693 // so just add all but the alignment operand. 11694 for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i) 11695 Ops.push_back(N->getOperand(i)); 11696 } 11697 11698 // For all node types, the alignment operand is always the last one. 11699 Ops.push_back(DAG.getConstant(Alignment, dl, MVT::i32)); 11700 11701 // If this is a non-standard-aligned STORE, the penultimate operand is the 11702 // stored value. Bitcast it to the aligned type. 11703 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) { 11704 SDValue &StVal = Ops[Ops.size()-2]; 11705 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal); 11706 } 11707 11708 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy; 11709 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT, 11710 MemN->getMemOperand()); 11711 11712 // Update the uses. 11713 SmallVector<SDValue, 5> NewResults; 11714 for (unsigned i = 0; i < NumResultVecs; ++i) 11715 NewResults.push_back(SDValue(UpdN.getNode(), i)); 11716 11717 // If this is an non-standard-aligned LOAD, the first result is the loaded 11718 // value. Bitcast it to the expected result type. 11719 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) { 11720 SDValue &LdVal = NewResults[0]; 11721 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal); 11722 } 11723 11724 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs+1)); // chain 11725 DCI.CombineTo(N, NewResults); 11726 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs)); 11727 11728 break; 11729 } 11730 return SDValue(); 11731 } 11732 11733 static SDValue PerformVLDCombine(SDNode *N, 11734 TargetLowering::DAGCombinerInfo &DCI) { 11735 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 11736 return SDValue(); 11737 11738 return CombineBaseUpdate(N, DCI); 11739 } 11740 11741 /// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a 11742 /// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic 11743 /// are also VDUPLANEs. If so, combine them to a vldN-dup operation and 11744 /// return true. 11745 static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { 11746 SelectionDAG &DAG = DCI.DAG; 11747 EVT VT = N->getValueType(0); 11748 // vldN-dup instructions only support 64-bit vectors for N > 1. 11749 if (!VT.is64BitVector()) 11750 return false; 11751 11752 // Check if the VDUPLANE operand is a vldN-dup intrinsic. 11753 SDNode *VLD = N->getOperand(0).getNode(); 11754 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN) 11755 return false; 11756 unsigned NumVecs = 0; 11757 unsigned NewOpc = 0; 11758 unsigned IntNo = cast<ConstantSDNode>(VLD->getOperand(1))->getZExtValue(); 11759 if (IntNo == Intrinsic::arm_neon_vld2lane) { 11760 NumVecs = 2; 11761 NewOpc = ARMISD::VLD2DUP; 11762 } else if (IntNo == Intrinsic::arm_neon_vld3lane) { 11763 NumVecs = 3; 11764 NewOpc = ARMISD::VLD3DUP; 11765 } else if (IntNo == Intrinsic::arm_neon_vld4lane) { 11766 NumVecs = 4; 11767 NewOpc = ARMISD::VLD4DUP; 11768 } else { 11769 return false; 11770 } 11771 11772 // First check that all the vldN-lane uses are VDUPLANEs and that the lane 11773 // numbers match the load. 11774 unsigned VLDLaneNo = 11775 cast<ConstantSDNode>(VLD->getOperand(NumVecs+3))->getZExtValue(); 11776 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end(); 11777 UI != UE; ++UI) { 11778 // Ignore uses of the chain result. 11779 if (UI.getUse().getResNo() == NumVecs) 11780 continue; 11781 SDNode *User = *UI; 11782 if (User->getOpcode() != ARMISD::VDUPLANE || 11783 VLDLaneNo != cast<ConstantSDNode>(User->getOperand(1))->getZExtValue()) 11784 return false; 11785 } 11786 11787 // Create the vldN-dup node. 11788 EVT Tys[5]; 11789 unsigned n; 11790 for (n = 0; n < NumVecs; ++n) 11791 Tys[n] = VT; 11792 Tys[n] = MVT::Other; 11793 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumVecs+1)); 11794 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) }; 11795 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(VLD); 11796 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys, 11797 Ops, VLDMemInt->getMemoryVT(), 11798 VLDMemInt->getMemOperand()); 11799 11800 // Update the uses. 11801 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end(); 11802 UI != UE; ++UI) { 11803 unsigned ResNo = UI.getUse().getResNo(); 11804 // Ignore uses of the chain result. 11805 if (ResNo == NumVecs) 11806 continue; 11807 SDNode *User = *UI; 11808 DCI.CombineTo(User, SDValue(VLDDup.getNode(), ResNo)); 11809 } 11810 11811 // Now the vldN-lane intrinsic is dead except for its chain result. 11812 // Update uses of the chain. 11813 std::vector<SDValue> VLDDupResults; 11814 for (unsigned n = 0; n < NumVecs; ++n) 11815 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n)); 11816 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs)); 11817 DCI.CombineTo(VLD, VLDDupResults); 11818 11819 return true; 11820 } 11821 11822 /// PerformVDUPLANECombine - Target-specific dag combine xforms for 11823 /// ARMISD::VDUPLANE. 11824 static SDValue PerformVDUPLANECombine(SDNode *N, 11825 TargetLowering::DAGCombinerInfo &DCI) { 11826 SDValue Op = N->getOperand(0); 11827 11828 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses 11829 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation. 11830 if (CombineVLDDUP(N, DCI)) 11831 return SDValue(N, 0); 11832 11833 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is 11834 // redundant. Ignore bit_converts for now; element sizes are checked below. 11835 while (Op.getOpcode() == ISD::BITCAST) 11836 Op = Op.getOperand(0); 11837 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM) 11838 return SDValue(); 11839 11840 // Make sure the VMOV element size is not bigger than the VDUPLANE elements. 11841 unsigned EltSize = Op.getScalarValueSizeInBits(); 11842 // The canonical VMOV for a zero vector uses a 32-bit element size. 11843 unsigned Imm = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 11844 unsigned EltBits; 11845 if (ARM_AM::decodeNEONModImm(Imm, EltBits) == 0) 11846 EltSize = 8; 11847 EVT VT = N->getValueType(0); 11848 if (EltSize > VT.getScalarSizeInBits()) 11849 return SDValue(); 11850 11851 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op); 11852 } 11853 11854 /// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP. 11855 static SDValue PerformVDUPCombine(SDNode *N, 11856 TargetLowering::DAGCombinerInfo &DCI) { 11857 SelectionDAG &DAG = DCI.DAG; 11858 SDValue Op = N->getOperand(0); 11859 11860 // Match VDUP(LOAD) -> VLD1DUP. 11861 // We match this pattern here rather than waiting for isel because the 11862 // transform is only legal for unindexed loads. 11863 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode()); 11864 if (LD && Op.hasOneUse() && LD->isUnindexed() && 11865 LD->getMemoryVT() == N->getValueType(0).getVectorElementType()) { 11866 SDValue Ops[] = { LD->getOperand(0), LD->getOperand(1), 11867 DAG.getConstant(LD->getAlignment(), SDLoc(N), MVT::i32) }; 11868 SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other); 11869 SDValue VLDDup = DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys, 11870 Ops, LD->getMemoryVT(), 11871 LD->getMemOperand()); 11872 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1)); 11873 return VLDDup; 11874 } 11875 11876 return SDValue(); 11877 } 11878 11879 static SDValue PerformLOADCombine(SDNode *N, 11880 TargetLowering::DAGCombinerInfo &DCI) { 11881 EVT VT = N->getValueType(0); 11882 11883 // If this is a legal vector load, try to combine it into a VLD1_UPD. 11884 if (ISD::isNormalLoad(N) && VT.isVector() && 11885 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) 11886 return CombineBaseUpdate(N, DCI); 11887 11888 return SDValue(); 11889 } 11890 11891 /// PerformSTORECombine - Target-specific dag combine xforms for 11892 /// ISD::STORE. 11893 static SDValue PerformSTORECombine(SDNode *N, 11894 TargetLowering::DAGCombinerInfo &DCI) { 11895 StoreSDNode *St = cast<StoreSDNode>(N); 11896 if (St->isVolatile()) 11897 return SDValue(); 11898 11899 // Optimize trunc store (of multiple scalars) to shuffle and store. First, 11900 // pack all of the elements in one place. Next, store to memory in fewer 11901 // chunks. 11902 SDValue StVal = St->getValue(); 11903 EVT VT = StVal.getValueType(); 11904 if (St->isTruncatingStore() && VT.isVector()) { 11905 SelectionDAG &DAG = DCI.DAG; 11906 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11907 EVT StVT = St->getMemoryVT(); 11908 unsigned NumElems = VT.getVectorNumElements(); 11909 assert(StVT != VT && "Cannot truncate to the same type"); 11910 unsigned FromEltSz = VT.getScalarSizeInBits(); 11911 unsigned ToEltSz = StVT.getScalarSizeInBits(); 11912 11913 // From, To sizes and ElemCount must be pow of two 11914 if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz)) return SDValue(); 11915 11916 // We are going to use the original vector elt for storing. 11917 // Accumulated smaller vector elements must be a multiple of the store size. 11918 if (0 != (NumElems * FromEltSz) % ToEltSz) return SDValue(); 11919 11920 unsigned SizeRatio = FromEltSz / ToEltSz; 11921 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits()); 11922 11923 // Create a type on which we perform the shuffle. 11924 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(), 11925 NumElems*SizeRatio); 11926 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits()); 11927 11928 SDLoc DL(St); 11929 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal); 11930 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1); 11931 for (unsigned i = 0; i < NumElems; ++i) 11932 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() 11933 ? (i + 1) * SizeRatio - 1 11934 : i * SizeRatio; 11935 11936 // Can't shuffle using an illegal type. 11937 if (!TLI.isTypeLegal(WideVecVT)) return SDValue(); 11938 11939 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, DL, WideVec, 11940 DAG.getUNDEF(WideVec.getValueType()), 11941 ShuffleVec); 11942 // At this point all of the data is stored at the bottom of the 11943 // register. We now need to save it to mem. 11944 11945 // Find the largest store unit 11946 MVT StoreType = MVT::i8; 11947 for (MVT Tp : MVT::integer_valuetypes()) { 11948 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz) 11949 StoreType = Tp; 11950 } 11951 // Didn't find a legal store type. 11952 if (!TLI.isTypeLegal(StoreType)) 11953 return SDValue(); 11954 11955 // Bitcast the original vector into a vector of store-size units 11956 EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(), 11957 StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits()); 11958 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits()); 11959 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff); 11960 SmallVector<SDValue, 8> Chains; 11961 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL, 11962 TLI.getPointerTy(DAG.getDataLayout())); 11963 SDValue BasePtr = St->getBasePtr(); 11964 11965 // Perform one or more big stores into memory. 11966 unsigned E = (ToEltSz*NumElems)/StoreType.getSizeInBits(); 11967 for (unsigned I = 0; I < E; I++) { 11968 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 11969 StoreType, ShuffWide, 11970 DAG.getIntPtrConstant(I, DL)); 11971 SDValue Ch = DAG.getStore(St->getChain(), DL, SubVec, BasePtr, 11972 St->getPointerInfo(), St->getAlignment(), 11973 St->getMemOperand()->getFlags()); 11974 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, 11975 Increment); 11976 Chains.push_back(Ch); 11977 } 11978 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 11979 } 11980 11981 if (!ISD::isNormalStore(St)) 11982 return SDValue(); 11983 11984 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and 11985 // ARM stores of arguments in the same cache line. 11986 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR && 11987 StVal.getNode()->hasOneUse()) { 11988 SelectionDAG &DAG = DCI.DAG; 11989 bool isBigEndian = DAG.getDataLayout().isBigEndian(); 11990 SDLoc DL(St); 11991 SDValue BasePtr = St->getBasePtr(); 11992 SDValue NewST1 = DAG.getStore( 11993 St->getChain(), DL, StVal.getNode()->getOperand(isBigEndian ? 1 : 0), 11994 BasePtr, St->getPointerInfo(), St->getAlignment(), 11995 St->getMemOperand()->getFlags()); 11996 11997 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, 11998 DAG.getConstant(4, DL, MVT::i32)); 11999 return DAG.getStore(NewST1.getValue(0), DL, 12000 StVal.getNode()->getOperand(isBigEndian ? 0 : 1), 12001 OffsetPtr, St->getPointerInfo(), 12002 std::min(4U, St->getAlignment() / 2), 12003 St->getMemOperand()->getFlags()); 12004 } 12005 12006 if (StVal.getValueType() == MVT::i64 && 12007 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 12008 12009 // Bitcast an i64 store extracted from a vector to f64. 12010 // Otherwise, the i64 value will be legalized to a pair of i32 values. 12011 SelectionDAG &DAG = DCI.DAG; 12012 SDLoc dl(StVal); 12013 SDValue IntVec = StVal.getOperand(0); 12014 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, 12015 IntVec.getValueType().getVectorNumElements()); 12016 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec); 12017 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 12018 Vec, StVal.getOperand(1)); 12019 dl = SDLoc(N); 12020 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt); 12021 // Make the DAGCombiner fold the bitcasts. 12022 DCI.AddToWorklist(Vec.getNode()); 12023 DCI.AddToWorklist(ExtElt.getNode()); 12024 DCI.AddToWorklist(V.getNode()); 12025 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(), 12026 St->getPointerInfo(), St->getAlignment(), 12027 St->getMemOperand()->getFlags(), St->getAAInfo()); 12028 } 12029 12030 // If this is a legal vector store, try to combine it into a VST1_UPD. 12031 if (ISD::isNormalStore(N) && VT.isVector() && 12032 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) 12033 return CombineBaseUpdate(N, DCI); 12034 12035 return SDValue(); 12036 } 12037 12038 /// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) 12039 /// can replace combinations of VMUL and VCVT (floating-point to integer) 12040 /// when the VMUL has a constant operand that is a power of 2. 12041 /// 12042 /// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>): 12043 /// vmul.f32 d16, d17, d16 12044 /// vcvt.s32.f32 d16, d16 12045 /// becomes: 12046 /// vcvt.s32.f32 d16, d16, #3 12047 static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, 12048 const ARMSubtarget *Subtarget) { 12049 if (!Subtarget->hasNEON()) 12050 return SDValue(); 12051 12052 SDValue Op = N->getOperand(0); 12053 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() || 12054 Op.getOpcode() != ISD::FMUL) 12055 return SDValue(); 12056 12057 SDValue ConstVec = Op->getOperand(1); 12058 if (!isa<BuildVectorSDNode>(ConstVec)) 12059 return SDValue(); 12060 12061 MVT FloatTy = Op.getSimpleValueType().getVectorElementType(); 12062 uint32_t FloatBits = FloatTy.getSizeInBits(); 12063 MVT IntTy = N->getSimpleValueType(0).getVectorElementType(); 12064 uint32_t IntBits = IntTy.getSizeInBits(); 12065 unsigned NumLanes = Op.getValueType().getVectorNumElements(); 12066 if (FloatBits != 32 || IntBits > 32 || NumLanes > 4) { 12067 // These instructions only exist converting from f32 to i32. We can handle 12068 // smaller integers by generating an extra truncate, but larger ones would 12069 // be lossy. We also can't handle more then 4 lanes, since these intructions 12070 // only support v2i32/v4i32 types. 12071 return SDValue(); 12072 } 12073 12074 BitVector UndefElements; 12075 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec); 12076 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33); 12077 if (C == -1 || C == 0 || C > 32) 12078 return SDValue(); 12079 12080 SDLoc dl(N); 12081 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT; 12082 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs : 12083 Intrinsic::arm_neon_vcvtfp2fxu; 12084 SDValue FixConv = DAG.getNode( 12085 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, 12086 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0), 12087 DAG.getConstant(C, dl, MVT::i32)); 12088 12089 if (IntBits < FloatBits) 12090 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv); 12091 12092 return FixConv; 12093 } 12094 12095 /// PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD) 12096 /// can replace combinations of VCVT (integer to floating-point) and VDIV 12097 /// when the VDIV has a constant operand that is a power of 2. 12098 /// 12099 /// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>): 12100 /// vcvt.f32.s32 d16, d16 12101 /// vdiv.f32 d16, d17, d16 12102 /// becomes: 12103 /// vcvt.f32.s32 d16, d16, #3 12104 static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG, 12105 const ARMSubtarget *Subtarget) { 12106 if (!Subtarget->hasNEON()) 12107 return SDValue(); 12108 12109 SDValue Op = N->getOperand(0); 12110 unsigned OpOpcode = Op.getNode()->getOpcode(); 12111 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() || 12112 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP)) 12113 return SDValue(); 12114 12115 SDValue ConstVec = N->getOperand(1); 12116 if (!isa<BuildVectorSDNode>(ConstVec)) 12117 return SDValue(); 12118 12119 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType(); 12120 uint32_t FloatBits = FloatTy.getSizeInBits(); 12121 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType(); 12122 uint32_t IntBits = IntTy.getSizeInBits(); 12123 unsigned NumLanes = Op.getValueType().getVectorNumElements(); 12124 if (FloatBits != 32 || IntBits > 32 || NumLanes > 4) { 12125 // These instructions only exist converting from i32 to f32. We can handle 12126 // smaller integers by generating an extra extend, but larger ones would 12127 // be lossy. We also can't handle more then 4 lanes, since these intructions 12128 // only support v2i32/v4i32 types. 12129 return SDValue(); 12130 } 12131 12132 BitVector UndefElements; 12133 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec); 12134 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33); 12135 if (C == -1 || C == 0 || C > 32) 12136 return SDValue(); 12137 12138 SDLoc dl(N); 12139 bool isSigned = OpOpcode == ISD::SINT_TO_FP; 12140 SDValue ConvInput = Op.getOperand(0); 12141 if (IntBits < FloatBits) 12142 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 12143 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, 12144 ConvInput); 12145 12146 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp : 12147 Intrinsic::arm_neon_vcvtfxu2fp; 12148 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, 12149 Op.getValueType(), 12150 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), 12151 ConvInput, DAG.getConstant(C, dl, MVT::i32)); 12152 } 12153 12154 /// Getvshiftimm - Check if this is a valid build_vector for the immediate 12155 /// operand of a vector shift operation, where all the elements of the 12156 /// build_vector must have the same constant integer value. 12157 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) { 12158 // Ignore bit_converts. 12159 while (Op.getOpcode() == ISD::BITCAST) 12160 Op = Op.getOperand(0); 12161 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 12162 APInt SplatBits, SplatUndef; 12163 unsigned SplatBitSize; 12164 bool HasAnyUndefs; 12165 if (! BVN || ! BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, 12166 HasAnyUndefs, ElementBits) || 12167 SplatBitSize > ElementBits) 12168 return false; 12169 Cnt = SplatBits.getSExtValue(); 12170 return true; 12171 } 12172 12173 /// isVShiftLImm - Check if this is a valid build_vector for the immediate 12174 /// operand of a vector shift left operation. That value must be in the range: 12175 /// 0 <= Value < ElementBits for a left shift; or 12176 /// 0 <= Value <= ElementBits for a long left shift. 12177 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { 12178 assert(VT.isVector() && "vector shift count is not a vector type"); 12179 int64_t ElementBits = VT.getScalarSizeInBits(); 12180 if (! getVShiftImm(Op, ElementBits, Cnt)) 12181 return false; 12182 return (Cnt >= 0 && (isLong ? Cnt-1 : Cnt) < ElementBits); 12183 } 12184 12185 /// isVShiftRImm - Check if this is a valid build_vector for the immediate 12186 /// operand of a vector shift right operation. For a shift opcode, the value 12187 /// is positive, but for an intrinsic the value count must be negative. The 12188 /// absolute value must be in the range: 12189 /// 1 <= |Value| <= ElementBits for a right shift; or 12190 /// 1 <= |Value| <= ElementBits/2 for a narrow right shift. 12191 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic, 12192 int64_t &Cnt) { 12193 assert(VT.isVector() && "vector shift count is not a vector type"); 12194 int64_t ElementBits = VT.getScalarSizeInBits(); 12195 if (! getVShiftImm(Op, ElementBits, Cnt)) 12196 return false; 12197 if (!isIntrinsic) 12198 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits/2 : ElementBits)); 12199 if (Cnt >= -(isNarrow ? ElementBits/2 : ElementBits) && Cnt <= -1) { 12200 Cnt = -Cnt; 12201 return true; 12202 } 12203 return false; 12204 } 12205 12206 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics. 12207 static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { 12208 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 12209 switch (IntNo) { 12210 default: 12211 // Don't do anything for most intrinsics. 12212 break; 12213 12214 // Vector shifts: check for immediate versions and lower them. 12215 // Note: This is done during DAG combining instead of DAG legalizing because 12216 // the build_vectors for 64-bit vector element shift counts are generally 12217 // not legal, and it is hard to see their values after they get legalized to 12218 // loads from a constant pool. 12219 case Intrinsic::arm_neon_vshifts: 12220 case Intrinsic::arm_neon_vshiftu: 12221 case Intrinsic::arm_neon_vrshifts: 12222 case Intrinsic::arm_neon_vrshiftu: 12223 case Intrinsic::arm_neon_vrshiftn: 12224 case Intrinsic::arm_neon_vqshifts: 12225 case Intrinsic::arm_neon_vqshiftu: 12226 case Intrinsic::arm_neon_vqshiftsu: 12227 case Intrinsic::arm_neon_vqshiftns: 12228 case Intrinsic::arm_neon_vqshiftnu: 12229 case Intrinsic::arm_neon_vqshiftnsu: 12230 case Intrinsic::arm_neon_vqrshiftns: 12231 case Intrinsic::arm_neon_vqrshiftnu: 12232 case Intrinsic::arm_neon_vqrshiftnsu: { 12233 EVT VT = N->getOperand(1).getValueType(); 12234 int64_t Cnt; 12235 unsigned VShiftOpc = 0; 12236 12237 switch (IntNo) { 12238 case Intrinsic::arm_neon_vshifts: 12239 case Intrinsic::arm_neon_vshiftu: 12240 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) { 12241 VShiftOpc = ARMISD::VSHL; 12242 break; 12243 } 12244 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) { 12245 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? 12246 ARMISD::VSHRs : ARMISD::VSHRu); 12247 break; 12248 } 12249 return SDValue(); 12250 12251 case Intrinsic::arm_neon_vrshifts: 12252 case Intrinsic::arm_neon_vrshiftu: 12253 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) 12254 break; 12255 return SDValue(); 12256 12257 case Intrinsic::arm_neon_vqshifts: 12258 case Intrinsic::arm_neon_vqshiftu: 12259 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) 12260 break; 12261 return SDValue(); 12262 12263 case Intrinsic::arm_neon_vqshiftsu: 12264 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) 12265 break; 12266 llvm_unreachable("invalid shift count for vqshlu intrinsic"); 12267 12268 case Intrinsic::arm_neon_vrshiftn: 12269 case Intrinsic::arm_neon_vqshiftns: 12270 case Intrinsic::arm_neon_vqshiftnu: 12271 case Intrinsic::arm_neon_vqshiftnsu: 12272 case Intrinsic::arm_neon_vqrshiftns: 12273 case Intrinsic::arm_neon_vqrshiftnu: 12274 case Intrinsic::arm_neon_vqrshiftnsu: 12275 // Narrowing shifts require an immediate right shift. 12276 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt)) 12277 break; 12278 llvm_unreachable("invalid shift count for narrowing vector shift " 12279 "intrinsic"); 12280 12281 default: 12282 llvm_unreachable("unhandled vector shift"); 12283 } 12284 12285 switch (IntNo) { 12286 case Intrinsic::arm_neon_vshifts: 12287 case Intrinsic::arm_neon_vshiftu: 12288 // Opcode already set above. 12289 break; 12290 case Intrinsic::arm_neon_vrshifts: 12291 VShiftOpc = ARMISD::VRSHRs; break; 12292 case Intrinsic::arm_neon_vrshiftu: 12293 VShiftOpc = ARMISD::VRSHRu; break; 12294 case Intrinsic::arm_neon_vrshiftn: 12295 VShiftOpc = ARMISD::VRSHRN; break; 12296 case Intrinsic::arm_neon_vqshifts: 12297 VShiftOpc = ARMISD::VQSHLs; break; 12298 case Intrinsic::arm_neon_vqshiftu: 12299 VShiftOpc = ARMISD::VQSHLu; break; 12300 case Intrinsic::arm_neon_vqshiftsu: 12301 VShiftOpc = ARMISD::VQSHLsu; break; 12302 case Intrinsic::arm_neon_vqshiftns: 12303 VShiftOpc = ARMISD::VQSHRNs; break; 12304 case Intrinsic::arm_neon_vqshiftnu: 12305 VShiftOpc = ARMISD::VQSHRNu; break; 12306 case Intrinsic::arm_neon_vqshiftnsu: 12307 VShiftOpc = ARMISD::VQSHRNsu; break; 12308 case Intrinsic::arm_neon_vqrshiftns: 12309 VShiftOpc = ARMISD::VQRSHRNs; break; 12310 case Intrinsic::arm_neon_vqrshiftnu: 12311 VShiftOpc = ARMISD::VQRSHRNu; break; 12312 case Intrinsic::arm_neon_vqrshiftnsu: 12313 VShiftOpc = ARMISD::VQRSHRNsu; break; 12314 } 12315 12316 SDLoc dl(N); 12317 return DAG.getNode(VShiftOpc, dl, N->getValueType(0), 12318 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32)); 12319 } 12320 12321 case Intrinsic::arm_neon_vshiftins: { 12322 EVT VT = N->getOperand(1).getValueType(); 12323 int64_t Cnt; 12324 unsigned VShiftOpc = 0; 12325 12326 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt)) 12327 VShiftOpc = ARMISD::VSLI; 12328 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt)) 12329 VShiftOpc = ARMISD::VSRI; 12330 else { 12331 llvm_unreachable("invalid shift count for vsli/vsri intrinsic"); 12332 } 12333 12334 SDLoc dl(N); 12335 return DAG.getNode(VShiftOpc, dl, N->getValueType(0), 12336 N->getOperand(1), N->getOperand(2), 12337 DAG.getConstant(Cnt, dl, MVT::i32)); 12338 } 12339 12340 case Intrinsic::arm_neon_vqrshifts: 12341 case Intrinsic::arm_neon_vqrshiftu: 12342 // No immediate versions of these to check for. 12343 break; 12344 } 12345 12346 return SDValue(); 12347 } 12348 12349 /// PerformShiftCombine - Checks for immediate versions of vector shifts and 12350 /// lowers them. As with the vector shift intrinsics, this is done during DAG 12351 /// combining instead of DAG legalizing because the build_vectors for 64-bit 12352 /// vector element shift counts are generally not legal, and it is hard to see 12353 /// their values after they get legalized to loads from a constant pool. 12354 static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG, 12355 const ARMSubtarget *ST) { 12356 EVT VT = N->getValueType(0); 12357 if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) { 12358 // Canonicalize (srl (bswap x), 16) to (rotr (bswap x), 16) if the high 12359 // 16-bits of x is zero. This optimizes rev + lsr 16 to rev16. 12360 SDValue N1 = N->getOperand(1); 12361 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 12362 SDValue N0 = N->getOperand(0); 12363 if (C->getZExtValue() == 16 && N0.getOpcode() == ISD::BSWAP && 12364 DAG.MaskedValueIsZero(N0.getOperand(0), 12365 APInt::getHighBitsSet(32, 16))) 12366 return DAG.getNode(ISD::ROTR, SDLoc(N), VT, N0, N1); 12367 } 12368 } 12369 12370 // Nothing to be done for scalar shifts. 12371 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 12372 if (!VT.isVector() || !TLI.isTypeLegal(VT)) 12373 return SDValue(); 12374 12375 assert(ST->hasNEON() && "unexpected vector shift"); 12376 int64_t Cnt; 12377 12378 switch (N->getOpcode()) { 12379 default: llvm_unreachable("unexpected shift opcode"); 12380 12381 case ISD::SHL: 12382 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) { 12383 SDLoc dl(N); 12384 return DAG.getNode(ARMISD::VSHL, dl, VT, N->getOperand(0), 12385 DAG.getConstant(Cnt, dl, MVT::i32)); 12386 } 12387 break; 12388 12389 case ISD::SRA: 12390 case ISD::SRL: 12391 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { 12392 unsigned VShiftOpc = (N->getOpcode() == ISD::SRA ? 12393 ARMISD::VSHRs : ARMISD::VSHRu); 12394 SDLoc dl(N); 12395 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), 12396 DAG.getConstant(Cnt, dl, MVT::i32)); 12397 } 12398 } 12399 return SDValue(); 12400 } 12401 12402 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, 12403 /// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND. 12404 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, 12405 const ARMSubtarget *ST) { 12406 SDValue N0 = N->getOperand(0); 12407 12408 // Check for sign- and zero-extensions of vector extract operations of 8- 12409 // and 16-bit vector elements. NEON supports these directly. They are 12410 // handled during DAG combining because type legalization will promote them 12411 // to 32-bit types and it is messy to recognize the operations after that. 12412 if (ST->hasNEON() && N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 12413 SDValue Vec = N0.getOperand(0); 12414 SDValue Lane = N0.getOperand(1); 12415 EVT VT = N->getValueType(0); 12416 EVT EltVT = N0.getValueType(); 12417 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 12418 12419 if (VT == MVT::i32 && 12420 (EltVT == MVT::i8 || EltVT == MVT::i16) && 12421 TLI.isTypeLegal(Vec.getValueType()) && 12422 isa<ConstantSDNode>(Lane)) { 12423 12424 unsigned Opc = 0; 12425 switch (N->getOpcode()) { 12426 default: llvm_unreachable("unexpected opcode"); 12427 case ISD::SIGN_EXTEND: 12428 Opc = ARMISD::VGETLANEs; 12429 break; 12430 case ISD::ZERO_EXTEND: 12431 case ISD::ANY_EXTEND: 12432 Opc = ARMISD::VGETLANEu; 12433 break; 12434 } 12435 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane); 12436 } 12437 } 12438 12439 return SDValue(); 12440 } 12441 12442 static const APInt *isPowerOf2Constant(SDValue V) { 12443 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V); 12444 if (!C) 12445 return nullptr; 12446 const APInt *CV = &C->getAPIntValue(); 12447 return CV->isPowerOf2() ? CV : nullptr; 12448 } 12449 12450 SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const { 12451 // If we have a CMOV, OR and AND combination such as: 12452 // if (x & CN) 12453 // y |= CM; 12454 // 12455 // And: 12456 // * CN is a single bit; 12457 // * All bits covered by CM are known zero in y 12458 // 12459 // Then we can convert this into a sequence of BFI instructions. This will 12460 // always be a win if CM is a single bit, will always be no worse than the 12461 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is 12462 // three bits (due to the extra IT instruction). 12463 12464 SDValue Op0 = CMOV->getOperand(0); 12465 SDValue Op1 = CMOV->getOperand(1); 12466 auto CCNode = cast<ConstantSDNode>(CMOV->getOperand(2)); 12467 auto CC = CCNode->getAPIntValue().getLimitedValue(); 12468 SDValue CmpZ = CMOV->getOperand(4); 12469 12470 // The compare must be against zero. 12471 if (!isNullConstant(CmpZ->getOperand(1))) 12472 return SDValue(); 12473 12474 assert(CmpZ->getOpcode() == ARMISD::CMPZ); 12475 SDValue And = CmpZ->getOperand(0); 12476 if (And->getOpcode() != ISD::AND) 12477 return SDValue(); 12478 const APInt *AndC = isPowerOf2Constant(And->getOperand(1)); 12479 if (!AndC) 12480 return SDValue(); 12481 SDValue X = And->getOperand(0); 12482 12483 if (CC == ARMCC::EQ) { 12484 // We're performing an "equal to zero" compare. Swap the operands so we 12485 // canonicalize on a "not equal to zero" compare. 12486 std::swap(Op0, Op1); 12487 } else { 12488 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?"); 12489 } 12490 12491 if (Op1->getOpcode() != ISD::OR) 12492 return SDValue(); 12493 12494 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1)); 12495 if (!OrC) 12496 return SDValue(); 12497 SDValue Y = Op1->getOperand(0); 12498 12499 if (Op0 != Y) 12500 return SDValue(); 12501 12502 // Now, is it profitable to continue? 12503 APInt OrCI = OrC->getAPIntValue(); 12504 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2; 12505 if (OrCI.countPopulation() > Heuristic) 12506 return SDValue(); 12507 12508 // Lastly, can we determine that the bits defined by OrCI 12509 // are zero in Y? 12510 KnownBits Known; 12511 DAG.computeKnownBits(Y, Known); 12512 if ((OrCI & Known.Zero) != OrCI) 12513 return SDValue(); 12514 12515 // OK, we can do the combine. 12516 SDValue V = Y; 12517 SDLoc dl(X); 12518 EVT VT = X.getValueType(); 12519 unsigned BitInX = AndC->logBase2(); 12520 12521 if (BitInX != 0) { 12522 // We must shift X first. 12523 X = DAG.getNode(ISD::SRL, dl, VT, X, 12524 DAG.getConstant(BitInX, dl, VT)); 12525 } 12526 12527 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits(); 12528 BitInY < NumActiveBits; ++BitInY) { 12529 if (OrCI[BitInY] == 0) 12530 continue; 12531 APInt Mask(VT.getSizeInBits(), 0); 12532 Mask.setBit(BitInY); 12533 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X, 12534 // Confusingly, the operand is an *inverted* mask. 12535 DAG.getConstant(~Mask, dl, VT)); 12536 } 12537 12538 return V; 12539 } 12540 12541 /// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND. 12542 SDValue 12543 ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const { 12544 SDValue Cmp = N->getOperand(4); 12545 if (Cmp.getOpcode() != ARMISD::CMPZ) 12546 // Only looking at NE cases. 12547 return SDValue(); 12548 12549 EVT VT = N->getValueType(0); 12550 SDLoc dl(N); 12551 SDValue LHS = Cmp.getOperand(0); 12552 SDValue RHS = Cmp.getOperand(1); 12553 SDValue Chain = N->getOperand(0); 12554 SDValue BB = N->getOperand(1); 12555 SDValue ARMcc = N->getOperand(2); 12556 ARMCC::CondCodes CC = 12557 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue(); 12558 12559 // (brcond Chain BB ne CPSR (cmpz (and (cmov 0 1 CC CPSR Cmp) 1) 0)) 12560 // -> (brcond Chain BB CC CPSR Cmp) 12561 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() && 12562 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV && 12563 LHS->getOperand(0)->hasOneUse()) { 12564 auto *LHS00C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(0)); 12565 auto *LHS01C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(1)); 12566 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1)); 12567 auto *RHSC = dyn_cast<ConstantSDNode>(RHS); 12568 if ((LHS00C && LHS00C->getZExtValue() == 0) && 12569 (LHS01C && LHS01C->getZExtValue() == 1) && 12570 (LHS1C && LHS1C->getZExtValue() == 1) && 12571 (RHSC && RHSC->getZExtValue() == 0)) { 12572 return DAG.getNode( 12573 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2), 12574 LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4)); 12575 } 12576 } 12577 12578 return SDValue(); 12579 } 12580 12581 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV. 12582 SDValue 12583 ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const { 12584 SDValue Cmp = N->getOperand(4); 12585 if (Cmp.getOpcode() != ARMISD::CMPZ) 12586 // Only looking at EQ and NE cases. 12587 return SDValue(); 12588 12589 EVT VT = N->getValueType(0); 12590 SDLoc dl(N); 12591 SDValue LHS = Cmp.getOperand(0); 12592 SDValue RHS = Cmp.getOperand(1); 12593 SDValue FalseVal = N->getOperand(0); 12594 SDValue TrueVal = N->getOperand(1); 12595 SDValue ARMcc = N->getOperand(2); 12596 ARMCC::CondCodes CC = 12597 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue(); 12598 12599 // BFI is only available on V6T2+. 12600 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) { 12601 SDValue R = PerformCMOVToBFICombine(N, DAG); 12602 if (R) 12603 return R; 12604 } 12605 12606 // Simplify 12607 // mov r1, r0 12608 // cmp r1, x 12609 // mov r0, y 12610 // moveq r0, x 12611 // to 12612 // cmp r0, x 12613 // movne r0, y 12614 // 12615 // mov r1, r0 12616 // cmp r1, x 12617 // mov r0, x 12618 // movne r0, y 12619 // to 12620 // cmp r0, x 12621 // movne r0, y 12622 /// FIXME: Turn this into a target neutral optimization? 12623 SDValue Res; 12624 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) { 12625 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, 12626 N->getOperand(3), Cmp); 12627 } else if (CC == ARMCC::EQ && TrueVal == RHS) { 12628 SDValue ARMcc; 12629 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl); 12630 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, 12631 N->getOperand(3), NewCmp); 12632 } 12633 12634 // (cmov F T ne CPSR (cmpz (cmov 0 1 CC CPSR Cmp) 0)) 12635 // -> (cmov F T CC CPSR Cmp) 12636 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse()) { 12637 auto *LHS0C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)); 12638 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1)); 12639 auto *RHSC = dyn_cast<ConstantSDNode>(RHS); 12640 if ((LHS0C && LHS0C->getZExtValue() == 0) && 12641 (LHS1C && LHS1C->getZExtValue() == 1) && 12642 (RHSC && RHSC->getZExtValue() == 0)) { 12643 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, 12644 LHS->getOperand(2), LHS->getOperand(3), 12645 LHS->getOperand(4)); 12646 } 12647 } 12648 12649 if (!VT.isInteger()) 12650 return SDValue(); 12651 12652 // Materialize a boolean comparison for integers so we can avoid branching. 12653 if (isNullConstant(FalseVal)) { 12654 if (CC == ARMCC::EQ && isOneConstant(TrueVal)) { 12655 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) { 12656 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it 12657 // right 5 bits will make that 32 be 1, otherwise it will be 0. 12658 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5 12659 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); 12660 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub), 12661 DAG.getConstant(5, dl, MVT::i32)); 12662 } else { 12663 // CMOV 0, 1, ==, (CMPZ x, y) -> 12664 // (ADDCARRY (SUB x, y), t:0, t:1) 12665 // where t = (SUBCARRY 0, (SUB x, y), 0) 12666 // 12667 // The SUBCARRY computes 0 - (x - y) and this will give a borrow when 12668 // x != y. In other words, a carry C == 1 when x == y, C == 0 12669 // otherwise. 12670 // The final ADDCARRY computes 12671 // x - y + (0 - (x - y)) + C == C 12672 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); 12673 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 12674 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub); 12675 // ISD::SUBCARRY returns a borrow but we want the carry here 12676 // actually. 12677 SDValue Carry = 12678 DAG.getNode(ISD::SUB, dl, MVT::i32, 12679 DAG.getConstant(1, dl, MVT::i32), Neg.getValue(1)); 12680 Res = DAG.getNode(ISD::ADDCARRY, dl, VTs, Sub, Neg, Carry); 12681 } 12682 } else if (CC == ARMCC::NE && LHS != RHS && 12683 (!Subtarget->isThumb1Only() || isPowerOf2Constant(TrueVal))) { 12684 // This seems pointless but will allow us to combine it further below. 12685 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUB x, y), z, !=, (CMPZ x, y) 12686 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); 12687 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc, 12688 N->getOperand(3), Cmp); 12689 } 12690 } else if (isNullConstant(TrueVal)) { 12691 if (CC == ARMCC::EQ && LHS != RHS && 12692 (!Subtarget->isThumb1Only() || isPowerOf2Constant(FalseVal))) { 12693 // This seems pointless but will allow us to combine it further below 12694 // Note that we change == for != as this is the dual for the case above. 12695 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUB x, y), z, !=, (CMPZ x, y) 12696 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); 12697 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal, 12698 DAG.getConstant(ARMCC::NE, dl, MVT::i32), 12699 N->getOperand(3), Cmp); 12700 } 12701 } 12702 12703 // On Thumb1, the DAG above may be further combined if z is a power of 2 12704 // (z == 2 ^ K). 12705 // CMOV (SUB x, y), z, !=, (CMPZ x, y) -> 12706 // merge t3, t4 12707 // where t1 = (SUBCARRY (SUB x, y), z, 0) 12708 // t2 = (SUBCARRY (SUB x, y), t1:0, t1:1) 12709 // t3 = if K != 0 then (SHL t2:0, K) else t2:0 12710 // t4 = (SUB 1, t2:1) [ we want a carry, not a borrow ] 12711 const APInt *TrueConst; 12712 if (Subtarget->isThumb1Only() && CC == ARMCC::NE && 12713 (FalseVal.getOpcode() == ISD::SUB) && (FalseVal.getOperand(0) == LHS) && 12714 (FalseVal.getOperand(1) == RHS) && 12715 (TrueConst = isPowerOf2Constant(TrueVal))) { 12716 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 12717 unsigned ShiftAmount = TrueConst->logBase2(); 12718 if (ShiftAmount) 12719 TrueVal = DAG.getConstant(1, dl, VT); 12720 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal); 12721 Res = DAG.getNode(ISD::SUBCARRY, dl, VTs, FalseVal, Subc, Subc.getValue(1)); 12722 // Make it a carry, not a borrow. 12723 SDValue Carry = DAG.getNode( 12724 ISD::SUB, dl, VT, DAG.getConstant(1, dl, MVT::i32), Res.getValue(1)); 12725 Res = DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Res, Carry); 12726 12727 if (ShiftAmount) 12728 Res = DAG.getNode(ISD::SHL, dl, VT, Res, 12729 DAG.getConstant(ShiftAmount, dl, MVT::i32)); 12730 } 12731 12732 if (Res.getNode()) { 12733 KnownBits Known; 12734 DAG.computeKnownBits(SDValue(N,0), Known); 12735 // Capture demanded bits information that would be otherwise lost. 12736 if (Known.Zero == 0xfffffffe) 12737 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res, 12738 DAG.getValueType(MVT::i1)); 12739 else if (Known.Zero == 0xffffff00) 12740 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res, 12741 DAG.getValueType(MVT::i8)); 12742 else if (Known.Zero == 0xffff0000) 12743 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res, 12744 DAG.getValueType(MVT::i16)); 12745 } 12746 12747 return Res; 12748 } 12749 12750 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, 12751 DAGCombinerInfo &DCI) const { 12752 switch (N->getOpcode()) { 12753 default: break; 12754 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget); 12755 case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget); 12756 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget); 12757 case ISD::SUB: return PerformSUBCombine(N, DCI); 12758 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget); 12759 case ISD::OR: return PerformORCombine(N, DCI, Subtarget); 12760 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget); 12761 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget); 12762 case ARMISD::ADDC: 12763 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget); 12764 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget); 12765 case ARMISD::BFI: return PerformBFICombine(N, DCI); 12766 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget); 12767 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG); 12768 case ISD::STORE: return PerformSTORECombine(N, DCI); 12769 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget); 12770 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI); 12771 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG); 12772 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI); 12773 case ARMISD::VDUP: return PerformVDUPCombine(N, DCI); 12774 case ISD::FP_TO_SINT: 12775 case ISD::FP_TO_UINT: 12776 return PerformVCVTCombine(N, DCI.DAG, Subtarget); 12777 case ISD::FDIV: 12778 return PerformVDIVCombine(N, DCI.DAG, Subtarget); 12779 case ISD::INTRINSIC_WO_CHAIN: return PerformIntrinsicCombine(N, DCI.DAG); 12780 case ISD::SHL: 12781 case ISD::SRA: 12782 case ISD::SRL: return PerformShiftCombine(N, DCI.DAG, Subtarget); 12783 case ISD::SIGN_EXTEND: 12784 case ISD::ZERO_EXTEND: 12785 case ISD::ANY_EXTEND: return PerformExtendCombine(N, DCI.DAG, Subtarget); 12786 case ARMISD::CMOV: return PerformCMOVCombine(N, DCI.DAG); 12787 case ARMISD::BRCOND: return PerformBRCONDCombine(N, DCI.DAG); 12788 case ISD::LOAD: return PerformLOADCombine(N, DCI); 12789 case ARMISD::VLD1DUP: 12790 case ARMISD::VLD2DUP: 12791 case ARMISD::VLD3DUP: 12792 case ARMISD::VLD4DUP: 12793 return PerformVLDCombine(N, DCI); 12794 case ARMISD::BUILD_VECTOR: 12795 return PerformARMBUILD_VECTORCombine(N, DCI); 12796 case ARMISD::SMULWB: { 12797 unsigned BitWidth = N->getValueType(0).getSizeInBits(); 12798 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16); 12799 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)) 12800 return SDValue(); 12801 break; 12802 } 12803 case ARMISD::SMULWT: { 12804 unsigned BitWidth = N->getValueType(0).getSizeInBits(); 12805 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16); 12806 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)) 12807 return SDValue(); 12808 break; 12809 } 12810 case ARMISD::SMLALBB: { 12811 unsigned BitWidth = N->getValueType(0).getSizeInBits(); 12812 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16); 12813 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || 12814 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))) 12815 return SDValue(); 12816 break; 12817 } 12818 case ARMISD::SMLALBT: { 12819 unsigned LowWidth = N->getOperand(0).getValueType().getSizeInBits(); 12820 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16); 12821 unsigned HighWidth = N->getOperand(1).getValueType().getSizeInBits(); 12822 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16); 12823 if ((SimplifyDemandedBits(N->getOperand(0), LowMask, DCI)) || 12824 (SimplifyDemandedBits(N->getOperand(1), HighMask, DCI))) 12825 return SDValue(); 12826 break; 12827 } 12828 case ARMISD::SMLALTB: { 12829 unsigned HighWidth = N->getOperand(0).getValueType().getSizeInBits(); 12830 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16); 12831 unsigned LowWidth = N->getOperand(1).getValueType().getSizeInBits(); 12832 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16); 12833 if ((SimplifyDemandedBits(N->getOperand(0), HighMask, DCI)) || 12834 (SimplifyDemandedBits(N->getOperand(1), LowMask, DCI))) 12835 return SDValue(); 12836 break; 12837 } 12838 case ARMISD::SMLALTT: { 12839 unsigned BitWidth = N->getValueType(0).getSizeInBits(); 12840 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16); 12841 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || 12842 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))) 12843 return SDValue(); 12844 break; 12845 } 12846 case ISD::INTRINSIC_VOID: 12847 case ISD::INTRINSIC_W_CHAIN: 12848 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 12849 case Intrinsic::arm_neon_vld1: 12850 case Intrinsic::arm_neon_vld1x2: 12851 case Intrinsic::arm_neon_vld1x3: 12852 case Intrinsic::arm_neon_vld1x4: 12853 case Intrinsic::arm_neon_vld2: 12854 case Intrinsic::arm_neon_vld3: 12855 case Intrinsic::arm_neon_vld4: 12856 case Intrinsic::arm_neon_vld2lane: 12857 case Intrinsic::arm_neon_vld3lane: 12858 case Intrinsic::arm_neon_vld4lane: 12859 case Intrinsic::arm_neon_vld2dup: 12860 case Intrinsic::arm_neon_vld3dup: 12861 case Intrinsic::arm_neon_vld4dup: 12862 case Intrinsic::arm_neon_vst1: 12863 case Intrinsic::arm_neon_vst1x2: 12864 case Intrinsic::arm_neon_vst1x3: 12865 case Intrinsic::arm_neon_vst1x4: 12866 case Intrinsic::arm_neon_vst2: 12867 case Intrinsic::arm_neon_vst3: 12868 case Intrinsic::arm_neon_vst4: 12869 case Intrinsic::arm_neon_vst2lane: 12870 case Intrinsic::arm_neon_vst3lane: 12871 case Intrinsic::arm_neon_vst4lane: 12872 return PerformVLDCombine(N, DCI); 12873 default: break; 12874 } 12875 break; 12876 } 12877 return SDValue(); 12878 } 12879 12880 bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc, 12881 EVT VT) const { 12882 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE); 12883 } 12884 12885 bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 12886 unsigned, 12887 unsigned, 12888 bool *Fast) const { 12889 // Depends what it gets converted into if the type is weird. 12890 if (!VT.isSimple()) 12891 return false; 12892 12893 // The AllowsUnaliged flag models the SCTLR.A setting in ARM cpus 12894 bool AllowsUnaligned = Subtarget->allowsUnalignedMem(); 12895 12896 switch (VT.getSimpleVT().SimpleTy) { 12897 default: 12898 return false; 12899 case MVT::i8: 12900 case MVT::i16: 12901 case MVT::i32: { 12902 // Unaligned access can use (for example) LRDB, LRDH, LDR 12903 if (AllowsUnaligned) { 12904 if (Fast) 12905 *Fast = Subtarget->hasV7Ops(); 12906 return true; 12907 } 12908 return false; 12909 } 12910 case MVT::f64: 12911 case MVT::v2f64: { 12912 // For any little-endian targets with neon, we can support unaligned ld/st 12913 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8. 12914 // A big-endian target may also explicitly support unaligned accesses 12915 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) { 12916 if (Fast) 12917 *Fast = true; 12918 return true; 12919 } 12920 return false; 12921 } 12922 } 12923 } 12924 12925 static bool memOpAlign(unsigned DstAlign, unsigned SrcAlign, 12926 unsigned AlignCheck) { 12927 return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) && 12928 (DstAlign == 0 || DstAlign % AlignCheck == 0)); 12929 } 12930 12931 EVT ARMTargetLowering::getOptimalMemOpType(uint64_t Size, 12932 unsigned DstAlign, unsigned SrcAlign, 12933 bool IsMemset, bool ZeroMemset, 12934 bool MemcpyStrSrc, 12935 MachineFunction &MF) const { 12936 const Function &F = MF.getFunction(); 12937 12938 // See if we can use NEON instructions for this... 12939 if ((!IsMemset || ZeroMemset) && Subtarget->hasNEON() && 12940 !F.hasFnAttribute(Attribute::NoImplicitFloat)) { 12941 bool Fast; 12942 if (Size >= 16 && 12943 (memOpAlign(SrcAlign, DstAlign, 16) || 12944 (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1, &Fast) && Fast))) { 12945 return MVT::v2f64; 12946 } else if (Size >= 8 && 12947 (memOpAlign(SrcAlign, DstAlign, 8) || 12948 (allowsMisalignedMemoryAccesses(MVT::f64, 0, 1, &Fast) && 12949 Fast))) { 12950 return MVT::f64; 12951 } 12952 } 12953 12954 // Let the target-independent logic figure it out. 12955 return MVT::Other; 12956 } 12957 12958 // 64-bit integers are split into their high and low parts and held in two 12959 // different registers, so the trunc is free since the low register can just 12960 // be used. 12961 bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const { 12962 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy()) 12963 return false; 12964 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits(); 12965 unsigned DestBits = DstTy->getPrimitiveSizeInBits(); 12966 return (SrcBits == 64 && DestBits == 32); 12967 } 12968 12969 bool ARMTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const { 12970 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() || 12971 !DstVT.isInteger()) 12972 return false; 12973 unsigned SrcBits = SrcVT.getSizeInBits(); 12974 unsigned DestBits = DstVT.getSizeInBits(); 12975 return (SrcBits == 64 && DestBits == 32); 12976 } 12977 12978 bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 12979 if (Val.getOpcode() != ISD::LOAD) 12980 return false; 12981 12982 EVT VT1 = Val.getValueType(); 12983 if (!VT1.isSimple() || !VT1.isInteger() || 12984 !VT2.isSimple() || !VT2.isInteger()) 12985 return false; 12986 12987 switch (VT1.getSimpleVT().SimpleTy) { 12988 default: break; 12989 case MVT::i1: 12990 case MVT::i8: 12991 case MVT::i16: 12992 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits. 12993 return true; 12994 } 12995 12996 return false; 12997 } 12998 12999 bool ARMTargetLowering::isFNegFree(EVT VT) const { 13000 if (!VT.isSimple()) 13001 return false; 13002 13003 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that 13004 // negate values directly (fneg is free). So, we don't want to let the DAG 13005 // combiner rewrite fneg into xors and some other instructions. For f16 and 13006 // FullFP16 argument passing, some bitcast nodes may be introduced, 13007 // triggering this DAG combine rewrite, so we are avoiding that with this. 13008 switch (VT.getSimpleVT().SimpleTy) { 13009 default: break; 13010 case MVT::f16: 13011 return Subtarget->hasFullFP16(); 13012 } 13013 13014 return false; 13015 } 13016 13017 bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const { 13018 EVT VT = ExtVal.getValueType(); 13019 13020 if (!isTypeLegal(VT)) 13021 return false; 13022 13023 // Don't create a loadext if we can fold the extension into a wide/long 13024 // instruction. 13025 // If there's more than one user instruction, the loadext is desirable no 13026 // matter what. There can be two uses by the same instruction. 13027 if (ExtVal->use_empty() || 13028 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode())) 13029 return true; 13030 13031 SDNode *U = *ExtVal->use_begin(); 13032 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB || 13033 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHL)) 13034 return false; 13035 13036 return true; 13037 } 13038 13039 bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const { 13040 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 13041 return false; 13042 13043 if (!isTypeLegal(EVT::getEVT(Ty1))) 13044 return false; 13045 13046 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop"); 13047 13048 // Assuming the caller doesn't have a zeroext or signext return parameter, 13049 // truncation all the way down to i1 is valid. 13050 return true; 13051 } 13052 13053 int ARMTargetLowering::getScalingFactorCost(const DataLayout &DL, 13054 const AddrMode &AM, Type *Ty, 13055 unsigned AS) const { 13056 if (isLegalAddressingMode(DL, AM, Ty, AS)) { 13057 if (Subtarget->hasFPAO()) 13058 return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster 13059 return 0; 13060 } 13061 return -1; 13062 } 13063 13064 static bool isLegalT1AddressImmediate(int64_t V, EVT VT) { 13065 if (V < 0) 13066 return false; 13067 13068 unsigned Scale = 1; 13069 switch (VT.getSimpleVT().SimpleTy) { 13070 default: return false; 13071 case MVT::i1: 13072 case MVT::i8: 13073 // Scale == 1; 13074 break; 13075 case MVT::i16: 13076 // Scale == 2; 13077 Scale = 2; 13078 break; 13079 case MVT::i32: 13080 // Scale == 4; 13081 Scale = 4; 13082 break; 13083 } 13084 13085 if ((V & (Scale - 1)) != 0) 13086 return false; 13087 V /= Scale; 13088 return V == (V & ((1LL << 5) - 1)); 13089 } 13090 13091 static bool isLegalT2AddressImmediate(int64_t V, EVT VT, 13092 const ARMSubtarget *Subtarget) { 13093 bool isNeg = false; 13094 if (V < 0) { 13095 isNeg = true; 13096 V = - V; 13097 } 13098 13099 switch (VT.getSimpleVT().SimpleTy) { 13100 default: return false; 13101 case MVT::i1: 13102 case MVT::i8: 13103 case MVT::i16: 13104 case MVT::i32: 13105 // + imm12 or - imm8 13106 if (isNeg) 13107 return V == (V & ((1LL << 8) - 1)); 13108 return V == (V & ((1LL << 12) - 1)); 13109 case MVT::f32: 13110 case MVT::f64: 13111 // Same as ARM mode. FIXME: NEON? 13112 if (!Subtarget->hasVFP2()) 13113 return false; 13114 if ((V & 3) != 0) 13115 return false; 13116 V >>= 2; 13117 return V == (V & ((1LL << 8) - 1)); 13118 } 13119 } 13120 13121 /// isLegalAddressImmediate - Return true if the integer value can be used 13122 /// as the offset of the target addressing mode for load / store of the 13123 /// given type. 13124 static bool isLegalAddressImmediate(int64_t V, EVT VT, 13125 const ARMSubtarget *Subtarget) { 13126 if (V == 0) 13127 return true; 13128 13129 if (!VT.isSimple()) 13130 return false; 13131 13132 if (Subtarget->isThumb1Only()) 13133 return isLegalT1AddressImmediate(V, VT); 13134 else if (Subtarget->isThumb2()) 13135 return isLegalT2AddressImmediate(V, VT, Subtarget); 13136 13137 // ARM mode. 13138 if (V < 0) 13139 V = - V; 13140 switch (VT.getSimpleVT().SimpleTy) { 13141 default: return false; 13142 case MVT::i1: 13143 case MVT::i8: 13144 case MVT::i32: 13145 // +- imm12 13146 return V == (V & ((1LL << 12) - 1)); 13147 case MVT::i16: 13148 // +- imm8 13149 return V == (V & ((1LL << 8) - 1)); 13150 case MVT::f32: 13151 case MVT::f64: 13152 if (!Subtarget->hasVFP2()) // FIXME: NEON? 13153 return false; 13154 if ((V & 3) != 0) 13155 return false; 13156 V >>= 2; 13157 return V == (V & ((1LL << 8) - 1)); 13158 } 13159 } 13160 13161 bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM, 13162 EVT VT) const { 13163 int Scale = AM.Scale; 13164 if (Scale < 0) 13165 return false; 13166 13167 switch (VT.getSimpleVT().SimpleTy) { 13168 default: return false; 13169 case MVT::i1: 13170 case MVT::i8: 13171 case MVT::i16: 13172 case MVT::i32: 13173 if (Scale == 1) 13174 return true; 13175 // r + r << imm 13176 Scale = Scale & ~1; 13177 return Scale == 2 || Scale == 4 || Scale == 8; 13178 case MVT::i64: 13179 // FIXME: What are we trying to model here? ldrd doesn't have an r + r 13180 // version in Thumb mode. 13181 // r + r 13182 if (Scale == 1) 13183 return true; 13184 // r * 2 (this can be lowered to r + r). 13185 if (!AM.HasBaseReg && Scale == 2) 13186 return true; 13187 return false; 13188 case MVT::isVoid: 13189 // Note, we allow "void" uses (basically, uses that aren't loads or 13190 // stores), because arm allows folding a scale into many arithmetic 13191 // operations. This should be made more precise and revisited later. 13192 13193 // Allow r << imm, but the imm has to be a multiple of two. 13194 if (Scale & 1) return false; 13195 return isPowerOf2_32(Scale); 13196 } 13197 } 13198 13199 bool ARMTargetLowering::isLegalT1ScaledAddressingMode(const AddrMode &AM, 13200 EVT VT) const { 13201 const int Scale = AM.Scale; 13202 13203 // Negative scales are not supported in Thumb1. 13204 if (Scale < 0) 13205 return false; 13206 13207 // Thumb1 addressing modes do not support register scaling excepting the 13208 // following cases: 13209 // 1. Scale == 1 means no scaling. 13210 // 2. Scale == 2 this can be lowered to r + r if there is no base register. 13211 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2); 13212 } 13213 13214 /// isLegalAddressingMode - Return true if the addressing mode represented 13215 /// by AM is legal for this target, for a load/store of the specified type. 13216 bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL, 13217 const AddrMode &AM, Type *Ty, 13218 unsigned AS, Instruction *I) const { 13219 EVT VT = getValueType(DL, Ty, true); 13220 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget)) 13221 return false; 13222 13223 // Can never fold addr of global into load/store. 13224 if (AM.BaseGV) 13225 return false; 13226 13227 switch (AM.Scale) { 13228 case 0: // no scale reg, must be "r+i" or "r", or "i". 13229 break; 13230 default: 13231 // ARM doesn't support any R+R*scale+imm addr modes. 13232 if (AM.BaseOffs) 13233 return false; 13234 13235 if (!VT.isSimple()) 13236 return false; 13237 13238 if (Subtarget->isThumb1Only()) 13239 return isLegalT1ScaledAddressingMode(AM, VT); 13240 13241 if (Subtarget->isThumb2()) 13242 return isLegalT2ScaledAddressingMode(AM, VT); 13243 13244 int Scale = AM.Scale; 13245 switch (VT.getSimpleVT().SimpleTy) { 13246 default: return false; 13247 case MVT::i1: 13248 case MVT::i8: 13249 case MVT::i32: 13250 if (Scale < 0) Scale = -Scale; 13251 if (Scale == 1) 13252 return true; 13253 // r + r << imm 13254 return isPowerOf2_32(Scale & ~1); 13255 case MVT::i16: 13256 case MVT::i64: 13257 // r +/- r 13258 if (Scale == 1 || (AM.HasBaseReg && Scale == -1)) 13259 return true; 13260 // r * 2 (this can be lowered to r + r). 13261 if (!AM.HasBaseReg && Scale == 2) 13262 return true; 13263 return false; 13264 13265 case MVT::isVoid: 13266 // Note, we allow "void" uses (basically, uses that aren't loads or 13267 // stores), because arm allows folding a scale into many arithmetic 13268 // operations. This should be made more precise and revisited later. 13269 13270 // Allow r << imm, but the imm has to be a multiple of two. 13271 if (Scale & 1) return false; 13272 return isPowerOf2_32(Scale); 13273 } 13274 } 13275 return true; 13276 } 13277 13278 /// isLegalICmpImmediate - Return true if the specified immediate is legal 13279 /// icmp immediate, that is the target has icmp instructions which can compare 13280 /// a register against the immediate without having to materialize the 13281 /// immediate into a register. 13282 bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 13283 // Thumb2 and ARM modes can use cmn for negative immediates. 13284 if (!Subtarget->isThumb()) 13285 return ARM_AM::getSOImmVal((uint32_t)Imm) != -1 || 13286 ARM_AM::getSOImmVal(-(uint32_t)Imm) != -1; 13287 if (Subtarget->isThumb2()) 13288 return ARM_AM::getT2SOImmVal((uint32_t)Imm) != -1 || 13289 ARM_AM::getT2SOImmVal(-(uint32_t)Imm) != -1; 13290 // Thumb1 doesn't have cmn, and only 8-bit immediates. 13291 return Imm >= 0 && Imm <= 255; 13292 } 13293 13294 /// isLegalAddImmediate - Return true if the specified immediate is a legal add 13295 /// *or sub* immediate, that is the target has add or sub instructions which can 13296 /// add a register with the immediate without having to materialize the 13297 /// immediate into a register. 13298 bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const { 13299 // Same encoding for add/sub, just flip the sign. 13300 int64_t AbsImm = std::abs(Imm); 13301 if (!Subtarget->isThumb()) 13302 return ARM_AM::getSOImmVal(AbsImm) != -1; 13303 if (Subtarget->isThumb2()) 13304 return ARM_AM::getT2SOImmVal(AbsImm) != -1; 13305 // Thumb1 only has 8-bit unsigned immediate. 13306 return AbsImm >= 0 && AbsImm <= 255; 13307 } 13308 13309 static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, 13310 bool isSEXTLoad, SDValue &Base, 13311 SDValue &Offset, bool &isInc, 13312 SelectionDAG &DAG) { 13313 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) 13314 return false; 13315 13316 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) { 13317 // AddressingMode 3 13318 Base = Ptr->getOperand(0); 13319 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { 13320 int RHSC = (int)RHS->getZExtValue(); 13321 if (RHSC < 0 && RHSC > -256) { 13322 assert(Ptr->getOpcode() == ISD::ADD); 13323 isInc = false; 13324 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); 13325 return true; 13326 } 13327 } 13328 isInc = (Ptr->getOpcode() == ISD::ADD); 13329 Offset = Ptr->getOperand(1); 13330 return true; 13331 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) { 13332 // AddressingMode 2 13333 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { 13334 int RHSC = (int)RHS->getZExtValue(); 13335 if (RHSC < 0 && RHSC > -0x1000) { 13336 assert(Ptr->getOpcode() == ISD::ADD); 13337 isInc = false; 13338 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); 13339 Base = Ptr->getOperand(0); 13340 return true; 13341 } 13342 } 13343 13344 if (Ptr->getOpcode() == ISD::ADD) { 13345 isInc = true; 13346 ARM_AM::ShiftOpc ShOpcVal= 13347 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode()); 13348 if (ShOpcVal != ARM_AM::no_shift) { 13349 Base = Ptr->getOperand(1); 13350 Offset = Ptr->getOperand(0); 13351 } else { 13352 Base = Ptr->getOperand(0); 13353 Offset = Ptr->getOperand(1); 13354 } 13355 return true; 13356 } 13357 13358 isInc = (Ptr->getOpcode() == ISD::ADD); 13359 Base = Ptr->getOperand(0); 13360 Offset = Ptr->getOperand(1); 13361 return true; 13362 } 13363 13364 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store. 13365 return false; 13366 } 13367 13368 static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, 13369 bool isSEXTLoad, SDValue &Base, 13370 SDValue &Offset, bool &isInc, 13371 SelectionDAG &DAG) { 13372 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) 13373 return false; 13374 13375 Base = Ptr->getOperand(0); 13376 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { 13377 int RHSC = (int)RHS->getZExtValue(); 13378 if (RHSC < 0 && RHSC > -0x100) { // 8 bits. 13379 assert(Ptr->getOpcode() == ISD::ADD); 13380 isInc = false; 13381 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); 13382 return true; 13383 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero. 13384 isInc = Ptr->getOpcode() == ISD::ADD; 13385 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0)); 13386 return true; 13387 } 13388 } 13389 13390 return false; 13391 } 13392 13393 /// getPreIndexedAddressParts - returns true by value, base pointer and 13394 /// offset pointer and addressing mode by reference if the node's address 13395 /// can be legally represented as pre-indexed load / store address. 13396 bool 13397 ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 13398 SDValue &Offset, 13399 ISD::MemIndexedMode &AM, 13400 SelectionDAG &DAG) const { 13401 if (Subtarget->isThumb1Only()) 13402 return false; 13403 13404 EVT VT; 13405 SDValue Ptr; 13406 bool isSEXTLoad = false; 13407 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 13408 Ptr = LD->getBasePtr(); 13409 VT = LD->getMemoryVT(); 13410 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; 13411 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 13412 Ptr = ST->getBasePtr(); 13413 VT = ST->getMemoryVT(); 13414 } else 13415 return false; 13416 13417 bool isInc; 13418 bool isLegal = false; 13419 if (Subtarget->isThumb2()) 13420 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base, 13421 Offset, isInc, DAG); 13422 else 13423 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base, 13424 Offset, isInc, DAG); 13425 if (!isLegal) 13426 return false; 13427 13428 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC; 13429 return true; 13430 } 13431 13432 /// getPostIndexedAddressParts - returns true by value, base pointer and 13433 /// offset pointer and addressing mode by reference if this node can be 13434 /// combined with a load / store to form a post-indexed load / store. 13435 bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, 13436 SDValue &Base, 13437 SDValue &Offset, 13438 ISD::MemIndexedMode &AM, 13439 SelectionDAG &DAG) const { 13440 EVT VT; 13441 SDValue Ptr; 13442 bool isSEXTLoad = false, isNonExt; 13443 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 13444 VT = LD->getMemoryVT(); 13445 Ptr = LD->getBasePtr(); 13446 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; 13447 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD; 13448 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 13449 VT = ST->getMemoryVT(); 13450 Ptr = ST->getBasePtr(); 13451 isNonExt = !ST->isTruncatingStore(); 13452 } else 13453 return false; 13454 13455 if (Subtarget->isThumb1Only()) { 13456 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It 13457 // must be non-extending/truncating, i32, with an offset of 4. 13458 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!"); 13459 if (Op->getOpcode() != ISD::ADD || !isNonExt) 13460 return false; 13461 auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1)); 13462 if (!RHS || RHS->getZExtValue() != 4) 13463 return false; 13464 13465 Offset = Op->getOperand(1); 13466 Base = Op->getOperand(0); 13467 AM = ISD::POST_INC; 13468 return true; 13469 } 13470 13471 bool isInc; 13472 bool isLegal = false; 13473 if (Subtarget->isThumb2()) 13474 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset, 13475 isInc, DAG); 13476 else 13477 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset, 13478 isInc, DAG); 13479 if (!isLegal) 13480 return false; 13481 13482 if (Ptr != Base) { 13483 // Swap base ptr and offset to catch more post-index load / store when 13484 // it's legal. In Thumb2 mode, offset must be an immediate. 13485 if (Ptr == Offset && Op->getOpcode() == ISD::ADD && 13486 !Subtarget->isThumb2()) 13487 std::swap(Base, Offset); 13488 13489 // Post-indexed load / store update the base pointer. 13490 if (Ptr != Base) 13491 return false; 13492 } 13493 13494 AM = isInc ? ISD::POST_INC : ISD::POST_DEC; 13495 return true; 13496 } 13497 13498 void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, 13499 KnownBits &Known, 13500 const APInt &DemandedElts, 13501 const SelectionDAG &DAG, 13502 unsigned Depth) const { 13503 unsigned BitWidth = Known.getBitWidth(); 13504 Known.resetAll(); 13505 switch (Op.getOpcode()) { 13506 default: break; 13507 case ARMISD::ADDC: 13508 case ARMISD::ADDE: 13509 case ARMISD::SUBC: 13510 case ARMISD::SUBE: 13511 // Special cases when we convert a carry to a boolean. 13512 if (Op.getResNo() == 0) { 13513 SDValue LHS = Op.getOperand(0); 13514 SDValue RHS = Op.getOperand(1); 13515 // (ADDE 0, 0, C) will give us a single bit. 13516 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(LHS) && 13517 isNullConstant(RHS)) { 13518 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 13519 return; 13520 } 13521 } 13522 break; 13523 case ARMISD::CMOV: { 13524 // Bits are known zero/one if known on the LHS and RHS. 13525 DAG.computeKnownBits(Op.getOperand(0), Known, Depth+1); 13526 if (Known.isUnknown()) 13527 return; 13528 13529 KnownBits KnownRHS; 13530 DAG.computeKnownBits(Op.getOperand(1), KnownRHS, Depth+1); 13531 Known.Zero &= KnownRHS.Zero; 13532 Known.One &= KnownRHS.One; 13533 return; 13534 } 13535 case ISD::INTRINSIC_W_CHAIN: { 13536 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1)); 13537 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue()); 13538 switch (IntID) { 13539 default: return; 13540 case Intrinsic::arm_ldaex: 13541 case Intrinsic::arm_ldrex: { 13542 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); 13543 unsigned MemBits = VT.getScalarSizeInBits(); 13544 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 13545 return; 13546 } 13547 } 13548 } 13549 case ARMISD::BFI: { 13550 // Conservatively, we can recurse down the first operand 13551 // and just mask out all affected bits. 13552 DAG.computeKnownBits(Op.getOperand(0), Known, Depth + 1); 13553 13554 // The operand to BFI is already a mask suitable for removing the bits it 13555 // sets. 13556 ConstantSDNode *CI = cast<ConstantSDNode>(Op.getOperand(2)); 13557 const APInt &Mask = CI->getAPIntValue(); 13558 Known.Zero &= Mask; 13559 Known.One &= Mask; 13560 return; 13561 } 13562 } 13563 } 13564 13565 //===----------------------------------------------------------------------===// 13566 // ARM Inline Assembly Support 13567 //===----------------------------------------------------------------------===// 13568 13569 bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const { 13570 // Looking for "rev" which is V6+. 13571 if (!Subtarget->hasV6Ops()) 13572 return false; 13573 13574 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue()); 13575 std::string AsmStr = IA->getAsmString(); 13576 SmallVector<StringRef, 4> AsmPieces; 13577 SplitString(AsmStr, AsmPieces, ";\n"); 13578 13579 switch (AsmPieces.size()) { 13580 default: return false; 13581 case 1: 13582 AsmStr = AsmPieces[0]; 13583 AsmPieces.clear(); 13584 SplitString(AsmStr, AsmPieces, " \t,"); 13585 13586 // rev $0, $1 13587 if (AsmPieces.size() == 3 && 13588 AsmPieces[0] == "rev" && AsmPieces[1] == "$0" && AsmPieces[2] == "$1" && 13589 IA->getConstraintString().compare(0, 4, "=l,l") == 0) { 13590 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 13591 if (Ty && Ty->getBitWidth() == 32) 13592 return IntrinsicLowering::LowerToByteSwap(CI); 13593 } 13594 break; 13595 } 13596 13597 return false; 13598 } 13599 13600 const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const { 13601 // At this point, we have to lower this constraint to something else, so we 13602 // lower it to an "r" or "w". However, by doing this we will force the result 13603 // to be in register, while the X constraint is much more permissive. 13604 // 13605 // Although we are correct (we are free to emit anything, without 13606 // constraints), we might break use cases that would expect us to be more 13607 // efficient and emit something else. 13608 if (!Subtarget->hasVFP2()) 13609 return "r"; 13610 if (ConstraintVT.isFloatingPoint()) 13611 return "w"; 13612 if (ConstraintVT.isVector() && Subtarget->hasNEON() && 13613 (ConstraintVT.getSizeInBits() == 64 || 13614 ConstraintVT.getSizeInBits() == 128)) 13615 return "w"; 13616 13617 return "r"; 13618 } 13619 13620 /// getConstraintType - Given a constraint letter, return the type of 13621 /// constraint it is for this target. 13622 ARMTargetLowering::ConstraintType 13623 ARMTargetLowering::getConstraintType(StringRef Constraint) const { 13624 if (Constraint.size() == 1) { 13625 switch (Constraint[0]) { 13626 default: break; 13627 case 'l': return C_RegisterClass; 13628 case 'w': return C_RegisterClass; 13629 case 'h': return C_RegisterClass; 13630 case 'x': return C_RegisterClass; 13631 case 't': return C_RegisterClass; 13632 case 'j': return C_Other; // Constant for movw. 13633 // An address with a single base register. Due to the way we 13634 // currently handle addresses it is the same as an 'r' memory constraint. 13635 case 'Q': return C_Memory; 13636 } 13637 } else if (Constraint.size() == 2) { 13638 switch (Constraint[0]) { 13639 default: break; 13640 // All 'U+' constraints are addresses. 13641 case 'U': return C_Memory; 13642 } 13643 } 13644 return TargetLowering::getConstraintType(Constraint); 13645 } 13646 13647 /// Examine constraint type and operand type and determine a weight value. 13648 /// This object must already have been set up with the operand type 13649 /// and the current alternative constraint selected. 13650 TargetLowering::ConstraintWeight 13651 ARMTargetLowering::getSingleConstraintMatchWeight( 13652 AsmOperandInfo &info, const char *constraint) const { 13653 ConstraintWeight weight = CW_Invalid; 13654 Value *CallOperandVal = info.CallOperandVal; 13655 // If we don't have a value, we can't do a match, 13656 // but allow it at the lowest weight. 13657 if (!CallOperandVal) 13658 return CW_Default; 13659 Type *type = CallOperandVal->getType(); 13660 // Look at the constraint type. 13661 switch (*constraint) { 13662 default: 13663 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 13664 break; 13665 case 'l': 13666 if (type->isIntegerTy()) { 13667 if (Subtarget->isThumb()) 13668 weight = CW_SpecificReg; 13669 else 13670 weight = CW_Register; 13671 } 13672 break; 13673 case 'w': 13674 if (type->isFloatingPointTy()) 13675 weight = CW_Register; 13676 break; 13677 } 13678 return weight; 13679 } 13680 13681 using RCPair = std::pair<unsigned, const TargetRegisterClass *>; 13682 13683 RCPair ARMTargetLowering::getRegForInlineAsmConstraint( 13684 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { 13685 if (Constraint.size() == 1) { 13686 // GCC ARM Constraint Letters 13687 switch (Constraint[0]) { 13688 case 'l': // Low regs or general regs. 13689 if (Subtarget->isThumb()) 13690 return RCPair(0U, &ARM::tGPRRegClass); 13691 return RCPair(0U, &ARM::GPRRegClass); 13692 case 'h': // High regs or no regs. 13693 if (Subtarget->isThumb()) 13694 return RCPair(0U, &ARM::hGPRRegClass); 13695 break; 13696 case 'r': 13697 if (Subtarget->isThumb1Only()) 13698 return RCPair(0U, &ARM::tGPRRegClass); 13699 return RCPair(0U, &ARM::GPRRegClass); 13700 case 'w': 13701 if (VT == MVT::Other) 13702 break; 13703 if (VT == MVT::f32) 13704 return RCPair(0U, &ARM::SPRRegClass); 13705 if (VT.getSizeInBits() == 64) 13706 return RCPair(0U, &ARM::DPRRegClass); 13707 if (VT.getSizeInBits() == 128) 13708 return RCPair(0U, &ARM::QPRRegClass); 13709 break; 13710 case 'x': 13711 if (VT == MVT::Other) 13712 break; 13713 if (VT == MVT::f32) 13714 return RCPair(0U, &ARM::SPR_8RegClass); 13715 if (VT.getSizeInBits() == 64) 13716 return RCPair(0U, &ARM::DPR_8RegClass); 13717 if (VT.getSizeInBits() == 128) 13718 return RCPair(0U, &ARM::QPR_8RegClass); 13719 break; 13720 case 't': 13721 if (VT == MVT::Other) 13722 break; 13723 if (VT == MVT::f32 || VT == MVT::i32) 13724 return RCPair(0U, &ARM::SPRRegClass); 13725 if (VT.getSizeInBits() == 64) 13726 return RCPair(0U, &ARM::DPR_VFP2RegClass); 13727 if (VT.getSizeInBits() == 128) 13728 return RCPair(0U, &ARM::QPR_VFP2RegClass); 13729 break; 13730 } 13731 } 13732 if (StringRef("{cc}").equals_lower(Constraint)) 13733 return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass); 13734 13735 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 13736 } 13737 13738 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 13739 /// vector. If it is invalid, don't add anything to Ops. 13740 void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op, 13741 std::string &Constraint, 13742 std::vector<SDValue>&Ops, 13743 SelectionDAG &DAG) const { 13744 SDValue Result; 13745 13746 // Currently only support length 1 constraints. 13747 if (Constraint.length() != 1) return; 13748 13749 char ConstraintLetter = Constraint[0]; 13750 switch (ConstraintLetter) { 13751 default: break; 13752 case 'j': 13753 case 'I': case 'J': case 'K': case 'L': 13754 case 'M': case 'N': case 'O': 13755 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 13756 if (!C) 13757 return; 13758 13759 int64_t CVal64 = C->getSExtValue(); 13760 int CVal = (int) CVal64; 13761 // None of these constraints allow values larger than 32 bits. Check 13762 // that the value fits in an int. 13763 if (CVal != CVal64) 13764 return; 13765 13766 switch (ConstraintLetter) { 13767 case 'j': 13768 // Constant suitable for movw, must be between 0 and 13769 // 65535. 13770 if (Subtarget->hasV6T2Ops()) 13771 if (CVal >= 0 && CVal <= 65535) 13772 break; 13773 return; 13774 case 'I': 13775 if (Subtarget->isThumb1Only()) { 13776 // This must be a constant between 0 and 255, for ADD 13777 // immediates. 13778 if (CVal >= 0 && CVal <= 255) 13779 break; 13780 } else if (Subtarget->isThumb2()) { 13781 // A constant that can be used as an immediate value in a 13782 // data-processing instruction. 13783 if (ARM_AM::getT2SOImmVal(CVal) != -1) 13784 break; 13785 } else { 13786 // A constant that can be used as an immediate value in a 13787 // data-processing instruction. 13788 if (ARM_AM::getSOImmVal(CVal) != -1) 13789 break; 13790 } 13791 return; 13792 13793 case 'J': 13794 if (Subtarget->isThumb1Only()) { 13795 // This must be a constant between -255 and -1, for negated ADD 13796 // immediates. This can be used in GCC with an "n" modifier that 13797 // prints the negated value, for use with SUB instructions. It is 13798 // not useful otherwise but is implemented for compatibility. 13799 if (CVal >= -255 && CVal <= -1) 13800 break; 13801 } else { 13802 // This must be a constant between -4095 and 4095. It is not clear 13803 // what this constraint is intended for. Implemented for 13804 // compatibility with GCC. 13805 if (CVal >= -4095 && CVal <= 4095) 13806 break; 13807 } 13808 return; 13809 13810 case 'K': 13811 if (Subtarget->isThumb1Only()) { 13812 // A 32-bit value where only one byte has a nonzero value. Exclude 13813 // zero to match GCC. This constraint is used by GCC internally for 13814 // constants that can be loaded with a move/shift combination. 13815 // It is not useful otherwise but is implemented for compatibility. 13816 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal)) 13817 break; 13818 } else if (Subtarget->isThumb2()) { 13819 // A constant whose bitwise inverse can be used as an immediate 13820 // value in a data-processing instruction. This can be used in GCC 13821 // with a "B" modifier that prints the inverted value, for use with 13822 // BIC and MVN instructions. It is not useful otherwise but is 13823 // implemented for compatibility. 13824 if (ARM_AM::getT2SOImmVal(~CVal) != -1) 13825 break; 13826 } else { 13827 // A constant whose bitwise inverse can be used as an immediate 13828 // value in a data-processing instruction. This can be used in GCC 13829 // with a "B" modifier that prints the inverted value, for use with 13830 // BIC and MVN instructions. It is not useful otherwise but is 13831 // implemented for compatibility. 13832 if (ARM_AM::getSOImmVal(~CVal) != -1) 13833 break; 13834 } 13835 return; 13836 13837 case 'L': 13838 if (Subtarget->isThumb1Only()) { 13839 // This must be a constant between -7 and 7, 13840 // for 3-operand ADD/SUB immediate instructions. 13841 if (CVal >= -7 && CVal < 7) 13842 break; 13843 } else if (Subtarget->isThumb2()) { 13844 // A constant whose negation can be used as an immediate value in a 13845 // data-processing instruction. This can be used in GCC with an "n" 13846 // modifier that prints the negated value, for use with SUB 13847 // instructions. It is not useful otherwise but is implemented for 13848 // compatibility. 13849 if (ARM_AM::getT2SOImmVal(-CVal) != -1) 13850 break; 13851 } else { 13852 // A constant whose negation can be used as an immediate value in a 13853 // data-processing instruction. This can be used in GCC with an "n" 13854 // modifier that prints the negated value, for use with SUB 13855 // instructions. It is not useful otherwise but is implemented for 13856 // compatibility. 13857 if (ARM_AM::getSOImmVal(-CVal) != -1) 13858 break; 13859 } 13860 return; 13861 13862 case 'M': 13863 if (Subtarget->isThumb1Only()) { 13864 // This must be a multiple of 4 between 0 and 1020, for 13865 // ADD sp + immediate. 13866 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0)) 13867 break; 13868 } else { 13869 // A power of two or a constant between 0 and 32. This is used in 13870 // GCC for the shift amount on shifted register operands, but it is 13871 // useful in general for any shift amounts. 13872 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0)) 13873 break; 13874 } 13875 return; 13876 13877 case 'N': 13878 if (Subtarget->isThumb()) { // FIXME thumb2 13879 // This must be a constant between 0 and 31, for shift amounts. 13880 if (CVal >= 0 && CVal <= 31) 13881 break; 13882 } 13883 return; 13884 13885 case 'O': 13886 if (Subtarget->isThumb()) { // FIXME thumb2 13887 // This must be a multiple of 4 between -508 and 508, for 13888 // ADD/SUB sp = sp + immediate. 13889 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0)) 13890 break; 13891 } 13892 return; 13893 } 13894 Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType()); 13895 break; 13896 } 13897 13898 if (Result.getNode()) { 13899 Ops.push_back(Result); 13900 return; 13901 } 13902 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 13903 } 13904 13905 static RTLIB::Libcall getDivRemLibcall( 13906 const SDNode *N, MVT::SimpleValueType SVT) { 13907 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || 13908 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && 13909 "Unhandled Opcode in getDivRemLibcall"); 13910 bool isSigned = N->getOpcode() == ISD::SDIVREM || 13911 N->getOpcode() == ISD::SREM; 13912 RTLIB::Libcall LC; 13913 switch (SVT) { 13914 default: llvm_unreachable("Unexpected request for libcall!"); 13915 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 13916 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 13917 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 13918 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 13919 } 13920 return LC; 13921 } 13922 13923 static TargetLowering::ArgListTy getDivRemArgList( 13924 const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget) { 13925 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || 13926 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && 13927 "Unhandled Opcode in getDivRemArgList"); 13928 bool isSigned = N->getOpcode() == ISD::SDIVREM || 13929 N->getOpcode() == ISD::SREM; 13930 TargetLowering::ArgListTy Args; 13931 TargetLowering::ArgListEntry Entry; 13932 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 13933 EVT ArgVT = N->getOperand(i).getValueType(); 13934 Type *ArgTy = ArgVT.getTypeForEVT(*Context); 13935 Entry.Node = N->getOperand(i); 13936 Entry.Ty = ArgTy; 13937 Entry.IsSExt = isSigned; 13938 Entry.IsZExt = !isSigned; 13939 Args.push_back(Entry); 13940 } 13941 if (Subtarget->isTargetWindows() && Args.size() >= 2) 13942 std::swap(Args[0], Args[1]); 13943 return Args; 13944 } 13945 13946 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const { 13947 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || 13948 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || 13949 Subtarget->isTargetWindows()) && 13950 "Register-based DivRem lowering only"); 13951 unsigned Opcode = Op->getOpcode(); 13952 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) && 13953 "Invalid opcode for Div/Rem lowering"); 13954 bool isSigned = (Opcode == ISD::SDIVREM); 13955 EVT VT = Op->getValueType(0); 13956 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); 13957 SDLoc dl(Op); 13958 13959 // If the target has hardware divide, use divide + multiply + subtract: 13960 // div = a / b 13961 // rem = a - b * div 13962 // return {div, rem} 13963 // This should be lowered into UDIV/SDIV + MLS later on. 13964 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode() 13965 : Subtarget->hasDivideInARMMode(); 13966 if (hasDivide && Op->getValueType(0).isSimple() && 13967 Op->getSimpleValueType(0) == MVT::i32) { 13968 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV; 13969 const SDValue Dividend = Op->getOperand(0); 13970 const SDValue Divisor = Op->getOperand(1); 13971 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor); 13972 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor); 13973 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); 13974 13975 SDValue Values[2] = {Div, Rem}; 13976 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values); 13977 } 13978 13979 RTLIB::Libcall LC = getDivRemLibcall(Op.getNode(), 13980 VT.getSimpleVT().SimpleTy); 13981 SDValue InChain = DAG.getEntryNode(); 13982 13983 TargetLowering::ArgListTy Args = getDivRemArgList(Op.getNode(), 13984 DAG.getContext(), 13985 Subtarget); 13986 13987 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC), 13988 getPointerTy(DAG.getDataLayout())); 13989 13990 Type *RetTy = StructType::get(Ty, Ty); 13991 13992 if (Subtarget->isTargetWindows()) 13993 InChain = WinDBZCheckDenominator(DAG, Op.getNode(), InChain); 13994 13995 TargetLowering::CallLoweringInfo CLI(DAG); 13996 CLI.setDebugLoc(dl).setChain(InChain) 13997 .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args)) 13998 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned); 13999 14000 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI); 14001 return CallInfo.first; 14002 } 14003 14004 // Lowers REM using divmod helpers 14005 // see RTABI section 4.2/4.3 14006 SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const { 14007 // Build return types (div and rem) 14008 std::vector<Type*> RetTyParams; 14009 Type *RetTyElement; 14010 14011 switch (N->getValueType(0).getSimpleVT().SimpleTy) { 14012 default: llvm_unreachable("Unexpected request for libcall!"); 14013 case MVT::i8: RetTyElement = Type::getInt8Ty(*DAG.getContext()); break; 14014 case MVT::i16: RetTyElement = Type::getInt16Ty(*DAG.getContext()); break; 14015 case MVT::i32: RetTyElement = Type::getInt32Ty(*DAG.getContext()); break; 14016 case MVT::i64: RetTyElement = Type::getInt64Ty(*DAG.getContext()); break; 14017 } 14018 14019 RetTyParams.push_back(RetTyElement); 14020 RetTyParams.push_back(RetTyElement); 14021 ArrayRef<Type*> ret = ArrayRef<Type*>(RetTyParams); 14022 Type *RetTy = StructType::get(*DAG.getContext(), ret); 14023 14024 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT(). 14025 SimpleTy); 14026 SDValue InChain = DAG.getEntryNode(); 14027 TargetLowering::ArgListTy Args = getDivRemArgList(N, DAG.getContext(), 14028 Subtarget); 14029 bool isSigned = N->getOpcode() == ISD::SREM; 14030 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC), 14031 getPointerTy(DAG.getDataLayout())); 14032 14033 if (Subtarget->isTargetWindows()) 14034 InChain = WinDBZCheckDenominator(DAG, N, InChain); 14035 14036 // Lower call 14037 CallLoweringInfo CLI(DAG); 14038 CLI.setChain(InChain) 14039 .setCallee(CallingConv::ARM_AAPCS, RetTy, Callee, std::move(Args)) 14040 .setSExtResult(isSigned).setZExtResult(!isSigned).setDebugLoc(SDLoc(N)); 14041 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 14042 14043 // Return second (rem) result operand (first contains div) 14044 SDNode *ResNode = CallResult.first.getNode(); 14045 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands"); 14046 return ResNode->getOperand(1); 14047 } 14048 14049 SDValue 14050 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 14051 assert(Subtarget->isTargetWindows() && "unsupported target platform"); 14052 SDLoc DL(Op); 14053 14054 // Get the inputs. 14055 SDValue Chain = Op.getOperand(0); 14056 SDValue Size = Op.getOperand(1); 14057 14058 if (DAG.getMachineFunction().getFunction().hasFnAttribute( 14059 "no-stack-arg-probe")) { 14060 unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 14061 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32); 14062 Chain = SP.getValue(1); 14063 SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size); 14064 if (Align) 14065 SP = DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0), 14066 DAG.getConstant(-(uint64_t)Align, DL, MVT::i32)); 14067 Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP); 14068 SDValue Ops[2] = { SP, Chain }; 14069 return DAG.getMergeValues(Ops, DL); 14070 } 14071 14072 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size, 14073 DAG.getConstant(2, DL, MVT::i32)); 14074 14075 SDValue Flag; 14076 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Flag); 14077 Flag = Chain.getValue(1); 14078 14079 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 14080 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Flag); 14081 14082 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32); 14083 Chain = NewSP.getValue(1); 14084 14085 SDValue Ops[2] = { NewSP, Chain }; 14086 return DAG.getMergeValues(Ops, DL); 14087 } 14088 14089 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { 14090 assert(Op.getValueType() == MVT::f64 && Subtarget->isFPOnlySP() && 14091 "Unexpected type for custom-lowering FP_EXTEND"); 14092 14093 RTLIB::Libcall LC; 14094 LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType()); 14095 14096 SDValue SrcVal = Op.getOperand(0); 14097 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false, 14098 SDLoc(Op)).first; 14099 } 14100 14101 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { 14102 assert(Op.getOperand(0).getValueType() == MVT::f64 && 14103 Subtarget->isFPOnlySP() && 14104 "Unexpected type for custom-lowering FP_ROUND"); 14105 14106 RTLIB::Libcall LC; 14107 LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType()); 14108 14109 SDValue SrcVal = Op.getOperand(0); 14110 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false, 14111 SDLoc(Op)).first; 14112 } 14113 14114 bool 14115 ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 14116 // The ARM target isn't yet aware of offsets. 14117 return false; 14118 } 14119 14120 bool ARM::isBitFieldInvertedMask(unsigned v) { 14121 if (v == 0xffffffff) 14122 return false; 14123 14124 // there can be 1's on either or both "outsides", all the "inside" 14125 // bits must be 0's 14126 return isShiftedMask_32(~v); 14127 } 14128 14129 /// isFPImmLegal - Returns true if the target can instruction select the 14130 /// specified FP immediate natively. If false, the legalizer will 14131 /// materialize the FP immediate as a load from a constant pool. 14132 bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 14133 if (!Subtarget->hasVFP3()) 14134 return false; 14135 if (VT == MVT::f16 && Subtarget->hasFullFP16()) 14136 return ARM_AM::getFP16Imm(Imm) != -1; 14137 if (VT == MVT::f32) 14138 return ARM_AM::getFP32Imm(Imm) != -1; 14139 if (VT == MVT::f64 && !Subtarget->isFPOnlySP()) 14140 return ARM_AM::getFP64Imm(Imm) != -1; 14141 return false; 14142 } 14143 14144 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as 14145 /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment 14146 /// specified in the intrinsic calls. 14147 bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 14148 const CallInst &I, 14149 MachineFunction &MF, 14150 unsigned Intrinsic) const { 14151 switch (Intrinsic) { 14152 case Intrinsic::arm_neon_vld1: 14153 case Intrinsic::arm_neon_vld2: 14154 case Intrinsic::arm_neon_vld3: 14155 case Intrinsic::arm_neon_vld4: 14156 case Intrinsic::arm_neon_vld2lane: 14157 case Intrinsic::arm_neon_vld3lane: 14158 case Intrinsic::arm_neon_vld4lane: 14159 case Intrinsic::arm_neon_vld2dup: 14160 case Intrinsic::arm_neon_vld3dup: 14161 case Intrinsic::arm_neon_vld4dup: { 14162 Info.opc = ISD::INTRINSIC_W_CHAIN; 14163 // Conservatively set memVT to the entire set of vectors loaded. 14164 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); 14165 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64; 14166 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); 14167 Info.ptrVal = I.getArgOperand(0); 14168 Info.offset = 0; 14169 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1); 14170 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue(); 14171 // volatile loads with NEON intrinsics not supported 14172 Info.flags = MachineMemOperand::MOLoad; 14173 return true; 14174 } 14175 case Intrinsic::arm_neon_vld1x2: 14176 case Intrinsic::arm_neon_vld1x3: 14177 case Intrinsic::arm_neon_vld1x4: { 14178 Info.opc = ISD::INTRINSIC_W_CHAIN; 14179 // Conservatively set memVT to the entire set of vectors loaded. 14180 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); 14181 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64; 14182 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); 14183 Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1); 14184 Info.offset = 0; 14185 Info.align = 0; 14186 // volatile loads with NEON intrinsics not supported 14187 Info.flags = MachineMemOperand::MOLoad; 14188 return true; 14189 } 14190 case Intrinsic::arm_neon_vst1: 14191 case Intrinsic::arm_neon_vst2: 14192 case Intrinsic::arm_neon_vst3: 14193 case Intrinsic::arm_neon_vst4: 14194 case Intrinsic::arm_neon_vst2lane: 14195 case Intrinsic::arm_neon_vst3lane: 14196 case Intrinsic::arm_neon_vst4lane: { 14197 Info.opc = ISD::INTRINSIC_VOID; 14198 // Conservatively set memVT to the entire set of vectors stored. 14199 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); 14200 unsigned NumElts = 0; 14201 for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) { 14202 Type *ArgTy = I.getArgOperand(ArgI)->getType(); 14203 if (!ArgTy->isVectorTy()) 14204 break; 14205 NumElts += DL.getTypeSizeInBits(ArgTy) / 64; 14206 } 14207 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); 14208 Info.ptrVal = I.getArgOperand(0); 14209 Info.offset = 0; 14210 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1); 14211 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue(); 14212 // volatile stores with NEON intrinsics not supported 14213 Info.flags = MachineMemOperand::MOStore; 14214 return true; 14215 } 14216 case Intrinsic::arm_neon_vst1x2: 14217 case Intrinsic::arm_neon_vst1x3: 14218 case Intrinsic::arm_neon_vst1x4: { 14219 Info.opc = ISD::INTRINSIC_VOID; 14220 // Conservatively set memVT to the entire set of vectors stored. 14221 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); 14222 unsigned NumElts = 0; 14223 for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) { 14224 Type *ArgTy = I.getArgOperand(ArgI)->getType(); 14225 if (!ArgTy->isVectorTy()) 14226 break; 14227 NumElts += DL.getTypeSizeInBits(ArgTy) / 64; 14228 } 14229 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); 14230 Info.ptrVal = I.getArgOperand(0); 14231 Info.offset = 0; 14232 Info.align = 0; 14233 // volatile stores with NEON intrinsics not supported 14234 Info.flags = MachineMemOperand::MOStore; 14235 return true; 14236 } 14237 case Intrinsic::arm_ldaex: 14238 case Intrinsic::arm_ldrex: { 14239 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); 14240 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType()); 14241 Info.opc = ISD::INTRINSIC_W_CHAIN; 14242 Info.memVT = MVT::getVT(PtrTy->getElementType()); 14243 Info.ptrVal = I.getArgOperand(0); 14244 Info.offset = 0; 14245 Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); 14246 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; 14247 return true; 14248 } 14249 case Intrinsic::arm_stlex: 14250 case Intrinsic::arm_strex: { 14251 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); 14252 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType()); 14253 Info.opc = ISD::INTRINSIC_W_CHAIN; 14254 Info.memVT = MVT::getVT(PtrTy->getElementType()); 14255 Info.ptrVal = I.getArgOperand(1); 14256 Info.offset = 0; 14257 Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); 14258 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; 14259 return true; 14260 } 14261 case Intrinsic::arm_stlexd: 14262 case Intrinsic::arm_strexd: 14263 Info.opc = ISD::INTRINSIC_W_CHAIN; 14264 Info.memVT = MVT::i64; 14265 Info.ptrVal = I.getArgOperand(2); 14266 Info.offset = 0; 14267 Info.align = 8; 14268 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; 14269 return true; 14270 14271 case Intrinsic::arm_ldaexd: 14272 case Intrinsic::arm_ldrexd: 14273 Info.opc = ISD::INTRINSIC_W_CHAIN; 14274 Info.memVT = MVT::i64; 14275 Info.ptrVal = I.getArgOperand(0); 14276 Info.offset = 0; 14277 Info.align = 8; 14278 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; 14279 return true; 14280 14281 default: 14282 break; 14283 } 14284 14285 return false; 14286 } 14287 14288 /// Returns true if it is beneficial to convert a load of a constant 14289 /// to just the constant itself. 14290 bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 14291 Type *Ty) const { 14292 assert(Ty->isIntegerTy()); 14293 14294 unsigned Bits = Ty->getPrimitiveSizeInBits(); 14295 if (Bits == 0 || Bits > 32) 14296 return false; 14297 return true; 14298 } 14299 14300 bool ARMTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, 14301 unsigned Index) const { 14302 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) 14303 return false; 14304 14305 return (Index == 0 || Index == ResVT.getVectorNumElements()); 14306 } 14307 14308 Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder, 14309 ARM_MB::MemBOpt Domain) const { 14310 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 14311 14312 // First, if the target has no DMB, see what fallback we can use. 14313 if (!Subtarget->hasDataBarrier()) { 14314 // Some ARMv6 cpus can support data barriers with an mcr instruction. 14315 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get 14316 // here. 14317 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) { 14318 Function *MCR = Intrinsic::getDeclaration(M, Intrinsic::arm_mcr); 14319 Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0), 14320 Builder.getInt32(0), Builder.getInt32(7), 14321 Builder.getInt32(10), Builder.getInt32(5)}; 14322 return Builder.CreateCall(MCR, args); 14323 } else { 14324 // Instead of using barriers, atomic accesses on these subtargets use 14325 // libcalls. 14326 llvm_unreachable("makeDMB on a target so old that it has no barriers"); 14327 } 14328 } else { 14329 Function *DMB = Intrinsic::getDeclaration(M, Intrinsic::arm_dmb); 14330 // Only a full system barrier exists in the M-class architectures. 14331 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain; 14332 Constant *CDomain = Builder.getInt32(Domain); 14333 return Builder.CreateCall(DMB, CDomain); 14334 } 14335 } 14336 14337 // Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html 14338 Instruction *ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder, 14339 Instruction *Inst, 14340 AtomicOrdering Ord) const { 14341 switch (Ord) { 14342 case AtomicOrdering::NotAtomic: 14343 case AtomicOrdering::Unordered: 14344 llvm_unreachable("Invalid fence: unordered/non-atomic"); 14345 case AtomicOrdering::Monotonic: 14346 case AtomicOrdering::Acquire: 14347 return nullptr; // Nothing to do 14348 case AtomicOrdering::SequentiallyConsistent: 14349 if (!Inst->hasAtomicStore()) 14350 return nullptr; // Nothing to do 14351 LLVM_FALLTHROUGH; 14352 case AtomicOrdering::Release: 14353 case AtomicOrdering::AcquireRelease: 14354 if (Subtarget->preferISHSTBarriers()) 14355 return makeDMB(Builder, ARM_MB::ISHST); 14356 // FIXME: add a comment with a link to documentation justifying this. 14357 else 14358 return makeDMB(Builder, ARM_MB::ISH); 14359 } 14360 llvm_unreachable("Unknown fence ordering in emitLeadingFence"); 14361 } 14362 14363 Instruction *ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder, 14364 Instruction *Inst, 14365 AtomicOrdering Ord) const { 14366 switch (Ord) { 14367 case AtomicOrdering::NotAtomic: 14368 case AtomicOrdering::Unordered: 14369 llvm_unreachable("Invalid fence: unordered/not-atomic"); 14370 case AtomicOrdering::Monotonic: 14371 case AtomicOrdering::Release: 14372 return nullptr; // Nothing to do 14373 case AtomicOrdering::Acquire: 14374 case AtomicOrdering::AcquireRelease: 14375 case AtomicOrdering::SequentiallyConsistent: 14376 return makeDMB(Builder, ARM_MB::ISH); 14377 } 14378 llvm_unreachable("Unknown fence ordering in emitTrailingFence"); 14379 } 14380 14381 // Loads and stores less than 64-bits are already atomic; ones above that 14382 // are doomed anyway, so defer to the default libcall and blame the OS when 14383 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit 14384 // anything for those. 14385 bool ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const { 14386 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits(); 14387 return (Size == 64) && !Subtarget->isMClass(); 14388 } 14389 14390 // Loads and stores less than 64-bits are already atomic; ones above that 14391 // are doomed anyway, so defer to the default libcall and blame the OS when 14392 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit 14393 // anything for those. 14394 // FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that 14395 // guarantee, see DDI0406C ARM architecture reference manual, 14396 // sections A8.8.72-74 LDRD) 14397 TargetLowering::AtomicExpansionKind 14398 ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { 14399 unsigned Size = LI->getType()->getPrimitiveSizeInBits(); 14400 return ((Size == 64) && !Subtarget->isMClass()) ? AtomicExpansionKind::LLOnly 14401 : AtomicExpansionKind::None; 14402 } 14403 14404 // For the real atomic operations, we have ldrex/strex up to 32 bits, 14405 // and up to 64 bits on the non-M profiles 14406 TargetLowering::AtomicExpansionKind 14407 ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { 14408 unsigned Size = AI->getType()->getPrimitiveSizeInBits(); 14409 bool hasAtomicRMW = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); 14410 return (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) 14411 ? AtomicExpansionKind::LLSC 14412 : AtomicExpansionKind::None; 14413 } 14414 14415 bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR( 14416 AtomicCmpXchgInst *AI) const { 14417 // At -O0, fast-regalloc cannot cope with the live vregs necessary to 14418 // implement cmpxchg without spilling. If the address being exchanged is also 14419 // on the stack and close enough to the spill slot, this can lead to a 14420 // situation where the monitor always gets cleared and the atomic operation 14421 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. 14422 bool hasAtomicCmpXchg = 14423 !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); 14424 return getTargetMachine().getOptLevel() != 0 && hasAtomicCmpXchg; 14425 } 14426 14427 bool ARMTargetLowering::shouldInsertFencesForAtomic( 14428 const Instruction *I) const { 14429 return InsertFencesForAtomic; 14430 } 14431 14432 // This has so far only been implemented for MachO. 14433 bool ARMTargetLowering::useLoadStackGuardNode() const { 14434 return Subtarget->isTargetMachO(); 14435 } 14436 14437 bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx, 14438 unsigned &Cost) const { 14439 // If we do not have NEON, vector types are not natively supported. 14440 if (!Subtarget->hasNEON()) 14441 return false; 14442 14443 // Floating point values and vector values map to the same register file. 14444 // Therefore, although we could do a store extract of a vector type, this is 14445 // better to leave at float as we have more freedom in the addressing mode for 14446 // those. 14447 if (VectorTy->isFPOrFPVectorTy()) 14448 return false; 14449 14450 // If the index is unknown at compile time, this is very expensive to lower 14451 // and it is not possible to combine the store with the extract. 14452 if (!isa<ConstantInt>(Idx)) 14453 return false; 14454 14455 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type"); 14456 unsigned BitWidth = cast<VectorType>(VectorTy)->getBitWidth(); 14457 // We can do a store + vector extract on any vector that fits perfectly in a D 14458 // or Q register. 14459 if (BitWidth == 64 || BitWidth == 128) { 14460 Cost = 0; 14461 return true; 14462 } 14463 return false; 14464 } 14465 14466 bool ARMTargetLowering::isCheapToSpeculateCttz() const { 14467 return Subtarget->hasV6T2Ops(); 14468 } 14469 14470 bool ARMTargetLowering::isCheapToSpeculateCtlz() const { 14471 return Subtarget->hasV6T2Ops(); 14472 } 14473 14474 Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, 14475 AtomicOrdering Ord) const { 14476 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 14477 Type *ValTy = cast<PointerType>(Addr->getType())->getElementType(); 14478 bool IsAcquire = isAcquireOrStronger(Ord); 14479 14480 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd 14481 // intrinsic must return {i32, i32} and we have to recombine them into a 14482 // single i64 here. 14483 if (ValTy->getPrimitiveSizeInBits() == 64) { 14484 Intrinsic::ID Int = 14485 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd; 14486 Function *Ldrex = Intrinsic::getDeclaration(M, Int); 14487 14488 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); 14489 Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi"); 14490 14491 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); 14492 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); 14493 if (!Subtarget->isLittle()) 14494 std::swap (Lo, Hi); 14495 Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); 14496 Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); 14497 return Builder.CreateOr( 14498 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64"); 14499 } 14500 14501 Type *Tys[] = { Addr->getType() }; 14502 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex; 14503 Function *Ldrex = Intrinsic::getDeclaration(M, Int, Tys); 14504 14505 return Builder.CreateTruncOrBitCast( 14506 Builder.CreateCall(Ldrex, Addr), 14507 cast<PointerType>(Addr->getType())->getElementType()); 14508 } 14509 14510 void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance( 14511 IRBuilder<> &Builder) const { 14512 if (!Subtarget->hasV7Ops()) 14513 return; 14514 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 14515 Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::arm_clrex)); 14516 } 14517 14518 Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val, 14519 Value *Addr, 14520 AtomicOrdering Ord) const { 14521 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 14522 bool IsRelease = isReleaseOrStronger(Ord); 14523 14524 // Since the intrinsics must have legal type, the i64 intrinsics take two 14525 // parameters: "i32, i32". We must marshal Val into the appropriate form 14526 // before the call. 14527 if (Val->getType()->getPrimitiveSizeInBits() == 64) { 14528 Intrinsic::ID Int = 14529 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd; 14530 Function *Strex = Intrinsic::getDeclaration(M, Int); 14531 Type *Int32Ty = Type::getInt32Ty(M->getContext()); 14532 14533 Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo"); 14534 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi"); 14535 if (!Subtarget->isLittle()) 14536 std::swap(Lo, Hi); 14537 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); 14538 return Builder.CreateCall(Strex, {Lo, Hi, Addr}); 14539 } 14540 14541 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex; 14542 Type *Tys[] = { Addr->getType() }; 14543 Function *Strex = Intrinsic::getDeclaration(M, Int, Tys); 14544 14545 return Builder.CreateCall( 14546 Strex, {Builder.CreateZExtOrBitCast( 14547 Val, Strex->getFunctionType()->getParamType(0)), 14548 Addr}); 14549 } 14550 14551 /// A helper function for determining the number of interleaved accesses we 14552 /// will generate when lowering accesses of the given type. 14553 unsigned 14554 ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy, 14555 const DataLayout &DL) const { 14556 return (DL.getTypeSizeInBits(VecTy) + 127) / 128; 14557 } 14558 14559 bool ARMTargetLowering::isLegalInterleavedAccessType( 14560 VectorType *VecTy, const DataLayout &DL) const { 14561 14562 unsigned VecSize = DL.getTypeSizeInBits(VecTy); 14563 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType()); 14564 14565 // Ensure the vector doesn't have f16 elements. Even though we could do an 14566 // i16 vldN, we can't hold the f16 vectors and will end up converting via 14567 // f32. 14568 if (VecTy->getElementType()->isHalfTy()) 14569 return false; 14570 14571 // Ensure the number of vector elements is greater than 1. 14572 if (VecTy->getNumElements() < 2) 14573 return false; 14574 14575 // Ensure the element type is legal. 14576 if (ElSize != 8 && ElSize != 16 && ElSize != 32) 14577 return false; 14578 14579 // Ensure the total vector size is 64 or a multiple of 128. Types larger than 14580 // 128 will be split into multiple interleaved accesses. 14581 return VecSize == 64 || VecSize % 128 == 0; 14582 } 14583 14584 /// Lower an interleaved load into a vldN intrinsic. 14585 /// 14586 /// E.g. Lower an interleaved load (Factor = 2): 14587 /// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4 14588 /// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements 14589 /// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements 14590 /// 14591 /// Into: 14592 /// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4) 14593 /// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0 14594 /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1 14595 bool ARMTargetLowering::lowerInterleavedLoad( 14596 LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles, 14597 ArrayRef<unsigned> Indices, unsigned Factor) const { 14598 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && 14599 "Invalid interleave factor"); 14600 assert(!Shuffles.empty() && "Empty shufflevector input"); 14601 assert(Shuffles.size() == Indices.size() && 14602 "Unmatched number of shufflevectors and indices"); 14603 14604 VectorType *VecTy = Shuffles[0]->getType(); 14605 Type *EltTy = VecTy->getVectorElementType(); 14606 14607 const DataLayout &DL = LI->getModule()->getDataLayout(); 14608 14609 // Skip if we do not have NEON and skip illegal vector types. We can 14610 // "legalize" wide vector types into multiple interleaved accesses as long as 14611 // the vector types are divisible by 128. 14612 if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VecTy, DL)) 14613 return false; 14614 14615 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL); 14616 14617 // A pointer vector can not be the return type of the ldN intrinsics. Need to 14618 // load integer vectors first and then convert to pointer vectors. 14619 if (EltTy->isPointerTy()) 14620 VecTy = 14621 VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements()); 14622 14623 IRBuilder<> Builder(LI); 14624 14625 // The base address of the load. 14626 Value *BaseAddr = LI->getPointerOperand(); 14627 14628 if (NumLoads > 1) { 14629 // If we're going to generate more than one load, reset the sub-vector type 14630 // to something legal. 14631 VecTy = VectorType::get(VecTy->getVectorElementType(), 14632 VecTy->getVectorNumElements() / NumLoads); 14633 14634 // We will compute the pointer operand of each load from the original base 14635 // address using GEPs. Cast the base address to a pointer to the scalar 14636 // element type. 14637 BaseAddr = Builder.CreateBitCast( 14638 BaseAddr, VecTy->getVectorElementType()->getPointerTo( 14639 LI->getPointerAddressSpace())); 14640 } 14641 14642 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!"); 14643 14644 Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace()); 14645 Type *Tys[] = {VecTy, Int8Ptr}; 14646 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2, 14647 Intrinsic::arm_neon_vld3, 14648 Intrinsic::arm_neon_vld4}; 14649 Function *VldnFunc = 14650 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys); 14651 14652 // Holds sub-vectors extracted from the load intrinsic return values. The 14653 // sub-vectors are associated with the shufflevector instructions they will 14654 // replace. 14655 DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs; 14656 14657 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) { 14658 // If we're generating more than one load, compute the base address of 14659 // subsequent loads as an offset from the previous. 14660 if (LoadCount > 0) 14661 BaseAddr = Builder.CreateConstGEP1_32( 14662 BaseAddr, VecTy->getVectorNumElements() * Factor); 14663 14664 SmallVector<Value *, 2> Ops; 14665 Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr)); 14666 Ops.push_back(Builder.getInt32(LI->getAlignment())); 14667 14668 CallInst *VldN = Builder.CreateCall(VldnFunc, Ops, "vldN"); 14669 14670 // Replace uses of each shufflevector with the corresponding vector loaded 14671 // by ldN. 14672 for (unsigned i = 0; i < Shuffles.size(); i++) { 14673 ShuffleVectorInst *SV = Shuffles[i]; 14674 unsigned Index = Indices[i]; 14675 14676 Value *SubVec = Builder.CreateExtractValue(VldN, Index); 14677 14678 // Convert the integer vector to pointer vector if the element is pointer. 14679 if (EltTy->isPointerTy()) 14680 SubVec = Builder.CreateIntToPtr( 14681 SubVec, VectorType::get(SV->getType()->getVectorElementType(), 14682 VecTy->getVectorNumElements())); 14683 14684 SubVecs[SV].push_back(SubVec); 14685 } 14686 } 14687 14688 // Replace uses of the shufflevector instructions with the sub-vectors 14689 // returned by the load intrinsic. If a shufflevector instruction is 14690 // associated with more than one sub-vector, those sub-vectors will be 14691 // concatenated into a single wide vector. 14692 for (ShuffleVectorInst *SVI : Shuffles) { 14693 auto &SubVec = SubVecs[SVI]; 14694 auto *WideVec = 14695 SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0]; 14696 SVI->replaceAllUsesWith(WideVec); 14697 } 14698 14699 return true; 14700 } 14701 14702 /// Lower an interleaved store into a vstN intrinsic. 14703 /// 14704 /// E.g. Lower an interleaved store (Factor = 3): 14705 /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1, 14706 /// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> 14707 /// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4 14708 /// 14709 /// Into: 14710 /// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3> 14711 /// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7> 14712 /// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11> 14713 /// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4) 14714 /// 14715 /// Note that the new shufflevectors will be removed and we'll only generate one 14716 /// vst3 instruction in CodeGen. 14717 /// 14718 /// Example for a more general valid mask (Factor 3). Lower: 14719 /// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1, 14720 /// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> 14721 /// store <12 x i32> %i.vec, <12 x i32>* %ptr 14722 /// 14723 /// Into: 14724 /// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7> 14725 /// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35> 14726 /// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19> 14727 /// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4) 14728 bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI, 14729 ShuffleVectorInst *SVI, 14730 unsigned Factor) const { 14731 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && 14732 "Invalid interleave factor"); 14733 14734 VectorType *VecTy = SVI->getType(); 14735 assert(VecTy->getVectorNumElements() % Factor == 0 && 14736 "Invalid interleaved store"); 14737 14738 unsigned LaneLen = VecTy->getVectorNumElements() / Factor; 14739 Type *EltTy = VecTy->getVectorElementType(); 14740 VectorType *SubVecTy = VectorType::get(EltTy, LaneLen); 14741 14742 const DataLayout &DL = SI->getModule()->getDataLayout(); 14743 14744 // Skip if we do not have NEON and skip illegal vector types. We can 14745 // "legalize" wide vector types into multiple interleaved accesses as long as 14746 // the vector types are divisible by 128. 14747 if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(SubVecTy, DL)) 14748 return false; 14749 14750 unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL); 14751 14752 Value *Op0 = SVI->getOperand(0); 14753 Value *Op1 = SVI->getOperand(1); 14754 IRBuilder<> Builder(SI); 14755 14756 // StN intrinsics don't support pointer vectors as arguments. Convert pointer 14757 // vectors to integer vectors. 14758 if (EltTy->isPointerTy()) { 14759 Type *IntTy = DL.getIntPtrType(EltTy); 14760 14761 // Convert to the corresponding integer vector. 14762 Type *IntVecTy = 14763 VectorType::get(IntTy, Op0->getType()->getVectorNumElements()); 14764 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy); 14765 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy); 14766 14767 SubVecTy = VectorType::get(IntTy, LaneLen); 14768 } 14769 14770 // The base address of the store. 14771 Value *BaseAddr = SI->getPointerOperand(); 14772 14773 if (NumStores > 1) { 14774 // If we're going to generate more than one store, reset the lane length 14775 // and sub-vector type to something legal. 14776 LaneLen /= NumStores; 14777 SubVecTy = VectorType::get(SubVecTy->getVectorElementType(), LaneLen); 14778 14779 // We will compute the pointer operand of each store from the original base 14780 // address using GEPs. Cast the base address to a pointer to the scalar 14781 // element type. 14782 BaseAddr = Builder.CreateBitCast( 14783 BaseAddr, SubVecTy->getVectorElementType()->getPointerTo( 14784 SI->getPointerAddressSpace())); 14785 } 14786 14787 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!"); 14788 14789 auto Mask = SVI->getShuffleMask(); 14790 14791 Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace()); 14792 Type *Tys[] = {Int8Ptr, SubVecTy}; 14793 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2, 14794 Intrinsic::arm_neon_vst3, 14795 Intrinsic::arm_neon_vst4}; 14796 14797 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) { 14798 // If we generating more than one store, we compute the base address of 14799 // subsequent stores as an offset from the previous. 14800 if (StoreCount > 0) 14801 BaseAddr = Builder.CreateConstGEP1_32(BaseAddr, LaneLen * Factor); 14802 14803 SmallVector<Value *, 6> Ops; 14804 Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr)); 14805 14806 Function *VstNFunc = 14807 Intrinsic::getDeclaration(SI->getModule(), StoreInts[Factor - 2], Tys); 14808 14809 // Split the shufflevector operands into sub vectors for the new vstN call. 14810 for (unsigned i = 0; i < Factor; i++) { 14811 unsigned IdxI = StoreCount * LaneLen * Factor + i; 14812 if (Mask[IdxI] >= 0) { 14813 Ops.push_back(Builder.CreateShuffleVector( 14814 Op0, Op1, createSequentialMask(Builder, Mask[IdxI], LaneLen, 0))); 14815 } else { 14816 unsigned StartMask = 0; 14817 for (unsigned j = 1; j < LaneLen; j++) { 14818 unsigned IdxJ = StoreCount * LaneLen * Factor + j; 14819 if (Mask[IdxJ * Factor + IdxI] >= 0) { 14820 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ; 14821 break; 14822 } 14823 } 14824 // Note: If all elements in a chunk are undefs, StartMask=0! 14825 // Note: Filling undef gaps with random elements is ok, since 14826 // those elements were being written anyway (with undefs). 14827 // In the case of all undefs we're defaulting to using elems from 0 14828 // Note: StartMask cannot be negative, it's checked in 14829 // isReInterleaveMask 14830 Ops.push_back(Builder.CreateShuffleVector( 14831 Op0, Op1, createSequentialMask(Builder, StartMask, LaneLen, 0))); 14832 } 14833 } 14834 14835 Ops.push_back(Builder.getInt32(SI->getAlignment())); 14836 Builder.CreateCall(VstNFunc, Ops); 14837 } 14838 return true; 14839 } 14840 14841 enum HABaseType { 14842 HA_UNKNOWN = 0, 14843 HA_FLOAT, 14844 HA_DOUBLE, 14845 HA_VECT64, 14846 HA_VECT128 14847 }; 14848 14849 static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, 14850 uint64_t &Members) { 14851 if (auto *ST = dyn_cast<StructType>(Ty)) { 14852 for (unsigned i = 0; i < ST->getNumElements(); ++i) { 14853 uint64_t SubMembers = 0; 14854 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers)) 14855 return false; 14856 Members += SubMembers; 14857 } 14858 } else if (auto *AT = dyn_cast<ArrayType>(Ty)) { 14859 uint64_t SubMembers = 0; 14860 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers)) 14861 return false; 14862 Members += SubMembers * AT->getNumElements(); 14863 } else if (Ty->isFloatTy()) { 14864 if (Base != HA_UNKNOWN && Base != HA_FLOAT) 14865 return false; 14866 Members = 1; 14867 Base = HA_FLOAT; 14868 } else if (Ty->isDoubleTy()) { 14869 if (Base != HA_UNKNOWN && Base != HA_DOUBLE) 14870 return false; 14871 Members = 1; 14872 Base = HA_DOUBLE; 14873 } else if (auto *VT = dyn_cast<VectorType>(Ty)) { 14874 Members = 1; 14875 switch (Base) { 14876 case HA_FLOAT: 14877 case HA_DOUBLE: 14878 return false; 14879 case HA_VECT64: 14880 return VT->getBitWidth() == 64; 14881 case HA_VECT128: 14882 return VT->getBitWidth() == 128; 14883 case HA_UNKNOWN: 14884 switch (VT->getBitWidth()) { 14885 case 64: 14886 Base = HA_VECT64; 14887 return true; 14888 case 128: 14889 Base = HA_VECT128; 14890 return true; 14891 default: 14892 return false; 14893 } 14894 } 14895 } 14896 14897 return (Members > 0 && Members <= 4); 14898 } 14899 14900 /// Return the correct alignment for the current calling convention. 14901 unsigned 14902 ARMTargetLowering::getABIAlignmentForCallingConv(Type *ArgTy, 14903 DataLayout DL) const { 14904 if (!ArgTy->isVectorTy()) 14905 return DL.getABITypeAlignment(ArgTy); 14906 14907 // Avoid over-aligning vector parameters. It would require realigning the 14908 // stack and waste space for no real benefit. 14909 return std::min(DL.getABITypeAlignment(ArgTy), DL.getStackAlignment()); 14910 } 14911 14912 /// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of 14913 /// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when 14914 /// passing according to AAPCS rules. 14915 bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters( 14916 Type *Ty, CallingConv::ID CallConv, bool isVarArg) const { 14917 if (getEffectiveCallingConv(CallConv, isVarArg) != 14918 CallingConv::ARM_AAPCS_VFP) 14919 return false; 14920 14921 HABaseType Base = HA_UNKNOWN; 14922 uint64_t Members = 0; 14923 bool IsHA = isHomogeneousAggregate(Ty, Base, Members); 14924 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump()); 14925 14926 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy(); 14927 return IsHA || IsIntArray; 14928 } 14929 14930 unsigned ARMTargetLowering::getExceptionPointerRegister( 14931 const Constant *PersonalityFn) const { 14932 // Platforms which do not use SjLj EH may return values in these registers 14933 // via the personality function. 14934 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R0; 14935 } 14936 14937 unsigned ARMTargetLowering::getExceptionSelectorRegister( 14938 const Constant *PersonalityFn) const { 14939 // Platforms which do not use SjLj EH may return values in these registers 14940 // via the personality function. 14941 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R1; 14942 } 14943 14944 void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { 14945 // Update IsSplitCSR in ARMFunctionInfo. 14946 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>(); 14947 AFI->setIsSplitCSR(true); 14948 } 14949 14950 void ARMTargetLowering::insertCopiesSplitCSR( 14951 MachineBasicBlock *Entry, 14952 const SmallVectorImpl<MachineBasicBlock *> &Exits) const { 14953 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); 14954 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); 14955 if (!IStart) 14956 return; 14957 14958 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 14959 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); 14960 MachineBasicBlock::iterator MBBI = Entry->begin(); 14961 for (const MCPhysReg *I = IStart; *I; ++I) { 14962 const TargetRegisterClass *RC = nullptr; 14963 if (ARM::GPRRegClass.contains(*I)) 14964 RC = &ARM::GPRRegClass; 14965 else if (ARM::DPRRegClass.contains(*I)) 14966 RC = &ARM::DPRRegClass; 14967 else 14968 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 14969 14970 unsigned NewVR = MRI->createVirtualRegister(RC); 14971 // Create copy from CSR to a virtual register. 14972 // FIXME: this currently does not emit CFI pseudo-instructions, it works 14973 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be 14974 // nounwind. If we want to generalize this later, we may need to emit 14975 // CFI pseudo-instructions. 14976 assert(Entry->getParent()->getFunction().hasFnAttribute( 14977 Attribute::NoUnwind) && 14978 "Function should be nounwind in insertCopiesSplitCSR!"); 14979 Entry->addLiveIn(*I); 14980 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) 14981 .addReg(*I); 14982 14983 // Insert the copy-back instructions right before the terminator. 14984 for (auto *Exit : Exits) 14985 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), 14986 TII->get(TargetOpcode::COPY), *I) 14987 .addReg(NewVR); 14988 } 14989 } 14990 14991 void ARMTargetLowering::finalizeLowering(MachineFunction &MF) const { 14992 MF.getFrameInfo().computeMaxCallFrameSize(MF); 14993 TargetLoweringBase::finalizeLowering(MF); 14994 } 14995
